From 52cb0c24a6a6674ee17c4bb86fa02527043ed90f Mon Sep 17 00:00:00 2001 From: Michael Widenius Date: Thu, 26 Jun 2008 08:18:28 +0300 Subject: Added versioning of Maria index Store max_trid in index file as state.create_trid. This is used to pack all transids in the index pages relative to max possible transid for file. Enable versioning for transactional tables with index. Tables with an auto-increment key, rtree or fulltext keys are not versioned. Changed info->lastkey to type MARIA_KEY. Removed info->lastkey_length as this is now part of info->lastkey Renamed old info->lastkey to info->lastkey_buff Use exact key lenghts for keys, not USE_WHOLE_KEY For partial key searches, use SEARCH_PART_KEY When searching to insert new key on page, use SEARCH_INSERT to mark that key has rowid Changes done in a lot of files: - Modified functions to use MARIA_KEY instead of key pointer and key length - Use keyinfo->root_lock instead of share->key_root_lock[keynr] - Simplify code by using local variable keyinfo instead if share->keyinfo[i] - Added #fdef EXTERNAL_LOCKING around removed state elements - HA_MAX_KEY_BUFF -> MARIA_MAX_KEY_BUFF (to reserve space for transid) - Changed type of 'nextflag' to uint32 to ensure all SEARCH_xxx flags fits into it .bzrignore: Added missing temporary directory extra/Makefile.am: comp_err is now deleted on make distclean include/maria.h: Added structure MARIA_KEY, which is used for intern key objects in Maria. Changed functions to take MARIA_KEY as an argument instead of pointer to packed key. Changed some functions that always return true or false to my_bool. Added virtual function make_key() to avoid if in _ma_make_key() Moved rw_lock_t for locking trees from share->key_root_lock to MARIA_KEYDEF. This makes usage of the locks simpler and faster include/my_base.h: Added HA_RTREE_INDEX flag to mark rtree index. Used for easier checks in ma_check() Added SEARCH_INSERT to be used when inserting new keys Added SEARCH_PART_KEY for partial searches Added SEARCH_USER_KEY_HAS_TRANSID to be used when key we use for searching in btree has a TRANSID Added SEARCH_PAGE_KEY_HAS_TRANSID to be used when key we found in btree has a transid include/my_handler.h: Make next_flag 32 bit to make sure we can handle all SEARCH_ bits mysql-test/include/maria_empty_logs.inc: Read and restore current database; Don't assume we are using mysqltest. Don't log use databasename to log. Using this include should not cause any result changes. mysql-test/r/maria-gis-rtree-dynamic.result: Updated results after adding some check table commands to help pinpoint errors mysql-test/r/maria-mvcc.result: New tests mysql-test/r/maria-purge.result: New result after adding removal of logs mysql-test/r/maria-recovery-big.result: maria_empty_logs doesn't log 'use mysqltest' anymore mysql-test/r/maria-recovery-bitmap.result: maria_empty_logs doesn't log 'use mysqltest' anymore mysql-test/r/maria-recovery-rtree-ft.result: maria_empty_logs doesn't log 'use mysqltest' anymore mysql-test/r/maria-recovery.result: maria_empty_logs doesn't log 'use mysqltest' anymore mysql-test/r/maria.result: New tests mysql-test/r/variables-big.result: Don't log id as it's not predictable mysql-test/suite/rpl_ndb/r/rpl_truncate_7ndb_2.result: Updated results to new binlog results. (Test has not been run in a long time as it requires --big) mysql-test/suite/rpl_ndb/t/rpl_truncate_7ndb_2-master.opt: Moved file to ndb replication test directory mysql-test/suite/rpl_ndb/t/rpl_truncate_7ndb_2.test: Fixed wrong path to included tests mysql-test/t/maria-gis-rtree-dynamic.test: Added some check table commands to help pinpoint errors mysql-test/t/maria-mvcc.test: New tests mysql-test/t/maria-purge.test: Remove logs to make test results predictable mysql-test/t/maria.test: New tests for some possible problems mysql-test/t/variables-big.test: Don't log id as it's not predictable mysys/my_handler.c: Updated function comment to reflect old code Changed nextflag to be uint32 to ensure we can have flags > 16 bit Changed checking if we are in insert with NULL keys as next_flag can now include additional bits that have to be ignored. Added SEARCH_INSERT flag to be used when inserting new keys in btree. This flag tells us the that the keys includes row position and it's thus safe to remove SEARCH_FIND Added comparision of transid. This is only done if the keys actually have a transid, which is indicated by nextflag mysys/my_lock.c: Fixed wrong test (Found by Guilhem) scripts/Makefile.am: Ensure that test programs are deleted by make clean sql/rpl_rli.cc: Moved assignment order to fix compiler warning storage/heap/hp_write.c: Add SEARCH_INSERT to signal ha_key_cmp that we we should also compare rowid for keys storage/maria/Makefile.am: Remove also maria log files when doing make distclean storage/maria/ha_maria.cc: Use 'file->start_state' as default state for transactional tables without versioning At table unlock, set file->state to point to live state. (Needed for information schema to pick up right number of rows) In ha_maria::implicit_commit() move all locked (ie open) tables to new transaction. This is needed to ensure ha_maria->info doesn't point to a deleted history event. Disable concurrent inserts for insert ... select and table changes with subqueries if statement based replication as this would cause wrong results on slave storage/maria/ma_blockrec.c: Updated comment storage/maria/ma_check.c: Compact key pages (removes transid) when doing --zerofill Check that 'page_flag' on key pages contains KEYPAGE_FLAG_HAS_TRANSID if there is a single key on the page with a transid Modified functions to use MARIA_KEY instead of key pointer and key length Use new interface to _ma_rec_pos(), _ma_dpointer(), _ma_ft_del(), ma_update_state_lsn() Removed not needed argument from get_record_for_key() Fixed that we check doesn't give errors for RTREE; We now treath these like SPATIAL Remove some SPATIAL specific code where the virtual functions can handle this in a general manner Use info->lastkey_buff instead of info->lastkey _ma_dpos() -> _ma_row_pos_from_key() _ma_make_key() -> keyinfo->make_key() _ma_print_key() -> _ma_print_keydata() _ma_move_key() -> ma_copy_copy() Add SEARCH_INSERT to signal ha_key_cmp that we we should also compare rowid for keys Ensure that data on page doesn't overwrite page checksum position Use DBUG_DUMP_KEY instead of DBUG_DUMP Use exact key lengths instead of USE_WHOLE_KEY to ha_key_cmp() Fixed check if rowid points outside of BLOCK_RECORD data file Use info->lastkey_buff instead of key on stack in some safe places Added #fdef EXTERNAL_LOCKING around removed state elements storage/maria/ma_close.c: Use keyinfo->root_lock instead of share->key_root_lock[keynr] storage/maria/ma_create.c: Removed assert that is already checked in maria_init() Force transactinal tables to be of type BLOCK_RECORD Fixed wrong usage of HA_PACK_RECORD (should be HA_OPTION_PACK_RECORD) Mark keys that uses HA_KEY_ALG_RTREE with HA_RTREE_INDEX for easier handling of these in ma_check Store max_trid in index file as state.create_trid. This is used to pack all transids in the index pages relative to max possible transid for file. storage/maria/ma_dbug.c: Changed _ma_print_key() to use MARIA_KEY storage/maria/ma_delete.c: Modified functions to use MARIA_KEY instead of key pointer and key length info->lastkey2-> info->lastkey_buff2 Added SEARCH_INSERT to signal ha_key_cmp that we we should also compare rowid for keys Use new interface for get_key(), _ma_get_last_key() and others _ma_dpos() -> ma_row_pos_from_key() Simplify setting of prev_key in del() Ensure that KEYPAGE_FLAG_HAS_TRANSID is set in page_flag if key page has transid Treath key pages that may have a transid as if keys would be of variable length storage/maria/ma_delete_all.c: Reset history state if maria_delete_all_rows() are called Update parameters to _ma_update_state_lsns() call storage/maria/ma_extra.c: Store and restore info->lastkey storage/maria/ma_ft_boolean_search.c: Modified functions to use MARIA_KEY instead of key pointer and key length storage/maria/ma_ft_nlq_search.c: Modified functions to use MARIA_KEY instead of key pointer and key length Use lastkey_buff2 instead of info->lastkey+info->s->base.max_key_length (same thing) storage/maria/ma_ft_update.c: Modified functions to use MARIA_KEY instead of key pointer and key length storage/maria/ma_ftdefs.h: Modified functions to use MARIA_KEY instead of key pointer and key length storage/maria/ma_fulltext.h: Modified functions to use MARIA_KEY instead of key pointer and key length storage/maria/ma_init.c: Check if blocksize is legal (Moved test here from ma_open()) storage/maria/ma_key.c: Added functions for storing/reading of transid Modified functions to use MARIA_KEY instead of key pointer and key length Moved _ma_sp_make_key() out of _ma_make_key() as we now use keyinfo->make_key to create keys Add transid to keys if table is versioned Added _ma_copy_key() storage/maria/ma_key_recover.c: Add logging of page_flag (holds information if there are keys with transid on page) Changed DBUG_PRINT("info" -> DBUG_PRINT("redo" as the redo logging can be quite extensive Added lots of DBUG_PRINT() Added support for index page operations: KEY_OP_SET_PAGEFLAG and KEY_OP_COMPACT_PAGE storage/maria/ma_key_recover.h: Modified functions to use MARIA_KEY instead of key pointer and key length storage/maria/ma_locking.c: Added new arguments to _ma_update_state_lsns_sub() storage/maria/ma_loghandler.c: Fixed all logging of LSN to look similar in DBUG log Changed if (left != 0) to if (left) as the later is used also later in the code storage/maria/ma_loghandler.h: Added new index page operations storage/maria/ma_open.c: Removed allocated "state_dummy" and instead use share->state.common for transactional tables that are not versioned This is needed to not get double increments of state.records (one in ma_write.c and on when log is written) Changed info->lastkey to MARIA_KEY type Removed resetting of MARIA_HA variables that have 0 as default value (as info is zerofilled) Enable versioning for transactional tables with index. Tables with an auto-increment key, rtree or fulltext keys are not versioned. Check on open that state.create_trid is correct Extend share->base.max_key_length in case of transactional table so that it can hold transid Removed 4.0 compatible fulltext key mode as this is not relevant for Maria Removed old and wrong #ifdef ENABLE_WHEN_WE_HAVE_TRANS_ROW_ID code block Initialize all new virtual function pointers Removed storing of state->unique, state->process and store state->create_trid instead storage/maria/ma_page.c: Added comment to describe key page structure Added functions to compact key page and log the compact operation storage/maria/ma_range.c: Modified functions to use MARIA_KEY instead of key pointer and key length Use SEARCH_PART_KEY indicator instead of USE_WHOLE_KEY to detect if we are doing a part key search Added handling of pages with transid storage/maria/ma_recovery.c: Don't assert if table we opened are not transactional. This may be a table which has been changed from transactional to not transactinal Added new arguments to _ma_update_state_lsns() storage/maria/ma_rename.c: Added new arguments to _ma_update_state_lsns() storage/maria/ma_rkey.c: Modified functions to use MARIA_KEY instead of key pointer and key length Don't use USE_WHOLE_KEY, use real length of key Use share->row_is_visible() to test if row is visible Moved search_flag == HA_READ_KEY_EXACT out of 'read-next-row' loop as this only need to be tested once Removed test if last_used_keyseg != 0 as this is always true storage/maria/ma_rnext.c: Modified functions to use MARIA_KEY instead of key pointer and key length Simplify code by using local variable keyinfo instead if share->keyinfo[i] Use share->row_is_visible() to test if row is visible storage/maria/ma_rnext_same.c: Modified functions to use MARIA_KEY instead of key pointer and key length lastkey2 -> lastkey_buff2 storage/maria/ma_rprev.c: Modified functions to use MARIA_KEY instead of key pointer and key length Simplify code by using local variable keyinfo instead if share->keyinfo[i] Use share->row_is_visible() to test if row is visible storage/maria/ma_rsame.c: Updated comment Simplify code by using local variable keyinfo instead if share->keyinfo[i] Modified functions to use MARIA_KEY instead of key pointer and key length storage/maria/ma_rsamepos.c: Modified functions to use MARIA_KEY instead of key pointer and key length storage/maria/ma_rt_index.c: Modified functions to use MARIA_KEY instead of key pointer and key length Use better variable names Removed not needed casts _ma_dpos() -> _ma_row_pos_from_key() Use info->last_rtree_keypos to save position to key instead of info->int_keypos Simplify err: condition Changed return type for maria_rtree_insert() to my_bool as we are only intressed in ok/fail from this function storage/maria/ma_rt_index.h: Modified functions to use MARIA_KEY instead of key pointer and key length storage/maria/ma_rt_key.c: Modified functions to use MARIA_KEY instead of key pointer and key length Simplify maria_rtree_add_key by combining idenitcal code and removing added_len storage/maria/ma_rt_key.h: Modified functions to use MARIA_KEY instead of key pointer and key length storage/maria/ma_rt_mbr.c: Changed type of 'nextflag' to uint32 Added 'to' argument to RT_PAGE_MBR_XXX functions to more clearly see which variables changes value storage/maria/ma_rt_mbr.h: Changed type of 'nextflag' to uint32 storage/maria/ma_rt_split.c: Modified functions to use MARIA_KEY instead of key pointer and key length key_length -> key_data_length to catch possible errors storage/maria/ma_rt_test.c: Fixed wrong comment Reset recinfo to avoid valgrind varnings Fixed wrong argument to create_record() that caused test to fail storage/maria/ma_search.c: Modified functions to use MARIA_KEY instead of key pointer and key length Added support of keys with optional trid Test for SEARCH_PART_KEY instead of USE_WHOLE_KEY to detect part key reads _ma_dpos() -> _ma_row_pos_from_key() If there may be keys with transid on the page, have _ma_bin_search() call _ma_seq_search() Add _ma_skip_xxx() functions to quickly step over keys (faster than calling get_key() in most cases as we don't have to copy key data) Combine similar code at end of _ma_get_binary_pack_key() Removed not used function _ma_move_key() In _ma_search_next() don't call _ma_search() if we aren't on a nod page. Update info->cur_row.trid with trid for found key Removed some not needed casts Added _ma_trid_from_key() Use MARIA_SHARE instead of MARIA_HA as arguments to _ma_rec_pos(), _ma_dpointer() and _ma_xxx_keypos_to_recpos() to make functions faster and smaller storage/maria/ma_sort.c: Modified functions to use MARIA_KEY instead of key pointer and key length storage/maria/ma_sp_defs.h: _ma_sp_make_key() now fills in and returns (MARIA_KEY *) value storage/maria/ma_sp_key.c: _ma_sp_make_key() now fills in and returns (MARIA_KEY *) value Don't test sizeof(double), test against 8 as we are using float8store() Use mi_float8store() instead of doing swap of value (same thing but faster) storage/maria/ma_state.c: maria_versioning() now only calls _ma_block_get_status() if table supports versioning Added _ma_row_visible_xxx() functions for different occasions When emptying history, set info->state to point to the first history event. storage/maria/ma_state.h: Added _ma_row_visible_xxx() prototypes storage/maria/ma_static.c: Indentation changes storage/maria/ma_statrec.c: Fixed arguments to _ma_dpointer() and _ma_rec_pos() storage/maria/ma_test1.c: Call init_thr_lock() if we have versioning storage/maria/ma_test2.c: Call init_thr_lock() if we have versioning storage/maria/ma_unique.c: Modified functions to use MARIA_KEY storage/maria/ma_update.c: Modified functions to use MARIA_KEY instead of key pointer and key length storage/maria/ma_write.c: Modified functions to use MARIA_KEY instead of key pointer and key length Simplify code by using local variable keyinfo instead if share->keyinfo[i] In _ma_enlarge_root(), mark in page_flag if new key has transid _ma_dpos() -> _ma_row_pos_from_key() Changed return type of _ma_ck_write_tree() to my_bool as we are only testing if result is true or not Moved 'reversed' to outside block as area was used later storage/maria/maria_chk.c: Added error if trying to sort with HA_BINARY_PACK_KEY Use new interface to get_key() and _ma_dpointer() _ma_dpos() -> _ma_row_pos_from_key() storage/maria/maria_def.h: Modified functions to use MARIA_KEY instead of key pointer and key length Added 'common' to MARIA_SHARE->state for storing state for transactional tables without versioning Added create_trid to MARIA_SHARE Removed not used state variables 'process' and 'unique' Added defines for handling TRID's in index pages Changed to use MARIA_SHARE instead of MARIA_HA for some functions Added 'have_versioning' flag if table supports versioning Moved key_root_lock from MARIA_SHARE to MARIA_KEYDEF Changed last_key to be of type MARIA_KEY. Removed lastkey_length lastkey -> lastkey_buff, lastkey2 -> lastkey_buff2 Added _ma_get_used_and_nod_with_flag() for faster access to page data when page_flag is read Added DBUG_DUMP_KEY for easier DBUG_DUMP of a key Changed 'nextflag' and assocaited variables to uint32 storage/maria/maria_ftdump.c: lastkey -> lastkey_buff storage/maria/trnman.c: Fixed wrong initialization of min_read_from and max_commit_trid Added trnman_get_min_safe_trid() storage/maria/unittest/ma_test_all-t: Added --start-from storage/myisam/mi_check.c: Added SEARCH_INSERT, as ha_key_cmp() needs it when doing key comparision for inserting key on page in rowid order storage/myisam/mi_delete.c: Added SEARCH_INSERT, as ha_key_cmp() needs it when doing key comparision for inserting key on page in rowid order storage/myisam/mi_range.c: Updated comment storage/myisam/mi_write.c: Added SEARCH_INSERT, as ha_key_cmp() needs it when doing key comparision for inserting key on page in rowid order storage/myisam/rt_index.c: Fixed wrong parameter to rtree_get_req() which could cause crash --- storage/maria/ma_page.c | 176 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 174 insertions(+), 2 deletions(-) (limited to 'storage/maria/ma_page.c') diff --git a/storage/maria/ma_page.c b/storage/maria/ma_page.c index 24a97409aa3..2cbaf3ec690 100644 --- a/storage/maria/ma_page.c +++ b/storage/maria/ma_page.c @@ -13,7 +13,32 @@ along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ -/* Read and write key blocks */ +/* + Read and write key blocks + + The basic structure of a key block is as follows: + + LSN 7 (LSN_STORE_SIZE); Log number for last change; + Only for transactional pages + PACK_TRANSID 6 (TRANSID_SIZE); Relative transid to pack page transid's + Only for transactional pages + KEYNR 1 (KEYPAGE_KEYID_SIZE) Which index this page belongs to + FLAG 1 (KEYPAGE_FLAG_SIZE) Flags for page + PAGE_SIZE 2 (KEYPAGE_USED_SIZE) How much of the page is used. + high-byte-first + + The flag is a combination of the following values: + + KEYPAGE_FLAG_ISNOD Page is a node + KEYPAGE_FLAG_HAS_TRANSID There may be a transid on the page. + + After this we store key data, either packed or not packed, directly + after each other. If the page is a node flag, there is a pointer to + the next key page at page start and after each key. + + At end of page the last KEYPAGE_CHECKSUM_SIZE bytes are reserved for a + page checksum. +*/ #include "maria_def.h" #include "trnman.h" @@ -163,7 +188,7 @@ int _ma_write_keypage(register MARIA_HA *info, } -/* +/** @brief Put page in free list @fn _ma_dispose() @@ -366,3 +391,150 @@ my_off_t _ma_new(register MARIA_HA *info, int level, DBUG_PRINT("exit",("Pos: %ld",(long) pos)); DBUG_RETURN(pos); } /* _ma_new */ + + +/** + Log compactation of a index page +*/ + +static my_bool _ma_log_compact_keypage(MARIA_HA *info, my_off_t page, + TrID min_read_from) +{ + LSN lsn; + uchar log_data[FILEID_STORE_SIZE + PAGE_STORE_SIZE + 1 + TRANSID_SIZE]; + LEX_CUSTRING log_array[TRANSLOG_INTERNAL_PARTS + 1]; + MARIA_SHARE *share= info->s; + DBUG_ENTER("_ma_log_compact_keypage"); + DBUG_PRINT("enter", ("page: %lu", (ulong) page)); + + /* Store address of new root page */ + page/= share->block_size; + page_store(log_data + FILEID_STORE_SIZE, page); + + log_data[FILEID_STORE_SIZE + PAGE_STORE_SIZE]= KEY_OP_COMPACT_PAGE; + transid_store(log_data + FILEID_STORE_SIZE + PAGE_STORE_SIZE +1, + min_read_from); + + log_array[TRANSLOG_INTERNAL_PARTS + 0].str= log_data; + log_array[TRANSLOG_INTERNAL_PARTS + 0].length= sizeof(log_data); + + if (translog_write_record(&lsn, LOGREC_REDO_INDEX, + info->trn, info, + (translog_size_t) sizeof(log_data), + TRANSLOG_INTERNAL_PARTS + 1, log_array, + log_data, NULL)) + DBUG_RETURN(1); + DBUG_RETURN(0); +} + + +/** + Remove all transaction id's less than given one from a key page + + @fn _ma_compact_keypage() + @param keyinfo Key handler + @param page_pos Page position on disk + @param page Buffer for page + @param min_read_from Remove all trids from page less than this + + @retval 0 Ok + Ūretval 1 Error; my_errno contains the error +*/ + +my_bool _ma_compact_keypage(MARIA_HA *info, MARIA_KEYDEF *keyinfo, + my_off_t page_pos, uchar *page, TrID min_read_from) +{ + MARIA_SHARE *share= keyinfo->share; + MARIA_KEY key; + uchar *start_of_page, *endpos, *start_of_empty_space; + uint page_flag, nod_flag, saved_space; + my_bool page_has_transid; + DBUG_ENTER("_ma_compact_keypage"); + + page_flag= _ma_get_keypage_flag(share, page); + if (!(page_flag & KEYPAGE_FLAG_HAS_TRANSID)) + DBUG_RETURN(0); /* No transaction id on page */ + + nod_flag= _ma_test_if_nod(share, page); + endpos= page + _ma_get_page_used(share, page); + key.data= info->lastkey_buff; + key.keyinfo= keyinfo; + + start_of_page= page; + page_has_transid= 0; + page+= share->keypage_header + nod_flag; + key.data[0]= 0; /* safety */ + start_of_empty_space= 0; + saved_space= 0; + do + { + if (!(page= (*keyinfo->skip_key)(&key, 0, 0, page))) + { + DBUG_PRINT("error",("Couldn't find last key: page: 0x%lx", + (long) page)); + maria_print_error(share, HA_ERR_CRASHED); + my_errno=HA_ERR_CRASHED; + DBUG_RETURN(1); + } + if (key_has_transid(page-1)) + { + uint transid_length; + transid_length= transid_packed_length(page); + + if (min_read_from == ~(TrID) 0 || + min_read_from < transid_get_packed(share, page)) + { + page[-1]&= 254; /* Remove transid marker */ + transid_length= transid_packed_length(page); + if (start_of_empty_space) + { + /* Move block before the transid up in page */ + uint copy_length= (uint) (page - start_of_empty_space) - saved_space; + memmove(start_of_empty_space, start_of_empty_space + saved_space, + copy_length); + start_of_empty_space+= copy_length; + } + else + start_of_empty_space= page; + saved_space+= transid_length; + } + else + page_has_transid= 1; /* At least one id left */ + page+= transid_length; + } + page+= nod_flag; + } while (page < endpos); + + DBUG_ASSERT(page == endpos); + + if (start_of_empty_space) + { + /* + Move last block down + This is always true if any transid was removed + */ + uint copy_length= (uint) (endpos - start_of_empty_space) - saved_space; + uint page_length; + + if (copy_length) + memmove(start_of_empty_space, start_of_empty_space + saved_space, + copy_length); + page_length= (uint) (start_of_empty_space + copy_length - start_of_page); + _ma_store_page_used(share, start_of_page, page_length); + } + + if (!page_has_transid) + { + page_flag&= ~KEYPAGE_FLAG_HAS_TRANSID; + _ma_store_keypage_flag(share, start_of_page, page_flag); + /* Clear packed transid (in case of zerofill) */ + bzero(start_of_page + LSN_STORE_SIZE, TRANSID_SIZE); + } + + if (share->now_transactional) + { + if (_ma_log_compact_keypage(info, page_pos, min_read_from)) + DBUG_RETURN(1); + } + DBUG_RETURN(0); +} -- cgit v1.2.1