diff options
Diffstat (limited to 'storage/maria')
43 files changed, 1367 insertions, 729 deletions
diff --git a/storage/maria/CMakeLists.txt b/storage/maria/CMakeLists.txt index 0f30f8f3156..b8e3350ca76 100644 --- a/storage/maria/CMakeLists.txt +++ b/storage/maria/CMakeLists.txt @@ -15,6 +15,14 @@ INCLUDE(CMakeDependentOption) +INCLUDE_DIRECTORIES( +${SSL_INCLUDE_DIRS} +) + +IF(SSL_DEFINES) +SET_SOURCE_FILES_PROPERTIES(ma_crypt.c PROPERTIES COMPILE_FLAGS ${SSL_DEFINES}) +ENDIF() + SET(ARIA_SOURCES ma_init.c ma_open.c ma_extra.c ma_info.c ma_rkey.c ma_rnext.c ma_rnext_same.c ma_search.c ma_page.c ma_key_recover.c ma_key.c @@ -39,6 +47,7 @@ SET(ARIA_SOURCES ma_init.c ma_open.c ma_extra.c ma_info.c ma_rkey.c ma_checkpoint.c ma_recovery.c ma_commit.c ma_pagecrc.c ha_maria.h maria_def.h ma_recovery_util.c ma_servicethread.c ma_norec.c + ma_crypt.c ) IF(APPLE) @@ -54,7 +63,8 @@ IF(NOT WITH_ARIA_STORAGE_ENGINE) RETURN() ENDIF() -TARGET_LINK_LIBRARIES(aria myisam) +TARGET_LINK_LIBRARIES(aria myisam + mysys mysys_ssl) MYSQL_ADD_EXECUTABLE(aria_ftdump maria_ftdump.c COMPONENT Server) TARGET_LINK_LIBRARIES(aria_ftdump aria) diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index 14ab6ec7599..5fdd2a920a5 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -217,9 +217,7 @@ static MYSQL_SYSVAR_ULONG(group_commit_interval, maria_group_commit_interval, static MYSQL_SYSVAR_ENUM(log_purge_type, log_purge_type, PLUGIN_VAR_RQCMDARG, - "Specifies how Aria transactional log will be purged. " - "Possible values of name are \"immediate\", \"external\" " - "and \"at_flush\"", + "Specifies how Aria transactional log will be purged", NULL, NULL, TRANSLOG_PURGE_IMMIDIATE, &maria_translog_purge_type_typelib); @@ -258,9 +256,7 @@ static MYSQL_SYSVAR_ULONG(pagecache_file_hash_size, pagecache_file_hash_size, 512, 128, 16384, 1); static MYSQL_SYSVAR_SET(recover, maria_recover_options, PLUGIN_VAR_OPCMDARG, - "Specifies how corrupted tables should be automatically repaired." - " Possible values are one or more of \"NORMAL\" (the default), " - "\"BACKUP\", \"FORCE\", or \"QUICK\".", + "Specifies how corrupted tables should be automatically repaired", NULL, NULL, HA_RECOVER_DEFAULT, &maria_recover_typelib); static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG, @@ -275,13 +271,11 @@ static MYSQL_THDVAR_ULONGLONG(sort_buffer_size, PLUGIN_VAR_RQCMDARG, static MYSQL_THDVAR_ENUM(stats_method, PLUGIN_VAR_RQCMDARG, "Specifies how Aria index statistics collection code should treat " - "NULLs. Possible values are \"nulls_unequal\", \"nulls_equal\", " - "and \"nulls_ignored\".", 0, 0, 0, &maria_stats_method_typelib); + "NULLs", 0, 0, 0, &maria_stats_method_typelib); static MYSQL_SYSVAR_ENUM(sync_log_dir, sync_log_dir, PLUGIN_VAR_RQCMDARG, "Controls syncing directory after log file growth and new file " - "creation. Possible values are \"never\", \"newfile\" and " - "\"always\").", NULL, NULL, TRANSLOG_SYNC_DIR_NEWFILE, + "creation", NULL, NULL, TRANSLOG_SYNC_DIR_NEWFILE, &maria_sync_log_dir_typelib); #ifdef USE_ARIA_FOR_TMP_TABLES @@ -296,6 +290,11 @@ static MYSQL_SYSVAR_BOOL(used_for_temp_tables, "Whether temporary tables should be MyISAM or Aria", 0, 0, 1); +static MYSQL_SYSVAR_BOOL(encrypt_tables, maria_encrypt_tables, PLUGIN_VAR_OPCMDARG, + "Encrypt tables (only for tables with ROW_FORMAT=PAGE (default) " + "and not FIXED/DYNAMIC)", + 0, 0, 0); + #ifdef HAVE_PSI_INTERFACE static PSI_mutex_info all_aria_mutexes[]= @@ -1598,7 +1597,7 @@ int ha_maria::repair(THD *thd, HA_CHECK *param, bool do_optimize) param->db_name= table->s->db.str; param->table_name= table->alias.c_ptr(); - param->tmpfile_createflag= O_RDWR | O_TRUNC | O_EXCL; + param->tmpfile_createflag= O_RDWR | O_TRUNC; param->using_global_keycache= 1; param->thd= thd; param->tmpdir= &mysql_tmpdir_list; @@ -1615,7 +1614,7 @@ int ha_maria::repair(THD *thd, HA_CHECK *param, bool do_optimize) locking= 1; if (maria_lock_database(file, table->s->tmp_table ? F_EXTRA_LCK : F_WRLCK)) { - _ma_check_print_error(param, ER(ER_CANT_LOCK), my_errno); + _ma_check_print_error(param, ER_THD(thd, ER_CANT_LOCK), my_errno); DBUG_RETURN(HA_ADMIN_FAILED); } } @@ -1689,7 +1688,7 @@ int ha_maria::repair(THD *thd, HA_CHECK *param, bool do_optimize) thd_proc_info(thd, "Sorting index"); error= maria_sort_index(param, file, fixed_name); } - if (!statistics_done && (local_testflag & T_STATISTICS)) + if (!error && !statistics_done && (local_testflag & T_STATISTICS)) { if (share->state.changed & STATE_NOT_ANALYZED) { @@ -2323,7 +2322,6 @@ int ha_maria::index_read_map(uchar * buf, const uchar * key, { DBUG_ASSERT(inited == INDEX); int error= maria_rkey(file, buf, active_index, key, keypart_map, find_flag); - table->status= error ? STATUS_NOT_FOUND : 0; return error; } @@ -2341,7 +2339,6 @@ int ha_maria::index_read_idx_map(uchar * buf, uint index, const uchar * key, error= maria_rkey(file, buf, index, key, keypart_map, find_flag); ma_set_index_cond_func(file, NULL, 0); - table->status= error ? STATUS_NOT_FOUND : 0; return error; } @@ -2353,7 +2350,6 @@ int ha_maria::index_read_last_map(uchar * buf, const uchar * key, DBUG_ASSERT(inited == INDEX); int error= maria_rkey(file, buf, active_index, key, keypart_map, HA_READ_PREFIX_LAST); - table->status= error ? STATUS_NOT_FOUND : 0; DBUG_RETURN(error); } @@ -2362,7 +2358,6 @@ int ha_maria::index_next(uchar * buf) { DBUG_ASSERT(inited == INDEX); int error= maria_rnext(file, buf, active_index); - table->status= error ? STATUS_NOT_FOUND : 0; return error; } @@ -2371,7 +2366,6 @@ int ha_maria::index_prev(uchar * buf) { DBUG_ASSERT(inited == INDEX); int error= maria_rprev(file, buf, active_index); - table->status= error ? STATUS_NOT_FOUND : 0; return error; } @@ -2380,7 +2374,6 @@ int ha_maria::index_first(uchar * buf) { DBUG_ASSERT(inited == INDEX); int error= maria_rfirst(file, buf, active_index); - table->status= error ? STATUS_NOT_FOUND : 0; return error; } @@ -2389,7 +2382,6 @@ int ha_maria::index_last(uchar * buf) { DBUG_ASSERT(inited == INDEX); int error= maria_rlast(file, buf, active_index); - table->status= error ? STATUS_NOT_FOUND : 0; return error; } @@ -2408,7 +2400,6 @@ int ha_maria::index_next_same(uchar * buf, { error= maria_rnext_same(file,buf); } while (error == HA_ERR_RECORD_DELETED); - table->status= error ? STATUS_NOT_FOUND : 0; return error; } @@ -2452,7 +2443,6 @@ int ha_maria::rnd_end() int ha_maria::rnd_next(uchar *buf) { int error= maria_scan(file, buf); - table->status= error ? STATUS_NOT_FOUND : 0; return error; } @@ -2475,7 +2465,6 @@ int ha_maria::restart_rnd_next(uchar *buf) int ha_maria::rnd_pos(uchar *buf, uchar *pos) { int error= maria_rrnd(file, buf, my_get_ptr(pos, ref_length)); - table->status= error ? STATUS_NOT_FOUND : 0; return error; } @@ -2666,7 +2655,7 @@ void ha_maria::drop_table(const char *name) { DBUG_ASSERT(file->s->temporary); (void) ha_close(); - (void) maria_delete_table_files(name, 0); + (void) maria_delete_table_files(name, 1, 0); } @@ -3267,7 +3256,6 @@ int ha_maria::ft_read(uchar * buf) error= ft_handler->please->read_next(ft_handler, (char*) buf); - table->status= error ? STATUS_NOT_FOUND : 0; return error; } @@ -3719,6 +3707,7 @@ struct st_mysql_sys_var* system_variables[]= { MYSQL_SYSVAR(stats_method), MYSQL_SYSVAR(sync_log_dir), MYSQL_SYSVAR(used_for_temp_tables), + MYSQL_SYSVAR(encrypt_tables), NULL }; diff --git a/storage/maria/lockman.c b/storage/maria/lockman.c index aa030b6f57a..f319fea1e0e 100644 --- a/storage/maria/lockman.c +++ b/storage/maria/lockman.c @@ -264,10 +264,10 @@ retry: compatible= TRUE; upgrading= FALSE; cursor->blocker= cursor->upgrade_from= 0; - _lf_unpin(pins, 3); + lf_unpin(pins, 3); do { cursor->curr= PTR(*cursor->prev); - _lf_pin(pins, 1, cursor->curr); + lf_pin(pins, 1, cursor->curr); } while(*cursor->prev != (intptr)cursor->curr && LF_BACKOFF); for (;;) { @@ -276,7 +276,7 @@ retry: do { cur_link= cursor->curr->link; cursor->next= PTR(cur_link); - _lf_pin(pins, 0, cursor->next); + lf_pin(pins, 0, cursor->next); } while (cur_link != cursor->curr->link && LF_BACKOFF); cur_hashnr= cursor->curr->hashnr; cur_resource= cursor->curr->resource; @@ -316,7 +316,7 @@ retry: if (prev_active && !cur_active) { cursor->blocker= cursor->curr; - _lf_pin(pins, 3, cursor->curr); + lf_pin(pins, 3, cursor->curr); } if (cur_loid == loid) { @@ -329,7 +329,7 @@ retry: if (cur_active) { cursor->blocker= cursor->curr; /* loose-locks! */ - _lf_unpin(pins, 3); /* loose-locks! */ + lf_unpin(pins, 3); /* loose-locks! */ return ALREADY_HAVE_THE_LOCK; } else @@ -345,7 +345,7 @@ retry: { compatible= FALSE; cursor->blocker= cursor->curr; - _lf_pin(pins, 3, cursor->curr); + lf_pin(pins, 3, cursor->curr); } } prev_lock= lock_combining_matrix[prev_lock][cur_lock]; @@ -353,13 +353,13 @@ retry: } } cursor->prev= &(cursor->curr->link); - _lf_pin(pins, 2, cursor->curr); + lf_pin(pins, 2, cursor->curr); } else { if (my_atomic_casptr((void **)cursor->prev, (void **)(char*) &cursor->curr, cursor->next)) - _lf_alloc_free(pins, cursor->curr); + lf_alloc_free(pins, cursor->curr); else { (void)LF_BACKOFF; @@ -367,7 +367,7 @@ retry: } } cursor->curr= cursor->next; - _lf_pin(pins, 1, cursor->curr); + lf_pin(pins, 1, cursor->curr); } /* either the end of lock list - no more locks for this resource, @@ -435,9 +435,9 @@ static int lockinsert(LOCK * volatile *head, LOCK *node, LF_PINS *pins, } } while (res == REPEAT_ONCE_MORE); - _lf_unpin(pins, 0); - _lf_unpin(pins, 1); - _lf_unpin(pins, 2); + lf_unpin(pins, 0); + lf_unpin(pins, 1); + lf_unpin(pins, 2); /* note that blocker is not necessarily pinned here (when it's == curr). this is ok as in such a case it's either a dummy node for @@ -461,9 +461,9 @@ static int lockpeek(LOCK * volatile *head, LOCK *node, LF_PINS *pins, res= lockfind(head, node, &cursor, pins); - _lf_unpin(pins, 0); - _lf_unpin(pins, 1); - _lf_unpin(pins, 2); + lf_unpin(pins, 0); + lf_unpin(pins, 1); + lf_unpin(pins, 2); if (blocker) *blocker= cursor.blocker; return res; @@ -502,7 +502,7 @@ static int lockdelete(LOCK * volatile *head, LOCK *node, LF_PINS *pins) { if (my_atomic_casptr((void **)cursor.prev, (void **)(char*)&cursor.curr, cursor.next)) - _lf_alloc_free(pins, cursor.curr); + lf_alloc_free(pins, cursor.curr); else lockfind(head, node, &cursor, pins); } @@ -513,10 +513,10 @@ static int lockdelete(LOCK * volatile *head, LOCK *node, LF_PINS *pins) cursor.upgrade_from->flags|= IGNORE_ME; } } while (res == REPEAT_ONCE_MORE); - _lf_unpin(pins, 0); - _lf_unpin(pins, 1); - _lf_unpin(pins, 2); - _lf_unpin(pins, 3); + lf_unpin(pins, 0); + lf_unpin(pins, 1); + lf_unpin(pins, 2); + lf_unpin(pins, 3); return res; } @@ -532,7 +532,7 @@ void lockman_init(LOCKMAN *lm, loid_to_lo_func *func, uint timeout) void lockman_destroy(LOCKMAN *lm) { - LOCK *el= *(LOCK **)_lf_dynarray_lvalue(&lm->array, 0); + LOCK *el= *(LOCK **)lf_dynarray_lvalue(&lm->array, 0); while (el) { intptr next= el->link; @@ -556,7 +556,7 @@ static void initialize_bucket(LOCKMAN *lm, LOCK * volatile *node, uint parent= my_clear_highest_bit(bucket); LOCK *dummy= (LOCK *)my_malloc(sizeof(LOCK), MYF(MY_WME)); LOCK **tmp= 0, *cur; - LOCK * volatile *el= _lf_dynarray_lvalue(&lm->array, parent); + LOCK * volatile *el= lf_dynarray_lvalue(&lm->array, parent); if (*el == NULL && bucket) initialize_bucket(lm, el, parent, pins); @@ -604,15 +604,14 @@ enum lockman_getlock_result lockman_getlock(LOCKMAN *lm, LOCK_OWNER *lo, enum lockman_lock_type old_lock; DBUG_ASSERT(lo->loid); - lf_rwlock_by_pins(pins); - node= (LOCK *)_lf_alloc_new(pins); + node= (LOCK *)lf_alloc_new(pins); node->flags= 0; node->lock= lock; node->loid= lo->loid; node->resource= resource; hashnr= calc_hash(resource); bucket= hashnr % lm->size; - el= _lf_dynarray_lvalue(&lm->array, bucket); + el= lf_dynarray_lvalue(&lm->array, bucket); if (*el == NULL) initialize_bucket(lm, el, bucket, pins); node->hashnr= my_reverse_bits(hashnr) | 1; @@ -621,8 +620,7 @@ enum lockman_getlock_result lockman_getlock(LOCKMAN *lm, LOCK_OWNER *lo, { int r; old_lock= blocker->lock; - _lf_alloc_free(pins, node); - lf_rwunlock_by_pins(pins); + lf_alloc_free(pins, node); r= getlock_result[old_lock][lock]; DBUG_ASSERT(r); return r; @@ -639,7 +637,7 @@ enum lockman_getlock_result lockman_getlock(LOCKMAN *lm, LOCK_OWNER *lo, ulonglong deadline; struct timespec timeout; - _lf_assert_pin(pins, 3); /* blocker must be pinned here */ + lf_assert_pin(pins, 3); /* blocker must be pinned here */ wait_for_lo= lm->loid_to_lo(blocker->loid); /* @@ -652,7 +650,7 @@ enum lockman_getlock_result lockman_getlock(LOCKMAN *lm, LOCK_OWNER *lo, if (lock_compatibility_matrix[blocker->lock][lock]) { blocker= wait_for_lo->all_locks; - _lf_pin(pins, 3, blocker); + lf_pin(pins, 3, blocker); if (blocker != wait_for_lo->all_locks) continue; wait_for_lo= wait_for_lo->waiting_for; @@ -667,7 +665,6 @@ enum lockman_getlock_result lockman_getlock(LOCKMAN *lm, LOCK_OWNER *lo, continue; lo->waiting_for= wait_for_lo; - lf_rwunlock_by_pins(pins); /* We lock a mutex - it may belong to a wrong LOCK_OWNER, but it must @@ -683,7 +680,6 @@ enum lockman_getlock_result lockman_getlock(LOCKMAN *lm, LOCK_OWNER *lo, the lock was rolled back. Either way - the lock was removed */ pthread_mutex_unlock(wait_for_lo->mutex); - lf_rwlock_by_pins(pins); continue; } @@ -695,7 +691,6 @@ enum lockman_getlock_result lockman_getlock(LOCKMAN *lm, LOCK_OWNER *lo, pthread_cond_timedwait(wait_for_lo->cond, wait_for_lo->mutex, &timeout); } while (!DELETED(blocker->link) && my_hrtime().val < deadline/1000); pthread_mutex_unlock(wait_for_lo->mutex); - lf_rwlock_by_pins(pins); if (!DELETED(blocker->link)) { /* @@ -704,14 +699,12 @@ enum lockman_getlock_result lockman_getlock(LOCKMAN *lm, LOCK_OWNER *lo, Instead we're relying on the caller to abort the transaction, and release all locks at once - see lockman_release_locks() */ - _lf_unpin(pins, 3); - lf_rwunlock_by_pins(pins); + lf_unpin(pins, 3); return DIDNT_GET_THE_LOCK; } } lo->waiting_for= 0; - _lf_assert_unpin(pins, 3); /* unpin should not be needed */ - lf_rwunlock_by_pins(pins); + lf_assert_unpin(pins, 3); /* unpin should not be needed */ return getlock_result[lock][lock]; } @@ -729,18 +722,16 @@ int lockman_release_locks(LOCKMAN *lm, LOCK_OWNER *lo) LF_PINS *pins= lo->pins; pthread_mutex_lock(lo->mutex); - lf_rwlock_by_pins(pins); for (node= lo->all_locks; node; node= next) { next= node->lonext; bucket= calc_hash(node->resource) % lm->size; - el= _lf_dynarray_lvalue(&lm->array, bucket); + el= lf_dynarray_lvalue(&lm->array, bucket); if (*el == NULL) initialize_bucket(lm, el, bucket, pins); lockdelete(el, node, pins); my_atomic_add32(&lm->count, -1); } - lf_rwunlock_by_pins(pins); lo->all_locks= 0; /* now signal all waiters */ pthread_cond_broadcast(lo->cond); @@ -757,7 +748,7 @@ static const char *lock2str[]= */ void print_lockhash(LOCKMAN *lm) { - LOCK *el= *(LOCK **)_lf_dynarray_lvalue(&lm->array, 0); + LOCK *el= *(LOCK **)lf_dynarray_lvalue(&lm->array, 0); printf("hash: size %u count %u\n", lm->size, lm->count); while (el) { diff --git a/storage/maria/ma_bitmap.c b/storage/maria/ma_bitmap.c index e1ccceb78cd..60d57b95d86 100644 --- a/storage/maria/ma_bitmap.c +++ b/storage/maria/ma_bitmap.c @@ -155,7 +155,7 @@ static inline my_bool write_changed_bitmap(MARIA_SHARE *share, my_bool res; DBUG_ENTER("write_changed_bitmap"); DBUG_ASSERT(share->pagecache->block_size == bitmap->block_size); - DBUG_ASSERT(bitmap->file.write_callback != 0); + DBUG_ASSERT(bitmap->file.pre_write_hook != 0); DBUG_PRINT("info", ("bitmap->non_flushable: %u", bitmap->non_flushable)); /* @@ -238,6 +238,7 @@ my_bool _ma_bitmap_init(MARIA_SHARE *share, File file, sizeof(MARIA_PINNED_PAGE), 1, 1, MYF(0))) return 1; + bitmap->share= share; bitmap->block_size= share->block_size; bitmap->file.file= file; _ma_bitmap_set_pagecache_callbacks(&bitmap->file, share); @@ -256,7 +257,7 @@ my_bool _ma_bitmap_init(MARIA_SHARE *share, File file, /* Update size for bits */ /* TODO; Make this dependent of the row size */ - max_page_size= share->block_size - PAGE_OVERHEAD_SIZE + DIR_ENTRY_SIZE; + max_page_size= share->block_size - PAGE_OVERHEAD_SIZE(share) + DIR_ENTRY_SIZE; bitmap->sizes[0]= max_page_size; /* Empty page */ bitmap->sizes[1]= max_page_size - max_page_size * 30 / 100; bitmap->sizes[2]= max_page_size - max_page_size * 60 / 100; @@ -1240,9 +1241,22 @@ static my_bool allocate_head(MARIA_FILE_BITMAP *bitmap, uint size, uchar *data= bitmap->map, *end= data + bitmap->used_size; uchar *best_data= 0; uint best_bits= (uint) -1, UNINIT_VAR(best_pos); + uint first_pattern= 0; /* if doing insert_order */ + MARIA_SHARE *share= bitmap->share; + my_bool insert_order= + MY_TEST(share->base.extra_options & MA_EXTRA_OPTIONS_INSERT_ORDER); DBUG_ENTER("allocate_head"); - DBUG_ASSERT(size <= FULL_PAGE_SIZE(bitmap->block_size)); + DBUG_ASSERT(size <= FULL_PAGE_SIZE(share)); + + if (insert_order && bitmap->page == share->last_insert_bitmap) + { + uint last_insert_page= share->last_insert_page; + uint byte= 6 * (last_insert_page / 16); + first_pattern= last_insert_page % 16; + DBUG_ASSERT(data + byte < end); + data+= byte; + } for (; data < end; data+= 6) { @@ -1257,8 +1271,12 @@ static my_bool allocate_head(MARIA_FILE_BITMAP *bitmap, uint size, */ if ((!bits && best_data) || ((bits & 04444444444444444LL) == 04444444444444444LL)) + { + first_pattern= 0; // always restart from 0 when moving to new 6-byte continue; - for (i= 0; i < 16 ; i++, bits >>= 3) + } + for (i= first_pattern, bits >>= (3 * first_pattern); i < 16 ; + i++, bits >>= 3) { uint pattern= (uint) (bits & 7); if (pattern <= min_bits) @@ -1279,6 +1297,7 @@ static my_bool allocate_head(MARIA_FILE_BITMAP *bitmap, uint size, } } } + first_pattern= 0; // always restart from 0 when moving to new 6-byte } if (!best_data) /* Found no place */ { @@ -1292,6 +1311,12 @@ static my_bool allocate_head(MARIA_FILE_BITMAP *bitmap, uint size, } found: + if (insert_order) + { + share->last_insert_page= + ((uint) (best_data - bitmap->map)) / 6 * 16 + best_pos; + share->last_insert_bitmap= bitmap->page; + } fill_block(bitmap, block, best_data, best_pos, best_bits, FULL_HEAD_PAGE); DBUG_RETURN(0); } @@ -1590,6 +1615,16 @@ static my_bool find_head(MARIA_HA *info, uint length, uint position) */ block= dynamic_element(&info->bitmap_blocks, position, MARIA_BITMAP_BLOCK *); + if (info->s->base.extra_options & MA_EXTRA_OPTIONS_INSERT_ORDER) + { + if (bitmap->page != info->s->last_insert_bitmap && + _ma_change_bitmap_page(info, bitmap, + info->s->last_insert_bitmap)) + return 1; + /* Don't allocate any blocks from earlier pages */ + info->s->state.first_bitmap_with_space= info->s->last_insert_bitmap; + } + /* We need to have DIRENTRY_SIZE here to take into account that we may need an extra directory entry for the row @@ -1621,7 +1656,7 @@ static my_bool find_tail(MARIA_HA *info, uint length, uint position) MARIA_FILE_BITMAP *bitmap= &info->s->bitmap; MARIA_BITMAP_BLOCK *block; DBUG_ENTER("find_tail"); - DBUG_ASSERT(length <= info->s->block_size - PAGE_OVERHEAD_SIZE); + DBUG_ASSERT(length <= info->s->block_size - PAGE_OVERHEAD_SIZE(info->s)); /* Needed, as there is no error checking in dynamic_element */ if (allocate_dynamic(&info->bitmap_blocks, position)) @@ -1694,14 +1729,13 @@ static my_bool find_mid(MARIA_HA *info, ulong pages, uint position) static my_bool find_blob(MARIA_HA *info, ulong length) { MARIA_FILE_BITMAP *bitmap= &info->s->bitmap; - uint full_page_size= FULL_PAGE_SIZE(info->s->block_size); + uint full_page_size= FULL_PAGE_SIZE(info->s); ulong pages; uint rest_length, used; - uint first_block_pos; + uint UNINIT_VAR(first_block_pos); MARIA_BITMAP_BLOCK *first_block= 0; DBUG_ENTER("find_blob"); DBUG_PRINT("enter", ("length: %lu", length)); - LINT_INIT(first_block_pos); pages= length / full_page_size; rest_length= (uint) (length - pages * full_page_size); @@ -1909,7 +1943,7 @@ static my_bool write_rest_of_head(MARIA_HA *info, uint position, ulong rest_length) { MARIA_SHARE *share= info->s; - uint full_page_size= FULL_PAGE_SIZE(share->block_size); + uint full_page_size= FULL_PAGE_SIZE(share); MARIA_BITMAP_BLOCK *block; DBUG_ENTER("write_rest_of_head"); DBUG_PRINT("enter", ("position: %u rest_length: %lu", position, @@ -1995,7 +2029,7 @@ my_bool _ma_bitmap_find_place(MARIA_HA *info, MARIA_ROW *row, */ info->bitmap_blocks.elements= ELEMENTS_RESERVED_FOR_MAIN_PART; - max_page_size= (share->block_size - PAGE_OVERHEAD_SIZE); + max_page_size= (share->block_size - PAGE_OVERHEAD_SIZE(share)); mysql_mutex_lock(&share->bitmap.bitmap_lock); @@ -2890,12 +2924,10 @@ int _ma_bitmap_create_first(MARIA_SHARE *share) */ static my_bool -flush_log_for_bitmap(uchar *page __attribute__((unused)), - pgcache_page_no_t page_no __attribute__((unused)), - uchar *data_ptr __attribute__((unused))) +flush_log_for_bitmap(PAGECACHE_IO_HOOK_ARGS *args __attribute__ ((unused))) { #ifndef DBUG_OFF - const MARIA_SHARE *share= (MARIA_SHARE*)data_ptr; + const MARIA_SHARE *share= (MARIA_SHARE*)args->data; #endif DBUG_ENTER("flush_log_for_bitmap"); DBUG_ASSERT(share->now_transactional); @@ -2918,22 +2950,23 @@ flush_log_for_bitmap(uchar *page __attribute__((unused)), void _ma_bitmap_set_pagecache_callbacks(PAGECACHE_FILE *file, MARIA_SHARE *share) { + pagecache_file_set_null_hooks(file); file->callback_data= (uchar*) share; file->flush_log_callback= maria_flush_log_for_page_none; - file->write_fail= maria_page_write_failure; + file->post_write_hook= maria_page_write_failure; if (share->temporary) { - file->read_callback= &maria_page_crc_check_none; - file->write_callback= &maria_page_filler_set_none; + file->post_read_hook= &maria_page_crc_check_none; + file->pre_write_hook= &maria_page_filler_set_none; } else { - file->read_callback= &maria_page_crc_check_bitmap; + file->post_read_hook= &maria_page_crc_check_bitmap; if (share->options & HA_OPTION_PAGE_CHECKSUM) - file->write_callback= &maria_page_crc_set_normal; + file->pre_write_hook= &maria_page_crc_set_normal; else - file->write_callback= &maria_page_filler_set_bitmap; + file->pre_write_hook= &maria_page_filler_set_bitmap; if (share->now_transactional) file->flush_log_callback= flush_log_for_bitmap; } @@ -3093,6 +3126,10 @@ static my_bool _ma_bitmap_create_missing(MARIA_HA *info, bzero(bitmap->map, bitmap->block_size); bitmap->used_size= 0; #ifndef DBUG_OFF + /* + Make a copy of the page to be able to print out bitmap changes during + debugging + */ memcpy(bitmap->map + bitmap->block_size, bitmap->map, bitmap->block_size); #endif diff --git a/storage/maria/ma_blockrec.c b/storage/maria/ma_blockrec.c index fe719888817..2f2558d22c0 100644 --- a/storage/maria/ma_blockrec.c +++ b/storage/maria/ma_blockrec.c @@ -340,10 +340,12 @@ static my_bool delete_head_or_tail(MARIA_HA *info, pgcache_page_no_t page, uint record_number, my_bool head, my_bool from_update); #ifndef DBUG_OFF -static void _ma_print_directory(FILE *file, uchar *buff, uint block_size); +static void _ma_print_directory(MARIA_SHARE *share, + FILE *file, uchar *buff, uint block_size); #endif -static uchar *store_page_range(uchar *to, MARIA_BITMAP_BLOCK *block, - uint block_size, ulong length, +static uchar *store_page_range(MARIA_SHARE *share, + uchar *to, MARIA_BITMAP_BLOCK *block, + ulong length, uint *tot_ranges); static size_t fill_insert_undo_parts(MARIA_HA *info, const uchar *record, LEX_CUSTRING *log_parts, @@ -523,7 +525,7 @@ my_bool _ma_init_block_record(MARIA_HA *info) /* Reserve some initial space to avoid mallocs during execution */ default_extents= (ELEMENTS_RESERVED_FOR_MAIN_PART + 1 + (AVERAGE_BLOB_SIZE / - FULL_PAGE_SIZE(share->block_size) / + FULL_PAGE_SIZE(share) / BLOB_SEGMENT_MIN_SIZE)); if (my_init_dynamic_array(&info->bitmap_blocks, @@ -616,7 +618,8 @@ static inline uint start_of_next_entry(uchar *dir) */ -static inline uint end_of_previous_entry(uchar *dir, uchar *end) +static inline uint end_of_previous_entry(MARIA_SHARE *share, + uchar *dir, uchar *end) { uchar *pos; for (pos= dir + DIR_ENTRY_SIZE ; pos < end ; pos+= DIR_ENTRY_SIZE) @@ -625,16 +628,17 @@ static inline uint end_of_previous_entry(uchar *dir, uchar *end) if ((offset= uint2korr(pos))) return offset + uint2korr(pos+2); } - return PAGE_HEADER_SIZE; + return PAGE_HEADER_SIZE(share); } #ifndef DBUG_OFF -static void _ma_print_directory(FILE *file, uchar *buff, uint block_size) +static void _ma_print_directory(MARIA_SHARE *share, + FILE *file, uchar *buff, uint block_size) { uint max_entry= (uint) ((uchar *) buff)[DIR_COUNT_OFFSET], row= 0; - uint end_of_prev_row= PAGE_HEADER_SIZE; + uint end_of_prev_row= PAGE_HEADER_SIZE(share); uchar *dir, *end; dir= dir_entry_pos(buff, block_size, max_entry-1); @@ -662,13 +666,14 @@ static void _ma_print_directory(FILE *file, uchar *buff, uint block_size) } -static void check_directory(uchar *buff, uint block_size, uint min_row_length, +static void check_directory(MARIA_SHARE *share, + uchar *buff, uint block_size, uint min_row_length, uint real_empty_size) { uchar *dir, *end; uint max_entry= (uint) buff[DIR_COUNT_OFFSET]; uint start_of_dir, deleted; - uint end_of_prev_row= PAGE_HEADER_SIZE; + uint end_of_prev_row= PAGE_HEADER_SIZE(share); uint empty_size_on_page; uint empty_size; uchar free_entry, prev_free_entry; @@ -715,7 +720,7 @@ static void check_directory(uchar *buff, uint block_size, uint min_row_length, DBUG_ASSERT(deleted == 0); } #else -#define check_directory(A,B,C,D) +#define check_directory(A,B,C,D,E) #endif /* DBUG_OFF */ @@ -779,7 +784,7 @@ my_bool enough_free_entries_on_page(MARIA_SHARE *share, @brief Extend a record area to fit a given size block @fn extend_area_on_page() - @param info Handler if head page and 0 if tail page + @param info Handler @param buff Page buffer @param dir Pointer to dir entry in buffer @param rownr Row number we working on @@ -788,6 +793,7 @@ my_bool enough_free_entries_on_page(MARIA_SHARE *share, @param empty_space Total empty space in buffer This is updated with length after dir is allocated and current block freed + @param head_page 1 if head page, 0 for tail page @implementation The logic is as follows (same as in _ma_update_block_record()) @@ -812,20 +818,23 @@ my_bool enough_free_entries_on_page(MARIA_SHARE *share, static my_bool extend_area_on_page(MARIA_HA *info, uchar *buff, uchar *dir, - uint rownr, uint block_size, + uint rownr, uint request_length, uint *empty_space, uint *ret_offset, - uint *ret_length) + uint *ret_length, + my_bool head_page) { uint rec_offset, length, org_rec_length; uint max_entry= (uint) buff[DIR_COUNT_OFFSET]; + MARIA_SHARE *share= info->s; + uint block_size= share->block_size; DBUG_ENTER("extend_area_on_page"); /* We can't check for min length here as we may have called extend_directory() to create a new (empty) entry just before */ - check_directory(buff, block_size, 0, *empty_space); + check_directory(share, buff, block_size, 0, *empty_space); rec_offset= uint2korr(dir); if (rec_offset) @@ -867,7 +876,8 @@ static my_bool extend_area_on_page(MARIA_HA *info, Find first possible position where to put new data. */ old_rec_offset= rec_offset; - rec_offset= end_of_previous_entry(dir, buff + block_size - + rec_offset= end_of_previous_entry(share, + dir, buff + block_size - PAGE_SUFFIX_SIZE); length+= (uint) (old_rec_offset - rec_offset); DBUG_ASSERT(old_rec_offset); @@ -896,9 +906,10 @@ static my_bool extend_area_on_page(MARIA_HA *info, int2store(dir, rec_offset); /* Reset length, as this may be a deleted block */ int2store(dir+2, 0); - _ma_compact_block_page(buff, block_size, rownr, 1, - info ? info->trn->min_read_from: 0, - info ? info->s->base.min_block_length : 0); + _ma_compact_block_page(share, + buff, rownr, 1, + head_page ? info->trn->min_read_from: 0, + head_page ? share->base.min_block_length : 0); rec_offset= uint2korr(dir); length= uint2korr(dir+2); if (length < request_length) @@ -906,7 +917,7 @@ static my_bool extend_area_on_page(MARIA_HA *info, DBUG_PRINT("error", ("Not enough space: " "length: %u request_length: %u", length, request_length)); - _ma_set_fatal_error(info->s, HA_ERR_WRONG_IN_RECORD); + _ma_set_fatal_error(share, HA_ERR_WRONG_IN_RECORD); DBUG_RETURN(1); /* Error in block */ } *empty_space= length; /* All space is here */ @@ -918,7 +929,9 @@ static my_bool extend_area_on_page(MARIA_HA *info, *ret_offset= rec_offset; *ret_length= length; - check_directory(buff, block_size, info ? info->s->base.min_block_length : 0, + check_directory(share, + buff, block_size, + head_page ? share->base.min_block_length : 0, *empty_space - length); DBUG_RETURN(0); } @@ -984,14 +997,15 @@ static uint empty_space_on_page(uchar *buff, uint block_size) @brief Ensure we have space for new directory entries @fn make_space_for_directory() + @param info Handler @param buff Page buffer - @param block_size Block size for pages @param max_entry Number of current entries in directory @param count Number of new entries to be added to directory @param first_dir First directory entry on page @param empty_space Total empty space in buffer. It's updated to reflect the new empty space @param first_pos Store position to last data byte on page here + @param head_page 1 if head page, 0 for tail page. @note This function is inline as the argument passing is the biggest @@ -1004,11 +1018,13 @@ static uint empty_space_on_page(uchar *buff, uint block_size) static inline my_bool make_space_for_directory(MARIA_HA *info, - uchar *buff, uint block_size, uint max_entry, + uchar *buff, uint max_entry, uint count, uchar *first_dir, uint *empty_space, - uint *first_pos) + uint *first_pos, + my_bool head_page) { uint length_needed= DIR_ENTRY_SIZE * count; + MARIA_SHARE *share= info->s; /* The following is not true only in the case and UNDO is used to reinsert @@ -1022,9 +1038,10 @@ make_space_for_directory(MARIA_HA *info, if ((uint) (first_dir - buff) < *first_pos + length_needed) { /* Create place for directory */ - _ma_compact_block_page(buff, block_size, max_entry - 1, 0, - info ? info->trn->min_read_from : 0, - info ? info->s->base.min_block_length : 0); + _ma_compact_block_page(share, + buff, max_entry - 1, 0, + head_page ? info->trn->min_read_from : 0, + head_page ? share->base.min_block_length : 0); *first_pos= (uint2korr(first_dir) + uint2korr(first_dir + 2)); *empty_space= uint2korr(buff + EMPTY_SPACE_OFFSET); if (*empty_space < length_needed) @@ -1040,7 +1057,7 @@ make_space_for_directory(MARIA_HA *info, } } else - *first_pos= PAGE_HEADER_SIZE; + *first_pos= PAGE_HEADER_SIZE(share); /* Reduce directory entry size from free space size */ (*empty_space)-= length_needed; @@ -1054,13 +1071,14 @@ make_space_for_directory(MARIA_HA *info, SYNOPSIS find_free_position() - info Handler if head page and 0 otherwise + info Handler buff Page block_size Size of page res_rownr Store index to free position here res_length Store length of found segment here empty_space Store length of empty space on disk here. This is all empty space, including the found block. + @param head_page 1 if head page, 0 for tail page. NOTES If there is a free directory entry (entry with position == 0), @@ -1088,11 +1106,13 @@ make_space_for_directory(MARIA_HA *info, static uchar *find_free_position(MARIA_HA *info, uchar *buff, uint block_size, uint *res_rownr, - uint *res_length, uint *empty_space) + uint *res_length, uint *empty_space, + my_bool head_page) { uint max_entry, free_entry; uint length, first_pos; uchar *dir, *first_dir; + MARIA_SHARE *share= info->s; DBUG_ENTER("find_free_position"); max_entry= (uint) buff[DIR_COUNT_OFFSET]; @@ -1119,7 +1139,8 @@ static uchar *find_free_position(MARIA_HA *info, next_entry[2]= END_OF_DIR_FREE_LIST; /* Backlink */ } - first_pos= end_of_previous_entry(dir, buff + block_size - + first_pos= end_of_previous_entry(share, + dir, buff + block_size - PAGE_SUFFIX_SIZE); length= start_of_next_entry(dir) - first_pos; int2store(dir, first_pos); /* Update dir entry */ @@ -1127,8 +1148,8 @@ static uchar *find_free_position(MARIA_HA *info, *res_rownr= free_entry; *res_length= length; - check_directory(buff, block_size, - info ? info->s->base.min_block_length : 0, (uint) -1); + check_directory(share, buff, block_size, + head_page ? share->base.min_block_length : 0, (uint) -1); DBUG_RETURN(dir); } /* No free places in dir; create a new one */ @@ -1137,8 +1158,8 @@ static uchar *find_free_position(MARIA_HA *info, if (max_entry == MAX_ROWS_PER_PAGE) DBUG_RETURN(0); - if (make_space_for_directory(info, buff, block_size, max_entry, 1, - first_dir, empty_space, &first_pos)) + if (make_space_for_directory(info, buff, max_entry, 1, + first_dir, empty_space, &first_pos, head_page)) DBUG_RETURN(0); dir= first_dir - DIR_ENTRY_SIZE; @@ -1149,7 +1170,9 @@ static uchar *find_free_position(MARIA_HA *info, *res_rownr= max_entry; *res_length= length; - check_directory(buff, block_size, info ? info->s->base.min_block_length : 0, + check_directory(share, + buff, block_size, + head_page ? share->base.min_block_length : 0, *empty_space); DBUG_RETURN(dir); } @@ -1159,13 +1182,14 @@ static uchar *find_free_position(MARIA_HA *info, @brief Enlarge page directory to hold more entries @fn extend_directory() - @param info Handler if head page and 0 otherwise + @param info Handler @param buff Page buffer @param block_size Block size @param max_entry Number of directory entries on page @param new_entry Position for new entry @param empty_space Total empty space in buffer. It's updated to reflect the new empty space + @param head_page 1 if head page, 0 for tail page. @note This is only called on UNDO when we want to expand the directory @@ -1180,7 +1204,7 @@ static uchar *find_free_position(MARIA_HA *info, static my_bool extend_directory(MARIA_HA *info, uchar *buff, uint block_size, uint max_entry, uint new_entry, - uint *empty_space) + uint *empty_space, my_bool head_page) { uint length, first_pos; uchar *dir, *first_dir; @@ -1193,9 +1217,9 @@ static my_bool extend_directory(MARIA_HA *info, uchar *buff, uint block_size, */ first_dir= dir_entry_pos(buff, block_size, max_entry) + DIR_ENTRY_SIZE; - if (make_space_for_directory(info, buff, block_size, max_entry, + if (make_space_for_directory(info, buff, max_entry, new_entry - max_entry + 1, - first_dir, empty_space, &first_pos)) + first_dir, empty_space, &first_pos, head_page)) DBUG_RETURN(1); /* Set the new directory entry to cover the max possible length */ @@ -1229,9 +1253,10 @@ static my_bool extend_directory(MARIA_HA *info, uchar *buff, uint block_size, } } - check_directory(buff, block_size, - info ? MY_MIN(info->s->base.min_block_length, length) : 0, - *empty_space); + check_directory(info->s, + buff, block_size, + head_page ? MY_MIN(info->s->base.min_block_length, length) : + 0, *empty_space); DBUG_RETURN(0); } @@ -1432,25 +1457,27 @@ static void calc_record_size(MARIA_HA *info, const uchar *record, @param min_read_from If <> 0, remove all trid's that are less than this */ -void _ma_compact_block_page(uchar *buff, uint block_size, uint rownr, +void _ma_compact_block_page(MARIA_SHARE *share, + uchar *buff, uint rownr, my_bool extend_block, TrID min_read_from, uint min_row_length) { uint max_entry= (uint) buff[DIR_COUNT_OFFSET]; uint page_pos, next_free_pos, start_of_found_block, diff, end_of_found_block; uint freed_size= 0; + uint block_size= share->block_size; uchar *dir, *end; DBUG_ENTER("_ma_compact_block_page"); DBUG_PRINT("enter", ("rownr: %u min_read_from: %lu", rownr, (ulong) min_read_from)); DBUG_ASSERT(max_entry > 0 && - max_entry < (block_size - PAGE_HEADER_SIZE - + max_entry < (block_size - PAGE_HEADER_SIZE(share) - PAGE_SUFFIX_SIZE) / DIR_ENTRY_SIZE); /* Move all entries before and including rownr up to start of page */ dir= dir_entry_pos(buff, block_size, rownr); end= dir_entry_pos(buff, block_size, 0); - page_pos= next_free_pos= start_of_found_block= PAGE_HEADER_SIZE; + page_pos= next_free_pos= start_of_found_block= PAGE_HEADER_SIZE(share); diff= 0; for (; dir <= end ; end-= DIR_ENTRY_SIZE) { @@ -1634,9 +1661,10 @@ void _ma_compact_block_page(uchar *buff, uint block_size, uint rownr, } buff[PAGE_TYPE_OFFSET]&= ~(uchar) PAGE_CAN_BE_COMPACTED; } - check_directory(buff, block_size, min_row_length, + check_directory(share, buff, block_size, min_row_length, extend_block ? 0 : (uint) -1); - DBUG_EXECUTE("directory", _ma_print_directory(DBUG_FILE, buff, block_size);); + DBUG_EXECUTE("directory", _ma_print_directory(share, + DBUG_FILE, buff, block_size);); DBUG_VOID_RETURN; } @@ -1661,7 +1689,7 @@ static void make_empty_page(MARIA_HA *info, uchar *buff, uint page_type, uint block_size= info->s->block_size; DBUG_ENTER("make_empty_page"); - bzero(buff, PAGE_HEADER_SIZE); + bzero(buff, PAGE_HEADER_SIZE(info->s)); #if !defined(DONT_ZERO_PAGE_BLOCKS) || defined(HAVE_valgrind) /* @@ -1670,7 +1698,8 @@ static void make_empty_page(MARIA_HA *info, uchar *buff, uint page_type, The code does not assume the block is zeroed. */ if (page_type != BLOB_PAGE) - bzero(buff+ PAGE_HEADER_SIZE, block_size - PAGE_HEADER_SIZE); + bzero(buff+ PAGE_HEADER_SIZE(info->s), + block_size - PAGE_HEADER_SIZE(info->s)); #endif buff[PAGE_TYPE_OFFSET]= (uchar) page_type; buff[DIR_COUNT_OFFSET]= (int) create_dir_entry; @@ -1679,7 +1708,7 @@ static void make_empty_page(MARIA_HA *info, uchar *buff, uint page_type, { /* Create directory entry to point to start of page with size 0 */ buff+= block_size - PAGE_SUFFIX_SIZE - DIR_ENTRY_SIZE; - int2store(buff, PAGE_HEADER_SIZE); + int2store(buff, PAGE_HEADER_SIZE(info->s)); int2store(buff+2, 0); } DBUG_VOID_RETURN; @@ -1738,8 +1767,8 @@ static my_bool get_head_or_tail_page(MARIA_HA *info, /* New page */ make_empty_page(info, buff, page_type, 1); res->buff= buff; - res->empty_space= res->length= (block_size - PAGE_OVERHEAD_SIZE); - res->data= (buff + PAGE_HEADER_SIZE); + res->empty_space= res->length= (block_size - PAGE_OVERHEAD_SIZE(share)); + res->data= (buff + PAGE_HEADER_SIZE(share)); res->dir= res->data + res->length; res->rownr= 0; DBUG_ASSERT(length <= res->length); @@ -1759,16 +1788,17 @@ static my_bool get_head_or_tail_page(MARIA_HA *info, DBUG_ASSERT((uint) (res->buff[PAGE_TYPE_OFFSET] & PAGE_TYPE_MASK) == page_type); - if (!(dir= find_free_position(page_type == HEAD_PAGE ? info : 0, - res->buff, block_size, &res->rownr, - &res->length, &res->empty_space))) + if (!(dir= find_free_position(info, res->buff, block_size, &res->rownr, + &res->length, &res->empty_space, + page_type == HEAD_PAGE))) goto crashed; if (res->length < length) { if (res->empty_space + res->length >= length) { - _ma_compact_block_page(res->buff, block_size, res->rownr, 1, + _ma_compact_block_page(share, + res->buff, res->rownr, 1, (page_type == HEAD_PAGE ? info->trn->min_read_from : 0), (page_type == HEAD_PAGE ? @@ -1839,7 +1869,7 @@ static my_bool get_rowpos_in_head_or_tail_page(MARIA_HA *info, { /* New page */ make_empty_page(info, buff, page_type, 0); - res->empty_space= block_size - PAGE_HEADER_SIZE - PAGE_SUFFIX_SIZE; + res->empty_space= block_size - PAGE_HEADER_SIZE(share) - PAGE_SUFFIX_SIZE; } else { @@ -1861,8 +1891,9 @@ static my_bool get_rowpos_in_head_or_tail_page(MARIA_HA *info, max_entry= (uint) buff[DIR_COUNT_OFFSET]; if (max_entry <= rownr) { - if (extend_directory(page_type == HEAD_PAGE ? info : 0, buff, block_size, - max_entry, rownr, &res->empty_space)) + if (extend_directory(info, buff, block_size, + max_entry, rownr, &res->empty_space, + page_type == HEAD_PAGE)) goto err; } @@ -1872,9 +1903,9 @@ static my_bool get_rowpos_in_head_or_tail_page(MARIA_HA *info, */ dir= dir_entry_pos(buff, block_size, rownr); - if (extend_area_on_page(page_type == HEAD_PAGE ? info : 0, buff, dir, - rownr, block_size, length, - &res->empty_space, &rec_offset, &max_length)) + if (extend_area_on_page(info, buff, dir, rownr, length, + &res->empty_space, &rec_offset, &max_length, + page_type == HEAD_PAGE)) goto err; res->buff= buff; @@ -2085,7 +2116,7 @@ static my_bool write_full_pages(MARIA_HA *info, pgcache_page_no_t page; MARIA_SHARE *share= info->s; uint block_size= share->block_size; - uint data_size= FULL_PAGE_SIZE(block_size); + uint data_size= FULL_PAGE_SIZE(share); uchar *buff= info->keyread_buff; uint page_count, sub_blocks; my_off_t position, max_position; @@ -2126,8 +2157,10 @@ static my_bool write_full_pages(MARIA_HA *info, } lsn_store(buff, lsn); buff[PAGE_TYPE_OFFSET]= (uchar) BLOB_PAGE; + bzero(buff + LSN_SIZE + PAGE_TYPE_SIZE, + FULL_PAGE_HEADER_SIZE(share) - (LSN_SIZE + PAGE_TYPE_SIZE)); copy_length= MY_MIN(data_size, length); - memcpy(buff + LSN_SIZE + PAGE_TYPE_SIZE, data, copy_length); + memcpy(buff + FULL_PAGE_HEADER_SIZE(share), data, copy_length); length-= copy_length; /* @@ -2163,7 +2196,6 @@ static my_bool write_full_pages(MARIA_HA *info, store_page_range() to Store data here block Where pages are to be written - block_size block size length Length of data to be written Normally this is full pages, except for the last tail block that may only partly fit the last page. @@ -2182,11 +2214,12 @@ static my_bool write_full_pages(MARIA_HA *info, # end position for 'to' */ -static uchar *store_page_range(uchar *to, MARIA_BITMAP_BLOCK *block, - uint block_size, ulong length, +static uchar *store_page_range(MARIA_SHARE *share, + uchar *to, MARIA_BITMAP_BLOCK *block, + ulong length, uint *tot_ranges) { - uint data_size= FULL_PAGE_SIZE(block_size); + uint data_size= FULL_PAGE_SIZE(share); ulong pages_left= (length + data_size -1) / data_size; uint page_count, ranges, empty_space; uchar *to_start; @@ -2853,7 +2886,8 @@ static my_bool write_block_record(MARIA_HA *info, head_block->empty_space= 0; /* Page is full */ head_block->used|= BLOCKUSED_USED; - check_directory(page_buff, share->block_size, share->base.min_block_length, + check_directory(share, + page_buff, share->block_size, share->base.min_block_length, (uint) -1); /* @@ -2889,7 +2923,7 @@ static my_bool write_block_record(MARIA_HA *info, uint length; length= column->length - portable_sizeof_char_ptr; memcpy(&blob_pos, record + column->offset + length, sizeof(char*)); - length= *blob_lengths % FULL_PAGE_SIZE(block_size); /* tail size */ + length= *blob_lengths % FULL_PAGE_SIZE(share); /* tail size */ if (length != *blob_lengths) blob_full_pages_exists= 1; if (write_tail(info, block + block->sub_blocks-1, @@ -2956,7 +2990,7 @@ static my_bool write_block_record(MARIA_HA *info, we find the empty page block. */ while (data_length >= (length= (cur_block->page_count * - FULL_PAGE_SIZE(block_size))) && + FULL_PAGE_SIZE(share))) && cur_block->page_count) { #ifdef SANITY_CHECKS @@ -3016,7 +3050,7 @@ static my_bool write_block_record(MARIA_HA *info, } else { - DBUG_ASSERT(data_length < length - FULL_PAGE_SIZE(block_size)); + DBUG_ASSERT(data_length < length - FULL_PAGE_SIZE(share)); DBUG_PRINT("info", ("Splitting blocks into full and tail")); cur_block[1].page= (cur_block->page + cur_block->page_count - 1); cur_block[1].page_count= 1; /* Avoid DBUG_ASSERT */ @@ -3052,7 +3086,7 @@ static my_bool write_block_record(MARIA_HA *info, ulong block_length= (ulong) (tmp_data - info->rec_buff); uchar *extent_data; - length= (uint) (block_length % FULL_PAGE_SIZE(block_size)); + length= (uint) (block_length % FULL_PAGE_SIZE(share)); if (write_tail(info, head_tail_block, info->rec_buff + block_length - length, length)) @@ -3216,7 +3250,8 @@ static my_bool write_block_record(MARIA_HA *info, /* Full head page */ translog_size_t block_length= (translog_size_t) (tmp_data - info->rec_buff); - log_pos= store_page_range(log_pos, head_block+1, block_size, + log_pos= store_page_range(share, + log_pos, head_block+1, (ulong) block_length, &extents); log_array_pos->str= info->rec_buff; log_array_pos->length= block_length; @@ -3245,7 +3280,7 @@ static my_bool write_block_record(MARIA_HA *info, reflect this */ if (tmp_block[tmp_block->sub_blocks - 1].used & BLOCKUSED_TAIL) - blob_length-= (blob_length % FULL_PAGE_SIZE(block_size)); + blob_length-= (blob_length % FULL_PAGE_SIZE(share)); if (blob_length) { memcpy((void*) &log_array_pos->str, @@ -3256,7 +3291,8 @@ static my_bool write_block_record(MARIA_HA *info, log_array_pos++; sub_extents++; - log_pos= store_page_range(log_pos, tmp_block, block_size, + log_pos= store_page_range(share, + log_pos, tmp_block, blob_length, &extents); } tmp_block+= tmp_block->sub_blocks; @@ -3418,7 +3454,7 @@ static my_bool write_block_record(MARIA_HA *info, /* remove tail part */ blob_length= *blob_lengths; if (block[block->sub_blocks - 1].used & BLOCKUSED_TAIL) - blob_length-= (blob_length % FULL_PAGE_SIZE(block_size)); + blob_length-= (blob_length % FULL_PAGE_SIZE(share)); if (blob_length && write_full_pages(info, lsn, block, blob_pos, blob_length)) @@ -3734,9 +3770,9 @@ static my_bool _ma_update_block_record2(MARIA_HA *info, */ block.org_bitmap_value= _ma_free_size_to_head_pattern(&share->bitmap, org_empty_size); - if (extend_area_on_page(info, buff, dir, rownr, block_size, + if (extend_area_on_page(info, buff, dir, rownr, new_row->total_length, &org_empty_size, - &rec_offset, &length)) + &rec_offset, &length, 1)) { errpos= 1; goto err; @@ -3803,7 +3839,8 @@ static my_bool _ma_update_block_record2(MARIA_HA *info, (new_row->total_length <= head_length && org_empty_size + head_length >= new_row->total_length))) { - _ma_compact_block_page(buff, block_size, rownr, 1, + _ma_compact_block_page(share, + buff, rownr, 1, info->trn->min_read_from, share->base.min_block_length); org_empty_size= 0; @@ -3914,9 +3951,9 @@ static my_bool _ma_update_at_original_place(MARIA_HA *info, of the row */ empty_size= org_empty_size; - if (extend_area_on_page(info, buff, dir, rownr, block_size, + if (extend_area_on_page(info, buff, dir, rownr, length_on_head_page, &empty_size, - &rec_offset, &length)) + &rec_offset, &length, 1)) goto err; row_pos.buff= buff; @@ -3991,7 +4028,6 @@ my_bool _ma_update_block_record(MARIA_HA *info, MARIA_RECORD_POS record_pos, SYNOPSIS delete_dir_entry() buff Page buffer - block_size Block size record_number Record number to delete empty_space Empty space on page after delete @@ -4001,9 +4037,11 @@ my_bool _ma_update_block_record(MARIA_HA *info, MARIA_RECORD_POS record_pos, 1 Page is now empty */ -static int delete_dir_entry(uchar *buff, uint block_size, uint record_number, +static int delete_dir_entry(MARIA_SHARE *share, + uchar *buff, uint record_number, uint *empty_space_res) { + uint block_size= share->block_size; uint number_of_records= (uint) buff[DIR_COUNT_OFFSET]; uint length, empty_space; uchar *dir; @@ -4023,7 +4061,7 @@ static int delete_dir_entry(uchar *buff, uint block_size, uint record_number, } #endif - check_directory(buff, block_size, 0, (uint) -1); + check_directory(share, buff, block_size, 0, (uint) -1); empty_space= uint2korr(buff + EMPTY_SPACE_OFFSET); dir= dir_entry_pos(buff, block_size, record_number); length= uint2korr(dir + 2); /* Length of entry we just deleted */ @@ -4099,7 +4137,7 @@ static int delete_dir_entry(uchar *buff, uint block_size, uint record_number, *empty_space_res= empty_space; - check_directory(buff, block_size, 0, empty_space); + check_directory(share, buff, block_size, 0, empty_space); DBUG_RETURN(0); } @@ -4161,7 +4199,7 @@ static my_bool delete_head_or_tail(MARIA_HA *info, lock_at_unpin= PAGECACHE_LOCK_READ_UNLOCK; } - res= delete_dir_entry(buff, share->block_size, record_number, &empty_space); + res= delete_dir_entry(share, buff, record_number, &empty_space); if (res < 0) DBUG_RETURN(1); if (res == 0) /* after our deletion, page is still not empty */ @@ -4378,9 +4416,10 @@ err: In this case *end_of_data is set. */ -static uchar *get_record_position(uchar *buff, uint block_size, +static uchar *get_record_position(MARIA_SHARE *share, uchar *buff, uint record_number, uchar **end_of_data) { + uint block_size= share->block_size; uint number_of_records= (uint) buff[DIR_COUNT_OFFSET]; uchar *dir; uchar *data; @@ -4388,8 +4427,8 @@ static uchar *get_record_position(uchar *buff, uint block_size, #ifdef SANITY_CHECKS if (record_number >= number_of_records || - record_number > ((block_size - PAGE_HEADER_SIZE - PAGE_SUFFIX_SIZE) / - DIR_ENTRY_SIZE)) + record_number > ((block_size - PAGE_HEADER_SIZE(share) - PAGE_SUFFIX_SIZE) + / DIR_ENTRY_SIZE)) { DBUG_PRINT("error", ("Wrong row number: record_number: %u number_of_records: %u", @@ -4402,7 +4441,7 @@ static uchar *get_record_position(uchar *buff, uint block_size, offset= uint2korr(dir); length= uint2korr(dir + 2); #ifdef SANITY_CHECKS - if (offset < PAGE_HEADER_SIZE || + if (offset < PAGE_HEADER_SIZE(share) || offset + length > (block_size - number_of_records * DIR_ENTRY_SIZE - PAGE_SUFFIX_SIZE)) @@ -4532,7 +4571,7 @@ static uchar *read_next_extent(MARIA_HA *info, MARIA_EXTENT_CURSOR *extent, extent->page_count--; *end_of_data= buff + share->block_size - PAGE_SUFFIX_SIZE; info->cur_row.full_page_count++; /* For maria_chk */ - DBUG_RETURN(extent->data_start= buff + LSN_SIZE + PAGE_TYPE_SIZE); + DBUG_RETURN(extent->data_start= buff + FULL_PAGE_HEADER_SIZE(share)); } /* Found tail */ @@ -4542,7 +4581,7 @@ static uchar *read_next_extent(MARIA_HA *info, MARIA_EXTENT_CURSOR *extent, extent->tail_row_nr); info->cur_row.tail_count++; /* For maria_chk */ - if (!(data= get_record_position(buff, share->block_size, + if (!(data= get_record_position(share, buff, extent->tail_row_nr, end_of_data))) goto crashed; @@ -5013,7 +5052,7 @@ static my_bool read_row_extent_info(MARIA_HA *info, uchar *buff, uchar *extents, *end; DBUG_ENTER("read_row_extent_info"); - if (!(data= get_record_position(buff, share->block_size, + if (!(data= get_record_position(share, buff, record_number, &end_of_data))) DBUG_RETURN(1); /* Wrong in record */ @@ -5107,7 +5146,6 @@ int _ma_read_block_record(MARIA_HA *info, uchar *record, MARIA_SHARE *share= info->s; uchar *data, *end_of_data, *buff; uint offset; - uint block_size= share->block_size; int ret; DBUG_ENTER("_ma_read_block_record"); DBUG_PRINT("enter", ("rowid: %lu page: %lu rownr: %u", @@ -5123,7 +5161,7 @@ int _ma_read_block_record(MARIA_HA *info, uchar *record, PAGECACHE_LOCK_LEFT_UNLOCKED, 0))) DBUG_RETURN(my_errno); DBUG_ASSERT((buff[PAGE_TYPE_OFFSET] & PAGE_TYPE_MASK) == HEAD_PAGE); - if (!(data= get_record_position(buff, block_size, offset, &end_of_data))) + if (!(data= get_record_position(share, buff, offset, &end_of_data))) { DBUG_ASSERT(!maria_assert_if_crashed_table); DBUG_PRINT("error", ("Wrong directory entry in data block")); @@ -5149,8 +5187,7 @@ my_bool _ma_cmp_block_unique(MARIA_HA *info, MARIA_UNIQUEDEF *def, Don't allocate more than 16K on the stack to ensure we don't get stack overflow. */ - if (!(old_record= my_safe_alloca(info->s->base.reclength, - MARIA_MAX_RECORD_ON_STACK))) + if (!(old_record= my_safe_alloca(info->s->base.reclength))) DBUG_RETURN(1); /* Don't let the compare destroy blobs that may be in use */ @@ -5172,8 +5209,7 @@ my_bool _ma_cmp_block_unique(MARIA_HA *info, MARIA_UNIQUEDEF *def, info->rec_buff_size= org_rec_buff_size; } DBUG_PRINT("exit", ("result: %d", error)); - my_safe_afree(old_record, info->s->base.reclength, - MARIA_MAX_RECORD_ON_STACK); + my_safe_afree(old_record, info->s->base.reclength); DBUG_RETURN(error != 0); } @@ -5413,10 +5449,11 @@ restart_record_read: info->scan.dir-= DIR_ENTRY_SIZE; /* Point to next row to process */ #ifdef SANITY_CHECKS if (end_of_data > info->scan.dir_end || - offset < PAGE_HEADER_SIZE || length < share->base.min_block_length) + offset < PAGE_HEADER_SIZE(share) || + length < share->base.min_block_length) { DBUG_ASSERT(!(end_of_data > info->scan.dir_end)); - DBUG_ASSERT(!(offset < PAGE_HEADER_SIZE)); + DBUG_ASSERT(!(offset < PAGE_HEADER_SIZE(share))); DBUG_ASSERT(!(length < share->base.min_block_length)); goto err; } @@ -6318,8 +6355,8 @@ uint _ma_apply_redo_insert_row_head_or_tail(MARIA_HA *info, LSN lsn, buff= info->keyread_buff; info->keyread_buff_used= 1; make_empty_page(info, buff, page_type, 1); - empty_space= (block_size - PAGE_OVERHEAD_SIZE); - rec_offset= PAGE_HEADER_SIZE; + empty_space= (block_size - PAGE_OVERHEAD_SIZE(share)); + rec_offset= PAGE_HEADER_SIZE(share); dir= buff+ block_size - PAGE_SUFFIX_SIZE - DIR_ENTRY_SIZE; } else @@ -6377,10 +6414,10 @@ uint _ma_apply_redo_insert_row_head_or_tail(MARIA_HA *info, LSN lsn, goto crashed_file; } make_empty_page(info, buff, page_type, 0); - empty_space= block_size - PAGE_HEADER_SIZE - PAGE_SUFFIX_SIZE; - (void) extend_directory(page_type == HEAD_PAGE ? info: 0, buff, - block_size, 0, rownr, &empty_space); - rec_offset= PAGE_HEADER_SIZE; + empty_space= block_size - PAGE_HEADER_SIZE(share) - PAGE_SUFFIX_SIZE; + (void) extend_directory(info, buff, block_size, 0, rownr, &empty_space, + page_type == HEAD_PAGE); + rec_offset= PAGE_HEADER_SIZE(share); dir= dir_entry_pos(buff, block_size, rownr); empty_space+= uint2korr(dir+2); } @@ -6396,14 +6433,13 @@ uint _ma_apply_redo_insert_row_head_or_tail(MARIA_HA *info, LSN lsn, if (max_entry <= rownr) { /* Add directory entry first in directory and data last on page */ - if (extend_directory(page_type == HEAD_PAGE ? info : 0, buff, - block_size, max_entry, rownr, &empty_space)) + if (extend_directory(info, buff, block_size, max_entry, rownr, + &empty_space, page_type == HEAD_PAGE)) goto crashed_file; } - if (extend_area_on_page(page_type == HEAD_PAGE ? info : 0, buff, - dir, rownr, block_size, + if (extend_area_on_page(info, buff, dir, rownr, (uint) data_length, &empty_space, - &rec_offset, &length)) + &rec_offset, &length, page_type == HEAD_PAGE)) goto crashed_file; } } @@ -6488,7 +6524,6 @@ uint _ma_apply_redo_purge_row_head_or_tail(MARIA_HA *info, LSN lsn, MARIA_SHARE *share= info->s; pgcache_page_no_t page; uint rownr, empty_space; - uint block_size= share->block_size; uchar *buff; int result; uint error; @@ -6535,7 +6570,7 @@ uint _ma_apply_redo_purge_row_head_or_tail(MARIA_HA *info, LSN lsn, DBUG_ASSERT((buff[PAGE_TYPE_OFFSET] & PAGE_TYPE_MASK) == (uchar) page_type); - if (delete_dir_entry(buff, block_size, rownr, &empty_space) < 0) + if (delete_dir_entry(share, buff, rownr, &empty_space) < 0) { _ma_set_fatal_error(share, HA_ERR_WRONG_IN_RECORD); goto err; @@ -6753,7 +6788,7 @@ uint _ma_apply_redo_insert_row_blobs(MARIA_HA *info, { MARIA_SHARE *share= info->s; const uchar *data; - uint data_size= FULL_PAGE_SIZE(share->block_size); + uint data_size= FULL_PAGE_SIZE(share); uint blob_count, ranges; uint16 sid; pgcache_page_no_t first_page2= ULONGLONG_MAX, last_page2= 0; @@ -6885,6 +6920,8 @@ uint _ma_apply_redo_insert_row_blobs(MARIA_HA *info, */ lsn_store(buff, lsn); buff[PAGE_TYPE_OFFSET]= BLOB_PAGE; + bzero(buff + LSN_SIZE + PAGE_TYPE_SIZE, + FULL_PAGE_HEADER_SIZE(share) - (LSN_SIZE + PAGE_TYPE_SIZE)); if (data_on_page != data_size) { @@ -6895,7 +6932,7 @@ uint _ma_apply_redo_insert_row_blobs(MARIA_HA *info, bzero(buff + share->block_size - PAGE_SUFFIX_SIZE - empty_space, empty_space); } - memcpy(buff+ PAGE_TYPE_OFFSET + 1, data, data_on_page); + memcpy(buff + FULL_PAGE_HEADER_SIZE(share), data, data_on_page); if (pagecache_write(share->pagecache, &info->dfile, page, 0, buff, PAGECACHE_PLAIN_PAGE, @@ -7499,7 +7536,7 @@ void maria_ignore_trids(MARIA_HA *info) /* The following functions are useful to call from debugger */ -void _ma_print_block_info(uchar *buff) +void _ma_print_block_info(MARIA_SHARE *share, uchar *buff) { LSN lsn= lsn_korr(buff); @@ -7512,6 +7549,6 @@ void _ma_print_block_info(uchar *buff) printf("Start of directory: %lu\n", maria_block_size - PAGE_SUFFIX_SIZE - (uint) buff[DIR_COUNT_OFFSET] * DIR_ENTRY_SIZE); - _ma_print_directory(stdout, buff, maria_block_size); + _ma_print_directory(share, stdout, buff, maria_block_size); } #endif diff --git a/storage/maria/ma_blockrec.h b/storage/maria/ma_blockrec.h index 1caf09ef1f5..3ea1fedf237 100644 --- a/storage/maria/ma_blockrec.h +++ b/storage/maria/ma_blockrec.h @@ -23,14 +23,27 @@ #define EMPTY_SPACE_SIZE 2 /* Stores empty space on page */ #define PAGE_TYPE_SIZE 1 #define PAGE_SUFFIX_SIZE 4 /* Bytes for checksum */ -#define PAGE_HEADER_SIZE (LSN_SIZE + DIR_COUNT_SIZE + DIR_FREE_SIZE +\ +#define PAGE_HEADER_SIZE_RAW (LSN_SIZE + DIR_COUNT_SIZE + DIR_FREE_SIZE + \ EMPTY_SPACE_SIZE + PAGE_TYPE_SIZE) -#define PAGE_OVERHEAD_SIZE (PAGE_HEADER_SIZE + DIR_ENTRY_SIZE + \ + +#define PAGE_HEADER_SIZE(share) (PAGE_HEADER_SIZE_RAW + \ + (share)->crypt_page_header_space) + +#define PAGE_OVERHEAD_SIZE_RAW (PAGE_HEADER_SIZE_RAW + DIR_ENTRY_SIZE + \ PAGE_SUFFIX_SIZE) +#define PAGE_OVERHEAD_SIZE(share) (PAGE_OVERHEAD_SIZE_RAW + \ + (share)->crypt_page_header_space) + #define BLOCK_RECORD_POINTER_SIZE 6 -#define FULL_PAGE_SIZE(block_size) ((block_size) - LSN_SIZE - \ - PAGE_TYPE_SIZE - PAGE_SUFFIX_SIZE) +#define FULL_PAGE_HEADER_SIZE(share) (LSN_SIZE + PAGE_TYPE_SIZE + \ + (share)->crypt_page_header_space) +#define FULL_PAGE_SIZE(share) ((share)->block_size - \ + FULL_PAGE_HEADER_SIZE(share) - \ + PAGE_SUFFIX_SIZE) + +#define FULL_PAGE_SIZE2(block_size, crypt_size) \ + ((block_size) - (LSN_SIZE + PAGE_TYPE_SIZE + PAGE_SUFFIX_SIZE + (crypt_size))) #define ROW_EXTENT_PAGE_SIZE 5 #define ROW_EXTENT_COUNT_SIZE 2 @@ -68,6 +81,9 @@ enum en_page_type { UNALLOCATED_PAGE, HEAD_PAGE, TAIL_PAGE, BLOB_PAGE, MAX_PAGE_ #define DIR_COUNT_OFFSET (LSN_SIZE+PAGE_TYPE_SIZE) #define DIR_FREE_OFFSET (DIR_COUNT_OFFSET+DIR_COUNT_SIZE) #define EMPTY_SPACE_OFFSET (DIR_FREE_OFFSET+DIR_FREE_SIZE) + /* for encryption */ +#define KEY_VERSION_OFFSET (EMPTY_SPACE_OFFSET+EMPTY_SPACE_SIZE) +#define FULL_PAGE_KEY_VERSION_OFFSET (PAGE_TYPE_OFFSET + PAGE_TYPE_SIZE) /* Bits used for flag uchar (one byte, first in record) */ #define ROW_FLAG_TRANSID 1 @@ -176,7 +192,7 @@ my_bool _ma_write_block_record(MARIA_HA *info, const uchar *record); my_bool _ma_write_abort_block_record(MARIA_HA *info); my_bool _ma_compare_block_record(register MARIA_HA *info, register const uchar *record); -void _ma_compact_block_page(uchar *buff, uint block_size, uint rownr, +void _ma_compact_block_page(MARIA_SHARE *share, uchar *buff, uint rownr, my_bool extend_block, TrID min_read_from, uint min_row_length); my_bool enough_free_entries_on_page(MARIA_SHARE *share, uchar *page_buff); diff --git a/storage/maria/ma_cache.c b/storage/maria/ma_cache.c index 35926d37e03..24739671be6 100644 --- a/storage/maria/ma_cache.c +++ b/storage/maria/ma_cache.c @@ -42,6 +42,7 @@ my_bool _ma_read_cache(MARIA_HA *handler, IO_CACHE *info, uchar *buff, my_off_t offset; uchar *in_buff_pos; DBUG_ENTER("_ma_read_cache"); + DBUG_ASSERT(!(info->myflags & MY_ENCRYPT)); if (pos < info->pos_in_file) { @@ -81,7 +82,7 @@ my_bool _ma_read_cache(MARIA_HA *handler, IO_CACHE *info, uchar *buff, } else info->read_pos=info->read_end; /* All block used */ - if (!(*info->read_function)(info,buff,length)) + if (!_my_b_read(info,buff,length)) DBUG_RETURN(0); read_length=info->error; } diff --git a/storage/maria/ma_check.c b/storage/maria/ma_check.c index a189af40362..5035df26b18 100644 --- a/storage/maria/ma_check.c +++ b/storage/maria/ma_check.c @@ -1583,7 +1583,7 @@ static int check_page_layout(HA_CHECK *param, MARIA_HA *info, block_size= info->s->block_size; empty= 0; - last_row_end= PAGE_HEADER_SIZE; + last_row_end= PAGE_HEADER_SIZE(info->s); *real_rows_found= 0; /* Check free directory list */ @@ -1862,8 +1862,6 @@ static int check_block_record(HA_CHECK *param, MARIA_HA *info, int extend, page_type, bitmap_pattern; uint bitmap_for_page; - LINT_INIT(empty_space); - if (_ma_killed_ptr(param)) { _ma_scan_end_block_record(info); @@ -1936,7 +1934,7 @@ static int check_block_record(HA_CHECK *param, MARIA_HA *info, int extend, row_count= page_buff[DIR_COUNT_OFFSET]; empty_space= uint2korr(page_buff + EMPTY_SPACE_OFFSET); param->used+= block_size - empty_space; - param->link_used+= (PAGE_HEADER_SIZE + PAGE_SUFFIX_SIZE + + param->link_used+= (PAGE_HEADER_SIZE(info->s) + PAGE_SUFFIX_SIZE + row_count * DIR_ENTRY_SIZE); if (empty_space < share->bitmap.sizes[3]) param->lost+= empty_space; @@ -1950,7 +1948,7 @@ static int check_block_record(HA_CHECK *param, MARIA_HA *info, int extend, row_count= page_buff[DIR_COUNT_OFFSET]; empty_space= uint2korr(page_buff + EMPTY_SPACE_OFFSET); param->used+= block_size - empty_space; - param->link_used+= (PAGE_HEADER_SIZE + PAGE_SUFFIX_SIZE + + param->link_used+= (PAGE_HEADER_SIZE(info->s) + PAGE_SUFFIX_SIZE + row_count * DIR_ENTRY_SIZE); if (empty_space < share->bitmap.sizes[6]) param->lost+= empty_space; @@ -1964,7 +1962,7 @@ static int check_block_record(HA_CHECK *param, MARIA_HA *info, int extend, full_page_count++; full_dir= 0; empty_space= block_size; /* for error reporting */ - param->link_used+= (LSN_SIZE + PAGE_TYPE_SIZE); + param->link_used+= FULL_PAGE_HEADER_SIZE(info->s); param->used+= block_size; break; } @@ -3189,17 +3187,23 @@ err2: /** - @brief put CRC on the page + @brief write a page directly to index file - @param buff reference on the page buffer. - @param pos position of the page in the file. - @param length length of the page */ -static void put_crc(uchar *buff, my_off_t pos, MARIA_SHARE *share) +static int write_page(MARIA_SHARE *share, File file, + uchar *buff, uint block_size, + my_off_t pos, int myf_rw) { - maria_page_crc_set_index(buff, (pgcache_page_no_t) (pos / share->block_size), - (uchar*) share); + int res; + PAGECACHE_IO_HOOK_ARGS args; + args.page= buff; + args.pageno= (pgcache_page_no_t) (pos / share->block_size); + args.data= (uchar*) share; + (* share->kfile.pre_write_hook)(&args); + res= my_pwrite(file, args.page, block_size, pos, myf_rw); + (* share->kfile.post_write_hook)(res, &args); + return res; } @@ -3288,9 +3292,8 @@ static int sort_one_index(HA_CHECK *param, MARIA_HA *info, /* Fill block with zero and write it to the new index file */ length= page.size; bzero(buff+length,keyinfo->block_length-length); - put_crc(buff, new_page_pos, share); - if (my_pwrite(new_file, buff,(uint) keyinfo->block_length, - new_page_pos,MYF(MY_NABP | MY_WAIT_IF_FULL))) + if (write_page(share, new_file, buff, keyinfo->block_length, + new_page_pos, MYF(MY_NABP | MY_WAIT_IF_FULL))) { _ma_check_print_error(param,"Can't write indexblock, error: %d",my_errno); goto err; @@ -3481,7 +3484,8 @@ static my_bool maria_zerofill_data(HA_CHECK *param, MARIA_HA *info, { my_bool is_head_page= (page_type == HEAD_PAGE); dir= dir_entry_pos(buff, block_size, max_entry - 1); - _ma_compact_block_page(buff, block_size, max_entry -1, 0, + _ma_compact_block_page(share, + buff, max_entry -1, 0, is_head_page ? ~(TrID) 0 : 0, is_head_page ? share->base.min_block_length : 0); @@ -4015,8 +4019,9 @@ int maria_repair_by_sort(HA_CHECK *param, register MARIA_HA *info, share->state.state.data_file_length=sort_param.max_pos; param->read_cache.file= info->dfile.file; /* re-init read cache */ - reinit_io_cache(¶m->read_cache,READ_CACHE,share->pack.header_length, - 1,1); + if (share->data_file_type != BLOCK_RECORD) + reinit_io_cache(¶m->read_cache, READ_CACHE, + share->pack.header_length, 1, 1); } if (param->testflag & T_WRITE_LOOP) @@ -4261,20 +4266,14 @@ int maria_repair_parallel(HA_CHECK *param, register MARIA_HA *info, if (!(sort_info.key_block= alloc_key_blocks(param, (uint) param->sort_key_blocks, - share->base.max_key_block_length)) || - init_io_cache(¶m->read_cache, info->dfile.file, + share->base.max_key_block_length))) + goto err; + + if (init_io_cache(¶m->read_cache, info->dfile.file, (uint) param->read_buffer_length, - READ_CACHE, share->pack.header_length, 1, MYF(MY_WME)) || - (!rep_quick && - (init_io_cache(&info->rec_cache, info->dfile.file, - (uint) param->write_buffer_length, - WRITE_CACHE, new_header_length, 1, - MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw) || - init_io_cache(&new_data_cache, -1, - (uint) param->write_buffer_length, - READ_CACHE, new_header_length, 1, - MYF(MY_WME | MY_DONT_CHECK_FILESIZE))))) + READ_CACHE, share->pack.header_length, 1, MYF(MY_WME))) goto err; + sort_info.key_block_end=sort_info.key_block+param->sort_key_blocks; info->opt_flag|=WRITE_CACHE_USED; info->rec_cache.file= info->dfile.file; /* for sort_delete_record */ @@ -4301,7 +4300,19 @@ int maria_repair_parallel(HA_CHECK *param, register MARIA_HA *info, if (param->testflag & T_UNPACK) restore_data_file_type(share); share->state.dellink= HA_OFFSET_ERROR; - info->rec_cache.file=new_file; + + if (init_io_cache(&new_data_cache, -1, + (uint) param->write_buffer_length, + READ_CACHE, new_header_length, 1, + MYF(MY_WME | MY_DONT_CHECK_FILESIZE))) + goto err; + + if (init_io_cache(&info->rec_cache, new_file, + (uint) param->write_buffer_length, + WRITE_CACHE, new_header_length, 1, + MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw)) + goto err; + } /* Optionally drop indexes and optionally modify the key_map. */ @@ -5789,9 +5800,8 @@ static int sort_insert_key(MARIA_SORT_PARAM *sort_param, } else { - put_crc(anc_buff, filepos, share); - if (my_pwrite(share->kfile.file, anc_buff, - (uint) keyinfo->block_length, filepos, param->myf_rw)) + if (write_page(share, share->kfile.file, anc_buff, + keyinfo->block_length, filepos, param->myf_rw)) DBUG_RETURN(1); } DBUG_DUMP("buff", anc_buff, _ma_get_page_used(share, anc_buff)); @@ -5919,9 +5929,8 @@ int _ma_flush_pending_blocks(MARIA_SORT_PARAM *sort_param) } else { - put_crc(key_block->buff, filepos, info->s); - if (my_pwrite(info->s->kfile.file, key_block->buff, - (uint) keyinfo->block_length,filepos, myf_rw)) + if (write_page(info->s, info->s->kfile.file, key_block->buff, + keyinfo->block_length, filepos, myf_rw)) goto err; } DBUG_DUMP("buff",key_block->buff,length); @@ -6626,17 +6635,6 @@ static void copy_data_file_state(MARIA_STATE_INFO *to, } -/* Return 1 if block is full of zero's */ - -static my_bool zero_filled_block(uchar *tmp, uint length) -{ - while (length--) - if (*(tmp++) != 0) - return 0; - return 1; -} - - /* Read 'safely' next record while scanning table. @@ -6698,7 +6696,8 @@ static int _ma_safe_scan_block_record(MARIA_SORT_INFO *sort_info, info->scan.dir-= DIR_ENTRY_SIZE; /* Point to previous row */ if (end_of_data > info->scan.dir_end || - offset < PAGE_HEADER_SIZE || length < share->base.min_block_length) + offset < PAGE_HEADER_SIZE(info->s) || + length < share->base.min_block_length) { _ma_check_print_info(sort_info->param, "Wrong directory entry %3u at page %s", @@ -6743,8 +6742,7 @@ read_next_page: sometimes be found at end of a bitmap when we wrote a big record last that was moved to the next bitmap. */ - if (!zero_filled_block(info->scan.page_buff, share->block_size) || - _ma_check_bitmap_data(info, UNALLOCATED_PAGE, 0, + if (_ma_check_bitmap_data(info, UNALLOCATED_PAGE, 0, _ma_bitmap_get_page_bits(info, &share->bitmap, page))) diff --git a/storage/maria/ma_check_standalone.h b/storage/maria/ma_check_standalone.h index 241bc7c2739..6626a62c4ef 100644 --- a/storage/maria/ma_check_standalone.h +++ b/storage/maria/ma_check_standalone.h @@ -23,6 +23,43 @@ void _mi_report_crashed(void *file __attribute__((unused)), { } +static unsigned int no_key() +{ + return ENCRYPTION_KEY_VERSION_INVALID; +} + +struct encryption_service_st encryption_handler= +{ + no_key, 0, 0, 0, 0, 0, 0 +}; + +int encryption_scheme_encrypt(const unsigned char* src __attribute__((unused)), + unsigned int slen __attribute__((unused)), + unsigned char* dst __attribute__((unused)), + unsigned int* dlen __attribute__((unused)), + struct st_encryption_scheme *scheme __attribute__((unused)), + unsigned int key_version __attribute__((unused)), + unsigned int i32_1 __attribute__((unused)), + unsigned int i32_2 __attribute__((unused)), + unsigned long long i64 __attribute__((unused))) +{ + return -1; +} + + +int encryption_scheme_decrypt(const unsigned char* src __attribute__((unused)), + unsigned int slen __attribute__((unused)), + unsigned char* dst __attribute__((unused)), + unsigned int* dlen __attribute__((unused)), + struct st_encryption_scheme *scheme __attribute__((unused)), + unsigned int key_version __attribute__((unused)), + unsigned int i32_1 __attribute__((unused)), + unsigned int i32_2 __attribute__((unused)), + unsigned long long i64 __attribute__((unused))) +{ + return -1; +} + /* only those that included myisamchk.h may need and can use the below */ #ifdef _myisamchk_h /* @@ -121,5 +158,6 @@ void _ma_check_print_error(HA_CHECK *param, const char *fmt,...) va_end(args); DBUG_VOID_RETURN; } + #endif diff --git a/storage/maria/ma_checkpoint.c b/storage/maria/ma_checkpoint.c index a625a828b33..21b4758a720 100644 --- a/storage/maria/ma_checkpoint.c +++ b/storage/maria/ma_checkpoint.c @@ -36,7 +36,7 @@ #include "ma_checkpoint.h" #include "ma_loghandler_lsn.h" #include "ma_servicethread.h" - +#include "ma_crypt.h" /** @brief type of checkpoint currently running */ static CHECKPOINT_LEVEL checkpoint_in_progress= CHECKPOINT_NONE; @@ -230,7 +230,7 @@ static int really_execute_checkpoint(void) sizeof(checkpoint_start_log_horizon_char); for (i= 0; i < (sizeof(record_pieces)/sizeof(record_pieces[0])); i++) { - log_array[TRANSLOG_INTERNAL_PARTS + 1 + i].str= (uchar*) record_pieces[i].str; + log_array[TRANSLOG_INTERNAL_PARTS + 1 + i].str= (uchar*)record_pieces[i].str; log_array[TRANSLOG_INTERNAL_PARTS + 1 + i].length= record_pieces[i].length; total_rec_length+= (translog_size_t) record_pieces[i].length; } @@ -1107,6 +1107,7 @@ static int collect_tables(LEX_STRING *str, LSN checkpoint_start_log_horizon) mysql_mutex_destroy(&share->intern_lock); mysql_mutex_unlock(&share->close_lock); mysql_mutex_destroy(&share->close_lock); + ma_crypt_free(share); my_free(share); } else @@ -1220,6 +1221,7 @@ err: { /* maria_close() left us to free the share */ mysql_mutex_destroy(&share->intern_lock); + ma_crypt_free(share); my_free(share); } else diff --git a/storage/maria/ma_close.c b/storage/maria/ma_close.c index 4532b029126..022da39002b 100644 --- a/storage/maria/ma_close.c +++ b/storage/maria/ma_close.c @@ -21,6 +21,7 @@ */ #include "maria_def.h" +#include "ma_crypt.h" int maria_close(register MARIA_HA *info) { @@ -240,6 +241,7 @@ int maria_close(register MARIA_HA *info) } if (share_can_be_freed) { + ma_crypt_free(share); (void) mysql_mutex_destroy(&share->intern_lock); (void) mysql_mutex_destroy(&share->close_lock); (void) mysql_cond_destroy(&share->key_del_cond); diff --git a/storage/maria/ma_create.c b/storage/maria/ma_create.c index 1176b2037b5..f160499a94e 100644 --- a/storage/maria/ma_create.c +++ b/storage/maria/ma_create.c @@ -20,6 +20,7 @@ #include <my_bit.h> #include "ma_blockrec.h" #include "trnman_public.h" +#include "ma_crypt.h" #if defined(MSDOS) || defined(__WIN__) #ifdef __WIN__ @@ -72,6 +73,9 @@ int maria_create(const char *name, enum data_file_type datafile_type, my_bool forced_packed; myf sync_dir= 0; uchar *log_data= NULL; + my_bool encrypted= maria_encrypt_tables && datafile_type == BLOCK_RECORD; + my_bool insert_order= MY_TEST(flags & HA_PRESERVE_INSERT_ORDER); + uint crypt_page_header_space= 0; DBUG_ENTER("maria_create"); DBUG_PRINT("enter", ("keys: %u columns: %u uniques: %u flags: %u", keys, columns, uniques, flags)); @@ -140,6 +144,12 @@ int maria_create(const char *name, enum data_file_type datafile_type, forced_packed= 0; column_nr= 0; + if (encrypted) + { + DBUG_ASSERT(datafile_type == BLOCK_RECORD); + crypt_page_header_space= ma_crypt_get_data_page_header_space(); + } + for (column= columndef, end_column= column + columns ; column != end_column ; column++) @@ -160,7 +170,8 @@ int maria_create(const char *name, enum data_file_type datafile_type, if (type == FIELD_SKIP_PRESPACE) type= column->type= FIELD_NORMAL; /* SKIP_PRESPACE not supported */ if (type == FIELD_NORMAL && - column->length > FULL_PAGE_SIZE(maria_block_size)) + column->length > FULL_PAGE_SIZE2(maria_block_size, + crypt_page_header_space)) { /* FIELD_NORMAL can't be split over many blocks, convert to a CHAR */ type= column->type= FIELD_SKIP_ENDSPACE; @@ -256,6 +267,19 @@ int maria_create(const char *name, enum data_file_type datafile_type, datafile_type= BLOCK_RECORD; } + if (encrypted) + { + /* + datafile_type is set (finally?) + update encryption that is only supported for BLOCK_RECORD + */ + if (datafile_type != BLOCK_RECORD) + { + encrypted= FALSE; + crypt_page_header_space= 0; + } + } + if (datafile_type == DYNAMIC_RECORD) options|= HA_OPTION_PACK_RECORD; /* Must use packed records */ @@ -340,9 +364,9 @@ int maria_create(const char *name, enum data_file_type datafile_type, { if (datafile_type == BLOCK_RECORD) { - uint rows_per_page= ((maria_block_size - PAGE_OVERHEAD_SIZE) / - (min_pack_length + extra_header_size + - DIR_ENTRY_SIZE)); + uint rows_per_page= + ((maria_block_size - PAGE_OVERHEAD_SIZE_RAW - crypt_page_header_space) + / (min_pack_length + extra_header_size + DIR_ENTRY_SIZE)); ulonglong data_file_length= ci->data_file_length; if (!data_file_length) data_file_length= ((((ulonglong) 1 << ((BLOCK_RECORD_POINTER_SIZE-1) * @@ -665,7 +689,20 @@ int maria_create(const char *name, enum data_file_type datafile_type, (key_segs + unique_key_parts)*HA_KEYSEG_SIZE+ columns*(MARIA_COLUMNDEF_SIZE + 2)); - DBUG_PRINT("info", ("info_length: %u", info_length)); + if (encrypted) + { + share.base.extra_options|= MA_EXTRA_OPTIONS_ENCRYPTED; + + /* store crypt data in info */ + info_length+= ma_crypt_get_file_length(); + } + + if (insert_order) + { + share.base.extra_options|= MA_EXTRA_OPTIONS_INSERT_ORDER; + } + + DBUG_PRINT("info", ("info_length: %u", info_length)); /* There are only 16 bits for the total header length. */ if (info_length > 65535) { @@ -1003,6 +1040,13 @@ int maria_create(const char *name, enum data_file_type datafile_type, if (_ma_column_nr_write(file, column_array, columns)) goto err; + if (encrypted) + { + if (ma_crypt_create(&share) || + ma_crypt_write(&share, file)) + goto err; + } + if ((kfile_size_before_extension= mysql_file_tell(file,MYF(0))) == MY_FILEPOS_ERROR) goto err; #ifndef DBUG_OFF @@ -1178,6 +1222,7 @@ int maria_create(const char *name, enum data_file_type datafile_type, mysql_mutex_unlock(&THR_LOCK_maria); res= 0; my_free((char*) rec_per_key_part); + ma_crypt_free(&share); errpos=0; if (mysql_file_close(file,MYF(0))) res= my_errno; @@ -1208,6 +1253,7 @@ err_no_lock: MY_UNPACK_FILENAME | MY_APPEND_EXT), sync_dir); } + ma_crypt_free(&share); my_free(log_data); my_free(rec_per_key_part); DBUG_RETURN(my_errno=save_errno); /* return the fatal errno */ diff --git a/storage/maria/ma_crypt.c b/storage/maria/ma_crypt.c new file mode 100644 index 00000000000..cc605d79933 --- /dev/null +++ b/storage/maria/ma_crypt.c @@ -0,0 +1,517 @@ +/* + Copyright (c) 2013 Google Inc. + Copyright (c) 2014, 2015 MariaDB Corporation + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include <my_global.h> +#include "maria_def.h" +#include "ma_blockrec.h" +#include <my_crypt.h> + +#define CRYPT_SCHEME_1 1 +#define CRYPT_SCHEME_1_ID_LEN 4 /* 4 bytes for counter-block */ +#define CRYPT_SCHEME_1_IV_LEN 16 +#define CRYPT_SCHEME_1_KEY_VERSION_SIZE 4 + +#ifdef HAVE_PSI_INTERFACE +PSI_mutex_key key_CRYPT_DATA_lock; +#endif + +struct st_crypt_key +{ + uint key_version; + uchar key[CRYPT_SCHEME_1_IV_LEN]; +}; + +struct st_maria_crypt_data +{ + struct st_encryption_scheme scheme; + uint space; + mysql_mutex_t lock; /* protecting keys */ +}; + +/** + determine what key id to use for Aria encryption + + Same logic as for tempfiles: if key id 2 exists - use it, + otherwise use key id 1. + + Key id 1 is system, it always exists. Key id 2 is optional, + it allows to specify fast low-grade encryption for temporary data. +*/ +static uint get_encryption_key_id(MARIA_SHARE *share) +{ + if (share->options & HA_OPTION_TMP_TABLE && + encryption_key_id_exists(ENCRYPTION_KEY_TEMPORARY_DATA)) + return ENCRYPTION_KEY_TEMPORARY_DATA; + else + return ENCRYPTION_KEY_SYSTEM_DATA; +} + +uint +ma_crypt_get_data_page_header_space() +{ + return CRYPT_SCHEME_1_KEY_VERSION_SIZE; +} + +uint +ma_crypt_get_index_page_header_space(MARIA_SHARE *share) +{ + if (share->base.born_transactional) + { + return CRYPT_SCHEME_1_KEY_VERSION_SIZE; + } + else + { + /* if the index is not transactional, we add 7 bytes LSN anyway + to be used for counter block + */ + return LSN_STORE_SIZE + CRYPT_SCHEME_1_KEY_VERSION_SIZE; + } +} + +uint +ma_crypt_get_file_length() +{ + return 2 + CRYPT_SCHEME_1_IV_LEN + CRYPT_SCHEME_1_ID_LEN; +} + +static void crypt_data_scheme_locker(struct st_encryption_scheme *scheme, + int unlock) +{ + MARIA_CRYPT_DATA *crypt_data = (MARIA_CRYPT_DATA*)scheme; + if (unlock) + mysql_mutex_unlock(&crypt_data->lock); + else + mysql_mutex_lock(&crypt_data->lock); +} + +int +ma_crypt_create(MARIA_SHARE* share) +{ + MARIA_CRYPT_DATA *crypt_data= + (MARIA_CRYPT_DATA*)my_malloc(sizeof(MARIA_CRYPT_DATA), MYF(MY_ZEROFILL)); + crypt_data->scheme.type= CRYPT_SCHEME_1; + crypt_data->scheme.locker= crypt_data_scheme_locker; + mysql_mutex_init(key_CRYPT_DATA_lock, &crypt_data->lock, MY_MUTEX_INIT_FAST); + crypt_data->scheme.key_id= get_encryption_key_id(share); + my_random_bytes(crypt_data->scheme.iv, sizeof(crypt_data->scheme.iv)); + my_random_bytes((uchar*)&crypt_data->space, sizeof(crypt_data->space)); + share->crypt_data= crypt_data; + share->crypt_page_header_space= CRYPT_SCHEME_1_KEY_VERSION_SIZE; + return 0; +} + +void +ma_crypt_free(MARIA_SHARE* share) +{ + if (share->crypt_data != NULL) + { + mysql_mutex_destroy(&share->crypt_data->lock); + my_free(share->crypt_data); + share->crypt_data= NULL; + } +} + +int +ma_crypt_write(MARIA_SHARE* share, File file) +{ + MARIA_CRYPT_DATA *crypt_data= share->crypt_data; + uchar buff[2 + 4 + sizeof(crypt_data->scheme.iv)]; + if (crypt_data == 0) + return 0; + + buff[0]= crypt_data->scheme.type; + buff[1]= sizeof(buff) - 2; + + int4store(buff + 2, crypt_data->space); + memcpy(buff + 6, crypt_data->scheme.iv, sizeof(crypt_data->scheme.iv)); + + if (mysql_file_write(file, buff, sizeof(buff), MYF(MY_NABP))) + return 1; + + return 0; +} + +uchar* +ma_crypt_read(MARIA_SHARE* share, uchar *buff) +{ + uchar type= buff[0]; + uchar iv_length= buff[1]; + + /* currently only supported type */ + if (type != CRYPT_SCHEME_1 || + iv_length != sizeof(((MARIA_CRYPT_DATA*)1)->scheme.iv) + 4) + { + my_printf_error(HA_ERR_UNSUPPORTED, + "Unsupported crypt scheme! type: %d iv_length: %d\n", + MYF(ME_FATALERROR|ME_NOREFRESH), + type, iv_length); + return 0; + } + + if (share->crypt_data == NULL) + { + /* opening a table */ + MARIA_CRYPT_DATA *crypt_data= + (MARIA_CRYPT_DATA*)my_malloc(sizeof(MARIA_CRYPT_DATA), MYF(MY_ZEROFILL)); + + crypt_data->scheme.type= type; + mysql_mutex_init(key_CRYPT_DATA_lock, &crypt_data->lock, + MY_MUTEX_INIT_FAST); + crypt_data->scheme.locker= crypt_data_scheme_locker; + crypt_data->scheme.key_id= get_encryption_key_id(share); + crypt_data->space= uint4korr(buff + 2); + memcpy(crypt_data->scheme.iv, buff + 6, sizeof(crypt_data->scheme.iv)); + share->crypt_data= crypt_data; + } + + share->crypt_page_header_space= CRYPT_SCHEME_1_KEY_VERSION_SIZE; + return buff + 2 + iv_length; +} + +static int ma_encrypt(MARIA_SHARE *, MARIA_CRYPT_DATA *, const uchar *, + uchar *, uint, uint, LSN, uint *); +static int ma_decrypt(MARIA_SHARE *, MARIA_CRYPT_DATA *, const uchar *, + uchar *, uint, uint, LSN, uint); + +static my_bool ma_crypt_pre_read_hook(PAGECACHE_IO_HOOK_ARGS *args) +{ + MARIA_SHARE *share= (MARIA_SHARE*) args->data; + uchar *crypt_buf= my_malloc(share->block_size, MYF(0)); + if (crypt_buf == NULL) + { + args->crypt_buf= NULL; /* for post-hook */ + return 1; + } + + /* swap pointers to read into crypt_buf */ + args->crypt_buf= args->page; + args->page= crypt_buf; + + return 0; +} + +static my_bool ma_crypt_data_post_read_hook(int res, + PAGECACHE_IO_HOOK_ARGS *args) +{ + MARIA_SHARE *share= (MARIA_SHARE*) args->data; + const uint size= share->block_size; + const uchar page_type= args->page[PAGE_TYPE_OFFSET] & PAGE_TYPE_MASK; + const uint32 key_version_offset= (page_type <= TAIL_PAGE) ? + KEY_VERSION_OFFSET : FULL_PAGE_KEY_VERSION_OFFSET; + + if (res == 0) + { + const uchar *src= args->page; + uchar* dst= args->crypt_buf; + uint pageno= (uint)args->pageno; + LSN lsn= lsn_korr(src); + const uint head= (page_type <= TAIL_PAGE) ? + PAGE_HEADER_SIZE(share) : FULL_PAGE_HEADER_SIZE(share); + const uint tail= CRC_SIZE; + const uint32 key_version= uint4korr(src + key_version_offset); + + /* 1 - copy head */ + memcpy(dst, src, head); + /* 2 - decrypt page */ + res= ma_decrypt(share, share->crypt_data, + src + head, dst + head, size - (head + tail), pageno, lsn, + key_version); + /* 3 - copy tail */ + memcpy(dst + size - tail, src + size - tail, tail); + /* 4 clear key version to get correct crc */ + int4store(dst + key_version_offset, 0); + } + + if (args->crypt_buf != NULL) + { + uchar *tmp= args->page; + args->page= args->crypt_buf; + args->crypt_buf= NULL; + my_free(tmp); + } + + return maria_page_crc_check_data(res, args); +} + +static void store_rand_lsn(uchar * page) +{ + LSN lsn= 0; + lsn+= rand(); + lsn<<= 32; + lsn+= rand(); + lsn_store(page, lsn); +} + +static my_bool ma_crypt_data_pre_write_hook(PAGECACHE_IO_HOOK_ARGS *args) +{ + MARIA_SHARE *share= (MARIA_SHARE*) args->data; + const uint size= share->block_size; + uint key_version; + uchar *crypt_buf= my_malloc(share->block_size, MYF(0)); + + if (crypt_buf == NULL) + { + args->crypt_buf= NULL; /* for post-hook */ + return 1; + } + + if (!share->now_transactional) + { + /* store a random number instead of LSN (for counter block) */ + store_rand_lsn(args->page); + } + + maria_page_crc_set_normal(args); + + { + const uchar *src= args->page; + uchar* dst= crypt_buf; + uint pageno= (uint)args->pageno; + LSN lsn= lsn_korr(src); + const uchar page_type= src[PAGE_TYPE_OFFSET] & PAGE_TYPE_MASK; + const uint head= (page_type <= TAIL_PAGE) ? + PAGE_HEADER_SIZE(share) : FULL_PAGE_HEADER_SIZE(share); + const uint tail= CRC_SIZE; + const uint32 key_version_offset= (page_type <= TAIL_PAGE) ? + KEY_VERSION_OFFSET : FULL_PAGE_KEY_VERSION_OFFSET; + + DBUG_ASSERT(page_type < MAX_PAGE_TYPE); + + /* 1 - copy head */ + memcpy(dst, src, head); + /* 2 - encrypt page */ + if (ma_encrypt(share, share->crypt_data, + src + head, dst + head, size - (head + tail), pageno, lsn, + &key_version)) + return 1; + /* 3 - copy tail */ + memcpy(dst + size - tail, src + size - tail, tail); + /* 4 - store key version */ + int4store(dst + key_version_offset, key_version); + } + + /* swap pointers to instead write out the encrypted block */ + args->crypt_buf= args->page; + args->page= crypt_buf; + + return 0; +} + +static void ma_crypt_post_write_hook(int res, + PAGECACHE_IO_HOOK_ARGS *args) +{ + if (args->crypt_buf != NULL) + { + uchar *tmp= args->page; + args->page= args->crypt_buf; + args->crypt_buf= NULL; + my_free(tmp); + } + + maria_page_write_failure(res, args); +} + +void ma_crypt_set_data_pagecache_callbacks(PAGECACHE_FILE *file, + MARIA_SHARE *share + __attribute__((unused))) +{ + /* Only use encryption if we have defined it */ + if (encryption_key_id_exists(get_encryption_key_id(share))) + { + file->pre_read_hook= ma_crypt_pre_read_hook; + file->post_read_hook= ma_crypt_data_post_read_hook; + file->pre_write_hook= ma_crypt_data_pre_write_hook; + file->post_write_hook= ma_crypt_post_write_hook; + } +} + +static my_bool ma_crypt_index_post_read_hook(int res, + PAGECACHE_IO_HOOK_ARGS *args) +{ + MARIA_SHARE *share= (MARIA_SHARE*) args->data; + const uint block_size= share->block_size; + const uint page_used= _ma_get_page_used(share, args->page); + + if (res == 0 && page_used <= block_size - CRC_SIZE) + { + const uchar *src= args->page; + uchar* dst= args->crypt_buf; + uint pageno= (uint)args->pageno; + LSN lsn= lsn_korr(src); + const uint head= share->keypage_header; + const uint tail= CRC_SIZE; + const uint32 key_version= _ma_get_key_version(share, src); + /* page_used includes header (but not trailer) */ + const uint size= page_used - head; + + /* 1 - copy head */ + memcpy(dst, src, head); + /* 2 - decrypt page */ + res= ma_decrypt(share, share->crypt_data, + src + head, dst + head, size, pageno, lsn, key_version); + /* 3 - copy tail */ + memcpy(dst + block_size - tail, src + block_size - tail, tail); + /* 4 clear key version to get correct crc */ + _ma_store_key_version(share, dst, 0); + } + + if (args->crypt_buf != NULL) + { + uchar *tmp= args->page; + args->page= args->crypt_buf; + args->crypt_buf= NULL; + my_free(tmp); + } + + return maria_page_crc_check_index(res, args); +} + +static my_bool ma_crypt_index_pre_write_hook(PAGECACHE_IO_HOOK_ARGS *args) +{ + MARIA_SHARE *share= (MARIA_SHARE*) args->data; + const uint block_size= share->block_size; + const uint page_used= _ma_get_page_used(share, args->page); + uint key_version; + uchar *crypt_buf= my_malloc(block_size, MYF(0)); + if (crypt_buf == NULL) + { + args->crypt_buf= NULL; /* for post-hook */ + return 1; + } + + if (!share->now_transactional) + { + /* store a random number instead of LSN (for counter block) */ + store_rand_lsn(args->page); + } + + maria_page_crc_set_index(args); + + { + const uchar *src= args->page; + uchar* dst= crypt_buf; + uint pageno= (uint)args->pageno; + LSN lsn= lsn_korr(src); + const uint head= share->keypage_header; + const uint tail= CRC_SIZE; + /* page_used includes header (but not trailer) */ + const uint size= page_used - head; + + /* 1 - copy head */ + memcpy(dst, src, head); + /* 2 - encrypt page */ + if (ma_encrypt(share, share->crypt_data, + src + head, dst + head, size, pageno, lsn, &key_version)) + { + my_free(crypt_buf); + return 1; + } + /* 3 - copy tail */ + memcpy(dst + block_size - tail, src + block_size - tail, tail); + /* 4 - store key version */ + _ma_store_key_version(share, dst, key_version); +#ifdef HAVE_valgrind + /* 5 - keep valgrind happy by zeroing not used bytes */ + bzero(dst+head+size, block_size - size - tail - head); +#endif + } + + /* swap pointers to instead write out the encrypted block */ + args->crypt_buf= args->page; + args->page= crypt_buf; + + return 0; +} + +void ma_crypt_set_index_pagecache_callbacks(PAGECACHE_FILE *file, + MARIA_SHARE *share + __attribute__((unused))) +{ + file->pre_read_hook= ma_crypt_pre_read_hook; + file->post_read_hook= ma_crypt_index_post_read_hook; + file->pre_write_hook= ma_crypt_index_pre_write_hook; + file->post_write_hook= ma_crypt_post_write_hook; +} + +static int ma_encrypt(MARIA_SHARE *share, MARIA_CRYPT_DATA *crypt_data, + const uchar *src, uchar *dst, uint size, + uint pageno, LSN lsn, + uint *key_version) +{ + int rc; + uint32 dstlen= 0; /* Must be set because of error message */ + + *key_version = encryption_key_get_latest_version(crypt_data->scheme.key_id); + if (*key_version == ENCRYPTION_KEY_VERSION_INVALID) + { + /* + We use this error for both encryption and decryption, as in normal + cases it should be impossible to get an error here. + */ + my_errno= HA_ERR_DECRYPTION_FAILED; + my_printf_error(HA_ERR_DECRYPTION_FAILED, + "Unknown key id %u. Can't continue!", + MYF(ME_FATALERROR|ME_NOREFRESH), + crypt_data->scheme.key_id); + return 1; + } + + rc= encryption_scheme_encrypt(src, size, dst, &dstlen, + &crypt_data->scheme, *key_version, + crypt_data->space, pageno, lsn); + + /* The following can only fail if the encryption key is wrong */ + DBUG_ASSERT(!my_assert_on_error || rc == MY_AES_OK); + DBUG_ASSERT(!my_assert_on_error || dstlen == size); + if (! (rc == MY_AES_OK && dstlen == size)) + { + my_errno= HA_ERR_DECRYPTION_FAILED; + my_printf_error(HA_ERR_DECRYPTION_FAILED, + "failed to encrypt '%s' rc: %d dstlen: %u size: %u\n", + MYF(ME_FATALERROR|ME_NOREFRESH), + share->open_file_name.str, rc, dstlen, size); + return 1; + } + + return 0; +} + +static int ma_decrypt(MARIA_SHARE *share, MARIA_CRYPT_DATA *crypt_data, + const uchar *src, uchar *dst, uint size, + uint pageno, LSN lsn, + uint key_version) +{ + int rc; + uint32 dstlen= 0; /* Must be set because of error message */ + + rc= encryption_scheme_decrypt(src, size, dst, &dstlen, + &crypt_data->scheme, key_version, + crypt_data->space, pageno, lsn); + + DBUG_ASSERT(!my_assert_on_error || rc == MY_AES_OK); + DBUG_ASSERT(!my_assert_on_error || dstlen == size); + if (! (rc == MY_AES_OK && dstlen == size)) + { + my_errno= HA_ERR_DECRYPTION_FAILED; + my_printf_error(HA_ERR_DECRYPTION_FAILED, + "failed to decrypt '%s' rc: %d dstlen: %u size: %u\n", + MYF(ME_FATALERROR|ME_NOREFRESH), + share->open_file_name.str, rc, dstlen, size); + return 1; + } + return 0; +} diff --git a/storage/maria/ma_crypt.h b/storage/maria/ma_crypt.h new file mode 100644 index 00000000000..309a8300eb8 --- /dev/null +++ b/storage/maria/ma_crypt.h @@ -0,0 +1,41 @@ +/* + Copyright (c) 2013 Google Inc. + Copyright (c) 2014, 2015 MariaDB Corporation + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#ifndef STORAGE_MARIA_MA_CRYPT_INCLUDED +#define STORAGE_MARIA_MA_CRYPT_INCLUDED + +#include <my_global.h> + +struct st_maria_share; +struct st_pagecache_file; + +uint ma_crypt_get_data_page_header_space();/* bytes in data/index page header */ +uint ma_crypt_get_index_page_header_space(struct st_maria_share *); +uint ma_crypt_get_file_length(); /* bytes needed in file */ +int ma_crypt_create(struct st_maria_share *); /* create encryption data */ +int ma_crypt_write(struct st_maria_share *, File); /* write encryption data */ +uchar* ma_crypt_read(struct st_maria_share *, uchar *buff); /* read crypt data*/ + +void ma_crypt_set_data_pagecache_callbacks(struct st_pagecache_file *file, + struct st_maria_share *share); + +void ma_crypt_set_index_pagecache_callbacks(struct st_pagecache_file *file, + struct st_maria_share *share); + +void ma_crypt_free(struct st_maria_share *share); + +#endif diff --git a/storage/maria/ma_delete.c b/storage/maria/ma_delete.c index 31773ef2dfc..bcea1d4054d 100644 --- a/storage/maria/ma_delete.c +++ b/storage/maria/ma_delete.c @@ -1505,7 +1505,7 @@ my_bool _ma_log_delete(MARIA_PAGE *ma_page, const uchar *key_pos, /* Store keypage_flag */ *log_pos++= KEY_OP_SET_PAGEFLAG; - *log_pos++= ma_page->buff[KEYPAGE_TRANSFLAG_OFFSET]; + *log_pos++= _ma_get_keypage_flag(info->s, ma_page->buff); log_pos[0]= KEY_OP_OFFSET; int2store(log_pos+1, offset); diff --git a/storage/maria/ma_delete_table.c b/storage/maria/ma_delete_table.c index 56a6dfc8e5f..ec68902485b 100644 --- a/storage/maria/ma_delete_table.c +++ b/storage/maria/ma_delete_table.c @@ -78,11 +78,11 @@ int maria_delete_table(const char *name) DBUG_RETURN(1); } - DBUG_RETURN(maria_delete_table_files(name, sync_dir)); + DBUG_RETURN(maria_delete_table_files(name, 0, sync_dir)); } -int maria_delete_table_files(const char *name, myf sync_dir) +int maria_delete_table_files(const char *name, my_bool temporary, myf sync_dir) { char from[FN_REFLEN]; DBUG_ENTER("maria_delete_table_files"); @@ -97,9 +97,12 @@ int maria_delete_table_files(const char *name, myf sync_dir) DBUG_RETURN(my_errno); // optional files from maria_pack: - fn_format(from,name,"",".TMD",MY_UNPACK_FILENAME|MY_APPEND_EXT); - mysql_file_delete_with_symlink(key_file_dfile, from, MYF(0)); - fn_format(from,name,"",".OLD",MY_UNPACK_FILENAME|MY_APPEND_EXT); - mysql_file_delete_with_symlink(key_file_dfile, from, MYF(0)); + if (!temporary) + { + fn_format(from,name,"",".TMD",MY_UNPACK_FILENAME|MY_APPEND_EXT); + mysql_file_delete_with_symlink(key_file_dfile, from, MYF(0)); + fn_format(from,name,"",".OLD",MY_UNPACK_FILENAME|MY_APPEND_EXT); + mysql_file_delete_with_symlink(key_file_dfile, from, MYF(0)); + } DBUG_RETURN(0); } diff --git a/storage/maria/ma_dynrec.c b/storage/maria/ma_dynrec.c index 9fed9dbe8da..2e17a95f390 100644 --- a/storage/maria/ma_dynrec.c +++ b/storage/maria/ma_dynrec.c @@ -250,8 +250,7 @@ my_bool _ma_write_blob_record(MARIA_HA *info, const uchar *record) MARIA_DYN_DELETE_BLOCK_HEADER+1); reclength= (info->s->base.pack_reclength + _ma_calc_total_blob_length(info,record)+ extra); - if (!(rec_buff=(uchar*) my_safe_alloca(reclength, - MARIA_MAX_RECORD_ON_STACK))) + if (!(rec_buff=(uchar*) my_safe_alloca(reclength))) { my_errno= HA_ERR_OUT_OF_MEM; /* purecov: inspected */ return(1); @@ -265,7 +264,7 @@ my_bool _ma_write_blob_record(MARIA_HA *info, const uchar *record) error= write_dynamic_record(info, rec_buff+ALIGN_SIZE(MARIA_MAX_DYN_BLOCK_HEADER), reclength2); - my_safe_afree(rec_buff, reclength, MARIA_MAX_RECORD_ON_STACK); + my_safe_afree(rec_buff, reclength); return(error != 0); } @@ -289,8 +288,7 @@ my_bool _ma_update_blob_record(MARIA_HA *info, MARIA_RECORD_POS pos, return 1; } #endif - if (!(rec_buff=(uchar*) my_safe_alloca(reclength, - MARIA_MAX_RECORD_ON_STACK))) + if (!(rec_buff=(uchar*) my_safe_alloca(reclength))) { my_errno= HA_ERR_OUT_OF_MEM; /* purecov: inspected */ return(1); @@ -300,7 +298,7 @@ my_bool _ma_update_blob_record(MARIA_HA *info, MARIA_RECORD_POS pos, error=update_dynamic_record(info,pos, rec_buff+ALIGN_SIZE(MARIA_MAX_DYN_BLOCK_HEADER), reclength); - my_safe_afree(rec_buff, reclength, MARIA_MAX_RECORD_ON_STACK); + my_safe_afree(rec_buff, reclength); return(error != 0); } @@ -1555,8 +1553,7 @@ my_bool _ma_cmp_dynamic_unique(MARIA_HA *info, MARIA_UNIQUEDEF *def, my_bool error; DBUG_ENTER("_ma_cmp_dynamic_unique"); - if (!(old_record= my_safe_alloca(info->s->base.reclength, - MARIA_MAX_RECORD_ON_STACK))) + if (!(old_record= my_safe_alloca(info->s->base.reclength))) DBUG_RETURN(1); /* Don't let the compare destroy blobs that may be in use */ @@ -1577,8 +1574,7 @@ my_bool _ma_cmp_dynamic_unique(MARIA_HA *info, MARIA_UNIQUEDEF *def, info->rec_buff= old_rec_buff; info->rec_buff_size= old_rec_buff_size; } - my_safe_afree(old_record, info->s->base.reclength, - MARIA_MAX_RECORD_ON_STACK); + my_safe_afree(old_record, info->s->base.reclength); DBUG_RETURN(error); } @@ -1593,9 +1589,8 @@ my_bool _ma_cmp_dynamic_record(register MARIA_HA *info, uchar *buffer; MARIA_BLOCK_INFO block_info; my_bool error= 1; - size_t buffer_length; + size_t UNINIT_VAR(buffer_length); DBUG_ENTER("_ma_cmp_dynamic_record"); - LINT_INIT(buffer_length); if (info->opt_flag & WRITE_CACHE_USED) { @@ -1614,8 +1609,7 @@ my_bool _ma_cmp_dynamic_record(register MARIA_HA *info, { buffer_length= (info->s->base.pack_reclength + _ma_calc_total_blob_length(info,record)); - if (!(buffer=(uchar*) my_safe_alloca(buffer_length, - MARIA_MAX_RECORD_ON_STACK))) + if (!(buffer=(uchar*) my_safe_alloca(buffer_length))) DBUG_RETURN(1); } reclength= _ma_rec_pack(info,buffer,record); @@ -1667,7 +1661,7 @@ my_bool _ma_cmp_dynamic_record(register MARIA_HA *info, error= 0; err: if (buffer != info->rec_buff) - my_safe_afree(buffer, buffer_length, MARIA_MAX_RECORD_ON_STACK); + my_safe_afree(buffer, buffer_length); DBUG_PRINT("exit", ("result: %d", error)); DBUG_RETURN(error); } diff --git a/storage/maria/ma_ft_nlq_search.c b/storage/maria/ma_ft_nlq_search.c index 613f13e64a9..5eb916112d7 100644 --- a/storage/maria/ma_ft_nlq_search.c +++ b/storage/maria/ma_ft_nlq_search.c @@ -83,8 +83,6 @@ static int walk_and_match(FT_WORD *word, uint32 count, ALL_IN_ONE *aio) #error #endif DBUG_ENTER("walk_and_match"); - LINT_INIT(subkeys.i); - LINT_INIT_STRUCT(subkeys); word->weight=LWS_FOR_QUERY; diff --git a/storage/maria/ma_key_recover.c b/storage/maria/ma_key_recover.c index ae9427981ea..e0a7bd35322 100644 --- a/storage/maria/ma_key_recover.c +++ b/storage/maria/ma_key_recover.c @@ -340,7 +340,7 @@ my_bool _ma_log_prefix(MARIA_PAGE *ma_page, uint changed_length, /* Store keypage_flag */ *log_pos++= KEY_OP_SET_PAGEFLAG; - *log_pos++= buff[KEYPAGE_TRANSFLAG_OFFSET]; + *log_pos++= _ma_get_keypage_flag(info->s, buff); if (move_length < 0) { @@ -424,7 +424,7 @@ my_bool _ma_log_suffix(MARIA_PAGE *ma_page, uint org_length, uint new_length) /* Store keypage_flag */ *log_pos++= KEY_OP_SET_PAGEFLAG; - *log_pos++= buff[KEYPAGE_TRANSFLAG_OFFSET]; + *log_pos++= _ma_get_keypage_flag(info->s, buff); if ((diff= (int) (new_length - org_length)) < 0) { @@ -526,7 +526,7 @@ my_bool _ma_log_add(MARIA_PAGE *ma_page, /* Store keypage_flag */ *log_pos++= KEY_OP_SET_PAGEFLAG; - *log_pos++= buff[KEYPAGE_TRANSFLAG_OFFSET]; + *log_pos++= _ma_get_keypage_flag(info->s, buff); /* Don't overwrite page boundary @@ -667,7 +667,7 @@ void _ma_log_key_changes(MARIA_PAGE *ma_page, LEX_CUSTRING *log_array, uint org_length; ha_checksum crc; - DBUG_ASSERT(ma_page->flag == (uint) ma_page->buff[KEYPAGE_TRANSFLAG_OFFSET]); + DBUG_ASSERT(ma_page->flag == (uint) _ma_get_keypage_flag(share, ma_page->buff)); /* We have to change length as the page may have been shortened */ org_length= _ma_get_page_used(share, ma_page->buff); @@ -1155,7 +1155,7 @@ uint _ma_apply_redo_index(MARIA_HA *info, } case KEY_OP_SET_PAGEFLAG: DBUG_PRINT("redo", ("key_op_set_pageflag")); - buff[KEYPAGE_TRANSFLAG_OFFSET]= *header++; + _ma_store_keypage_flag(share, buff, *header++); break; case KEY_OP_COMPACT_PAGE: { diff --git a/storage/maria/ma_loghandler.c b/storage/maria/ma_loghandler.c index 86a8970d7a7..8bcb84c2a20 100644 --- a/storage/maria/ma_loghandler.c +++ b/storage/maria/ma_loghandler.c @@ -419,15 +419,8 @@ static ulonglong flush_start= 0; #include <my_atomic.h> /* an array that maps id of a MARIA_SHARE to this MARIA_SHARE */ static MARIA_SHARE **id_to_share= NULL; -/* lock for id_to_share */ -static my_atomic_rwlock_t LOCK_id_to_share; -static my_bool translog_dummy_callback(uchar *page, - pgcache_page_no_t page_no, - uchar* data_ptr); -static my_bool translog_page_validator(uchar *page, - pgcache_page_no_t page_no, - uchar* data_ptr); +static my_bool translog_page_validator(int res, PAGECACHE_IO_HOOK_ARGS *args); static my_bool translog_get_next_chunk(TRANSLOG_SCANNER_DATA *scanner); static uint32 translog_first_file(TRANSLOG_ADDRESS horizon, int is_protected); @@ -1567,17 +1560,6 @@ static my_bool translog_close_log_file(TRANSLOG_FILE *file) /** - @brief Dummy function for write failure (the log to not use - pagecache writing) -*/ - -void translog_dummy_write_failure(uchar *data __attribute__((unused))) -{ - return; -} - - -/** @brief Initializes TRANSLOG_FILE structure @param file reference on the file to initialize @@ -1588,10 +1570,11 @@ void translog_dummy_write_failure(uchar *data __attribute__((unused))) static void translog_file_init(TRANSLOG_FILE *file, uint32 number, my_bool is_sync) { - pagecache_file_init(file->handler, &translog_page_validator, - &translog_dummy_callback, - &translog_dummy_write_failure, - maria_flush_log_for_page_none, file); + pagecache_file_set_null_hooks(&file->handler); + file->handler.post_read_hook= translog_page_validator; + file->handler.flush_log_callback= maria_flush_log_for_page_none; + file->handler.callback_data= (uchar*)file; + file->number= number; file->was_recovered= 0; file->is_sync= is_sync; @@ -2788,19 +2771,6 @@ static my_bool translog_recover_page_up_to_sector(uchar *page, uint16 offset) /** - @brief Dummy write callback. -*/ - -static my_bool -translog_dummy_callback(uchar *page __attribute__((unused)), - pgcache_page_no_t page_no __attribute__((unused)), - uchar* data_ptr __attribute__((unused))) -{ - return 0; -} - - -/** @brief Checks and removes sector protection. @param page reference on the page content. @@ -2876,14 +2846,14 @@ translog_check_sector_protection(uchar *page, TRANSLOG_FILE *file) @retval 1 Error */ -static my_bool translog_page_validator(uchar *page, - pgcache_page_no_t page_no, - uchar* data_ptr) +static my_bool translog_page_validator(int res, PAGECACHE_IO_HOOK_ARGS *args) { + uchar *page= args->page; + pgcache_page_no_t page_no= args->pageno; uint this_page_page_overhead; uint flags; uchar *page_pos; - TRANSLOG_FILE *data= (TRANSLOG_FILE *) data_ptr; + TRANSLOG_FILE *data= (TRANSLOG_FILE *) args->data; #ifndef DBUG_OFF pgcache_page_no_t offset= page_no * TRANSLOG_PAGE_SIZE; #endif @@ -2891,6 +2861,11 @@ static my_bool translog_page_validator(uchar *page, data->was_recovered= 0; + if (res) + { + DBUG_RETURN(1); + } + if ((pgcache_page_no_t) uint3korr(page) != page_no || (uint32) uint3korr(page + 3) != data->number) { @@ -3155,9 +3130,11 @@ restart: This IF should be true because we use in-memory data which supposed to be correct. */ - if (translog_page_validator(buffer, - LSN_OFFSET(addr) / TRANSLOG_PAGE_SIZE, - (uchar*) &file_copy)) + PAGECACHE_IO_HOOK_ARGS args; + args.page= buffer; + args.pageno= LSN_OFFSET(addr) / TRANSLOG_PAGE_SIZE; + args.data= (uchar*) &file_copy; + if (translog_page_validator(0, &args)) { DBUG_ASSERT(0); buffer= NULL; @@ -4042,7 +4019,6 @@ my_bool translog_init_with_table(const char *directory, Log records will refer to a MARIA_SHARE by a unique 2-byte id; set up structures for generating 2-byte ids: */ - my_atomic_rwlock_init(&LOCK_id_to_share); id_to_share= (MARIA_SHARE **) my_malloc(SHARE_ID_MAX * sizeof(MARIA_SHARE*), MYF(MY_WME | MY_ZEROFILL)); if (unlikely(!id_to_share)) @@ -4286,7 +4262,6 @@ void translog_destroy() if (log_descriptor.directory_fd >= 0) mysql_file_close(log_descriptor.directory_fd, MYF(MY_WME)); - my_atomic_rwlock_destroy(&LOCK_id_to_share); if (id_to_share != NULL) my_free(id_to_share + 1); DBUG_VOID_RETURN; @@ -8125,7 +8100,6 @@ int translog_assign_id_to_share(MARIA_HA *tbl_info, TRN *trn) id= 0; do { - my_atomic_rwlock_wrlock(&LOCK_id_to_share); for ( ; i <= SHARE_ID_MAX ; i++) /* the range is [1..SHARE_ID_MAX] */ { void *tmp= NULL; @@ -8136,7 +8110,6 @@ int translog_assign_id_to_share(MARIA_HA *tbl_info, TRN *trn) break; } } - my_atomic_rwlock_wrunlock(&LOCK_id_to_share); i= 1; /* scan the whole array */ } while (id == 0); DBUG_PRINT("info", ("id_to_share: 0x%lx -> %u", (ulong)share, id)); @@ -8199,9 +8172,7 @@ void translog_deassign_id_from_share(MARIA_SHARE *share) mutex: */ mysql_mutex_assert_owner(&share->intern_lock); - my_atomic_rwlock_rdlock(&LOCK_id_to_share); my_atomic_storeptr((void **)&id_to_share[share->id], 0); - my_atomic_rwlock_rdunlock(&LOCK_id_to_share); share->id= 0; /* useless but safety: */ share->lsn_of_file_id= LSN_IMPOSSIBLE; @@ -8661,8 +8632,8 @@ void translog_set_file_size(uint32 size) DBUG_ENTER("translog_set_file_size"); translog_lock(); DBUG_PRINT("enter", ("Size: %lu", (ulong) size)); - DBUG_ASSERT(size % TRANSLOG_PAGE_SIZE == 0 && - size >= TRANSLOG_MIN_FILE_SIZE); + DBUG_ASSERT(size % TRANSLOG_PAGE_SIZE == 0); + DBUG_ASSERT(size >= TRANSLOG_MIN_FILE_SIZE); log_descriptor.log_file_max_size= size; /* if current file longer then finish it*/ if (LSN_OFFSET(log_descriptor.horizon) >= log_descriptor.log_file_max_size) @@ -9112,8 +9083,8 @@ static void dump_datapage(uchar *buffer, File handler) } } tfile.number= file; + bzero(&tfile.handler, sizeof(tfile.handler)); tfile.handler.file= handler; - pagecache_file_init(tfile.handler, NULL, NULL, NULL, NULL, NULL); tfile.was_recovered= 0; tfile.is_sync= 1; if (translog_check_sector_protection(buffer, &tfile)) diff --git a/storage/maria/ma_open.c b/storage/maria/ma_open.c index 1d274d796be..42861e92ed4 100644 --- a/storage/maria/ma_open.c +++ b/storage/maria/ma_open.c @@ -20,6 +20,7 @@ #include "ma_rt_index.h" #include "ma_blockrec.h" #include <m_ctype.h> +#include "ma_crypt.h" #if defined(MSDOS) || defined(__WIN__) #ifdef __WIN__ @@ -279,7 +280,7 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags) char name_buff[FN_REFLEN], org_name[FN_REFLEN], index_name[FN_REFLEN], data_name[FN_REFLEN]; uchar *disk_cache, *disk_pos, *end_pos; - MARIA_HA info,*m_info,*old_info; + MARIA_HA info, *UNINIT_VAR(m_info), *old_info; MARIA_SHARE share_buff,*share; double *rec_per_key_part; ulong *nulls_per_key_part; @@ -289,7 +290,6 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags) File data_file= -1; DBUG_ENTER("maria_open"); - LINT_INIT(m_info); kfile= -1; errpos= 0; head_length=sizeof(share_buff.state.header); @@ -596,6 +596,12 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags) LSN_STORE_SIZE + TRANSID_SIZE : 0) + KEYPAGE_KEYID_SIZE + KEYPAGE_FLAG_SIZE + KEYPAGE_USED_SIZE); + + if (MY_TEST(share->base.extra_options & MA_EXTRA_OPTIONS_ENCRYPTED)) + { + share->keypage_header+= ma_crypt_get_index_page_header_space(share); + } + { HA_KEYSEG *pos=share->keyparts; uint32 ftkey_nr= 1; @@ -829,6 +835,12 @@ MARIA_HA *maria_open(const char *name, int mode, uint open_flags) disk_pos= _ma_column_nr_read(disk_pos, share->column_nr, share->base.fields); + if (MY_TEST(share->base.extra_options & MA_EXTRA_OPTIONS_ENCRYPTED)) + { + if (!(disk_pos= ma_crypt_read(share, disk_pos))) + goto err; + } + if ((share->data_file_type == BLOCK_RECORD || share->data_file_type == COMPRESSED_RECORD)) { @@ -1040,6 +1052,7 @@ err: (*share->once_end)(share); /* fall through */ case 4: + ma_crypt_free(share); my_free(share); /* fall through */ case 3: @@ -1819,24 +1832,31 @@ uchar *_ma_column_nr_read(uchar *ptr, uint16 *offsets, uint columns) void _ma_set_data_pagecache_callbacks(PAGECACHE_FILE *file, MARIA_SHARE *share) { + pagecache_file_set_null_hooks(file); file->callback_data= (uchar*) share; file->flush_log_callback= &maria_flush_log_for_page_none; /* Do nothing */ + file->post_write_hook= maria_page_write_failure; if (share->temporary) { - file->read_callback= &maria_page_crc_check_none; - file->write_callback= &maria_page_filler_set_none; + file->post_read_hook= &maria_page_crc_check_none; + file->pre_write_hook= &maria_page_filler_set_none; } else { - file->read_callback= &maria_page_crc_check_data; + file->post_read_hook= &maria_page_crc_check_data; if (share->options & HA_OPTION_PAGE_CHECKSUM) - file->write_callback= &maria_page_crc_set_normal; + file->pre_write_hook= &maria_page_crc_set_normal; else - file->write_callback= &maria_page_filler_set_normal; + file->pre_write_hook= &maria_page_filler_set_normal; if (share->now_transactional) file->flush_log_callback= maria_flush_log_for_page; } + + if (MY_TEST(share->base.extra_options & MA_EXTRA_OPTIONS_ENCRYPTED)) + { + ma_crypt_set_data_pagecache_callbacks(file, share); + } } @@ -1851,26 +1871,32 @@ void _ma_set_data_pagecache_callbacks(PAGECACHE_FILE *file, void _ma_set_index_pagecache_callbacks(PAGECACHE_FILE *file, MARIA_SHARE *share) { + pagecache_file_set_null_hooks(file); file->callback_data= (uchar*) share; file->flush_log_callback= &maria_flush_log_for_page_none; /* Do nothing */ - file->write_fail= maria_page_write_failure; + file->post_write_hook= maria_page_write_failure; if (share->temporary) { - file->read_callback= &maria_page_crc_check_none; - file->write_callback= &maria_page_filler_set_none; + file->post_read_hook= &maria_page_crc_check_none; + file->pre_write_hook= &maria_page_filler_set_none; } else { - file->read_callback= &maria_page_crc_check_index; + file->post_read_hook= &maria_page_crc_check_index; if (share->options & HA_OPTION_PAGE_CHECKSUM) - file->write_callback= &maria_page_crc_set_index; + file->pre_write_hook= &maria_page_crc_set_index; else - file->write_callback= &maria_page_filler_set_normal; + file->pre_write_hook= &maria_page_filler_set_normal; if (share->now_transactional) file->flush_log_callback= maria_flush_log_for_page; } + + if (MY_TEST(share->base.extra_options & MA_EXTRA_OPTIONS_ENCRYPTED)) + { + ma_crypt_set_index_pagecache_callbacks(file, share); + } } diff --git a/storage/maria/ma_page.c b/storage/maria/ma_page.c index ed62a80e4f7..3bf56b86064 100644 --- a/storage/maria/ma_page.c +++ b/storage/maria/ma_page.c @@ -219,13 +219,8 @@ my_bool _ma_write_keypage(MARIA_PAGE *page, enum pagecache_page_lock lock, /* Verify that keynr is correct */ DBUG_ASSERT(_ma_get_keynr(share, buff) == page->keyinfo->key_nr); -#if defined(EXTRA_DEBUG) && defined(HAVE_valgrind) && defined(NOT_ANYMORE) - { - /* This is here to catch uninitialized bytes */ - uint length= page->size; - ulong crc= my_checksum(0, buff, length); - int4store(buff + block_size - KEYPAGE_CHECKSUM_SIZE, crc); - } +#if defined(EXTRA_DEBUG) && defined(HAVE_valgrind) && defined(WHEN_DEBUGGING) + MEM_CHECK_DEFINED(buff, block_size); #endif page_cleanup(share, page); diff --git a/storage/maria/ma_pagecache.c b/storage/maria/ma_pagecache.c index d3eb687d064..4630a84334b 100644 --- a/storage/maria/ma_pagecache.c +++ b/storage/maria/ma_pagecache.c @@ -625,6 +625,8 @@ static my_bool pagecache_fwrite(PAGECACHE *pagecache, __attribute__((unused)), myf flags) { + int res; + PAGECACHE_IO_HOOK_ARGS args; DBUG_ENTER("pagecache_fwrite"); DBUG_ASSERT(type != PAGECACHE_READ_UNKNOWN_PAGE); @@ -648,24 +650,26 @@ static my_bool pagecache_fwrite(PAGECACHE *pagecache, } #endif + /* initialize hooks args */ + args.page= buffer; + args.pageno= pageno; + args.data= filedesc->callback_data; + /* Todo: Integrate this with write_callback so we have only one callback */ - if ((*filedesc->flush_log_callback)(buffer, pageno, filedesc->callback_data)) + if ((*filedesc->flush_log_callback)(&args)) DBUG_RETURN(1); - DBUG_PRINT("info", ("write_callback: 0x%lx data: 0x%lx", - (ulong) filedesc->write_callback, + DBUG_PRINT("info", ("pre_write_hook: 0x%lx data: 0x%lx", + (ulong) filedesc->pre_write_hook, (ulong) filedesc->callback_data)); - if ((*filedesc->write_callback)(buffer, pageno, filedesc->callback_data)) + if ((*filedesc->pre_write_hook)(&args)) { DBUG_PRINT("error", ("write callback problem")); DBUG_RETURN(1); } - if (my_pwrite(filedesc->file, buffer, pagecache->block_size, - ((my_off_t) pageno << pagecache->shift), flags)) - { - (*filedesc->write_fail)(filedesc->callback_data); - DBUG_RETURN(1); - } - DBUG_RETURN(0); + res= my_pwrite(filedesc->file, args.page, pagecache->block_size, + ((my_off_t) pageno << pagecache->shift), flags); + (*filedesc->post_write_hook)(res, &args); + DBUG_RETURN(res); } @@ -2695,6 +2699,7 @@ static void read_block(PAGECACHE *pagecache, if (primary) { size_t error; + PAGECACHE_IO_HOOK_ARGS args; /* This code is executed only by threads that submitted primary requests @@ -2707,10 +2712,18 @@ static void read_block(PAGECACHE *pagecache, They will register in block->wqueue[COND_FOR_REQUESTED]. */ pagecache_pthread_mutex_unlock(&pagecache->cache_lock); - error= pagecache_fread(pagecache, &block->hash_link->file, - block->buffer, - block->hash_link->pageno, - pagecache->readwrite_flags); + args.page= block->buffer; + args.pageno= block->hash_link->pageno; + args.data= block->hash_link->file.callback_data; + error= (*block->hash_link->file.pre_read_hook)(&args); + if (!error) + { + error= pagecache_fread(pagecache, &block->hash_link->file, + args.page, + block->hash_link->pageno, + pagecache->readwrite_flags); + } + error= (*block->hash_link->file.post_read_hook)(error != 0, &args); pagecache_pthread_mutex_lock(&pagecache->cache_lock); if (error) { @@ -2722,16 +2735,6 @@ static void read_block(PAGECACHE *pagecache, else { block->status|= PCBLOCK_READ; - if ((*block->hash_link->file.read_callback)(block->buffer, - block->hash_link->pageno, - block->hash_link-> - file.callback_data)) - { - DBUG_PRINT("error", ("read callback problem")); - block->status|= PCBLOCK_ERROR; - block->error= (int16) my_errno; - my_debug_put_break_here(); - } } DBUG_PRINT("read_block", ("primary request: new page in cache")); @@ -3373,8 +3376,7 @@ restart: /* Key cache is used */ PAGECACHE_BLOCK_LINK *block; uint status; - int page_st; - LINT_INIT(page_st); + int UNINIT_VAR(page_st); pagecache_pthread_mutex_lock(&pagecache->cache_lock); if (!pagecache->can_be_used) @@ -3510,9 +3512,21 @@ no_key_cache: /* Key cache is not used */ /* We can't use mutex here as the key cache may not be initialized */ pagecache->global_cache_r_requests++; pagecache->global_cache_read++; - if (pagecache_fread(pagecache, file, buff, pageno, - pagecache->readwrite_flags)) - error= 1; + + { + PAGECACHE_IO_HOOK_ARGS args; + args.page= buff; + args.pageno= pageno; + args.data= file->callback_data; + error= (* file->pre_read_hook)(&args); + if (!error) + { + error= pagecache_fread(pagecache, file, args.page, pageno, + pagecache->readwrite_flags) != 0; + } + error= (* file->post_read_hook)(error, &args); + } + DBUG_RETURN(error ? (uchar*) 0 : buff); } @@ -3603,17 +3617,16 @@ static my_bool pagecache_delete_internal(PAGECACHE *pagecache, } else { + PAGECACHE_IO_HOOK_ARGS args; PAGECACHE_FILE *filedesc= &block->hash_link->file; + args.page= block->buffer; + args.pageno= block->hash_link->pageno; + args.data= filedesc->callback_data; /* We are not going to write the page but have to call callbacks */ - DBUG_PRINT("info", ("flush_callback :0x%lx" - "write_callback: 0x%lx data: 0x%lx", + DBUG_PRINT("info", ("flush_callback :0x%lx data: 0x%lx", (ulong) filedesc->flush_log_callback, - (ulong) filedesc->write_callback, (ulong) filedesc->callback_data)); - if ((*filedesc->flush_log_callback) - (block->buffer, block->hash_link->pageno, filedesc->callback_data) || - (*filedesc->write_callback) - (block->buffer, block->hash_link->pageno, filedesc->callback_data)) + if ((*filedesc->flush_log_callback)(&args)) { DBUG_PRINT("error", ("flush or write callback problem")); error= 1; @@ -4083,23 +4096,6 @@ restart: /* Copy data from buff */ memcpy(block->buffer + offset, buff, size); block->status= PCBLOCK_READ; - /* - The read_callback can change the page content (removing page - protection) so it have to be called - */ - DBUG_PRINT("info", ("read_callback: 0x%lx data: 0x%lx", - (ulong) block->hash_link->file.read_callback, - (ulong) block->hash_link->file.callback_data)); - if ((*block->hash_link->file.read_callback)(block->buffer, - block->hash_link->pageno, - block->hash_link-> - file.callback_data)) - { - DBUG_PRINT("error", ("read callback problem")); - block->status|= PCBLOCK_ERROR; - block->error= (int16) my_errno; - my_debug_put_break_here(); - } KEYCACHE_DBUG_PRINT("key_cache_insert", ("Page injection")); /* Signal that all pending requests for this now can be processed. */ @@ -4187,14 +4183,21 @@ no_key_cache: if (offset != 0 || size != pagecache->block_size) { uchar *page_buffer= (uchar *) alloca(pagecache->block_size); + PAGECACHE_IO_HOOK_ARGS args; + args.page= page_buffer; + args.pageno= pageno; + args.data= file->callback_data; pagecache->global_cache_read++; - if ((error= (pagecache_fread(pagecache, file, - page_buffer, - pageno, - pagecache->readwrite_flags) != 0))) - goto end; - if ((file->read_callback)(page_buffer, pageno, file->callback_data)) + error= (*file->pre_read_hook)(&args); + if (!error) + { + error= pagecache_fread(pagecache, file, + page_buffer, + pageno, + pagecache->readwrite_flags) != 0; + } + if ((*file->post_read_hook)(error, &args)) { DBUG_PRINT("error", ("read callback problem")); error= 1; @@ -5257,3 +5260,37 @@ void pagecache_debug_log_close(void) #endif /* defined(PAGECACHE_DEBUG_LOG) */ #endif /* defined(PAGECACHE_DEBUG) */ + +/** + @brief null hooks +*/ + +static my_bool null_pre_hook(PAGECACHE_IO_HOOK_ARGS *args + __attribute__((unused))) +{ + return 0; +} + +static my_bool null_post_read_hook(int res, PAGECACHE_IO_HOOK_ARGS *args + __attribute__((unused))) +{ + return res != 0; +} + +static void null_post_write_hook(int res __attribute__((unused)), + PAGECACHE_IO_HOOK_ARGS *args + __attribute__((unused))) +{ + return; +} + +void +pagecache_file_set_null_hooks(PAGECACHE_FILE *file) +{ + file->pre_read_hook= null_pre_hook; + file->post_read_hook= null_post_read_hook; + file->pre_write_hook= null_pre_hook; + file->post_write_hook= null_post_write_hook; + file->flush_log_callback= null_pre_hook; + file->callback_data= NULL; +} diff --git a/storage/maria/ma_pagecache.h b/storage/maria/ma_pagecache.h index c6314ac6d8c..207ad69711f 100644 --- a/storage/maria/ma_pagecache.h +++ b/storage/maria/ma_pagecache.h @@ -76,20 +76,32 @@ enum pagecache_write_mode /* page number for maria */ typedef ulonglong pgcache_page_no_t; +/* args for read/write hooks */ +typedef struct st_pagecache_io_hook_args +{ + uchar * page; + pgcache_page_no_t pageno; + uchar * data; + + uchar *crypt_buf; /* when using encryption */ +} PAGECACHE_IO_HOOK_ARGS; + /* file descriptor for Maria */ typedef struct st_pagecache_file { File file; + /** Cannot be NULL */ - my_bool (*read_callback)(uchar *page, pgcache_page_no_t offset, - uchar *data); + my_bool (*pre_read_hook)(PAGECACHE_IO_HOOK_ARGS *args); + my_bool (*post_read_hook)(int error, PAGECACHE_IO_HOOK_ARGS *args); + /** Cannot be NULL */ - my_bool (*write_callback)(uchar *page, pgcache_page_no_t offset, - uchar *data); - void (*write_fail)(uchar *data); + my_bool (*pre_write_hook)(PAGECACHE_IO_HOOK_ARGS *args); + void (*post_write_hook)(int error, PAGECACHE_IO_HOOK_ARGS *args); + /** Cannot be NULL */ - my_bool (*flush_log_callback)(uchar *page, pgcache_page_no_t offset, - uchar *data); + my_bool (*flush_log_callback)(PAGECACHE_IO_HOOK_ARGS *args); + uchar *callback_data; } PAGECACHE_FILE; @@ -270,12 +282,8 @@ extern void pagecache_set_write_on_delete_by_link(PAGECACHE_BLOCK_LINK *block); /* PCFLUSH_ERROR and PCFLUSH_PINNED. */ #define PCFLUSH_PINNED_AND_ERROR (PCFLUSH_ERROR|PCFLUSH_PINNED) -#define pagecache_file_init(F,RC,WC,WF,GLC,D) \ - do{ \ - (F).read_callback= (RC); (F).write_callback= (WC); \ - (F).write_fail= (WF); \ - (F).flush_log_callback= (GLC); (F).callback_data= (uchar*)(D); \ - } while(0) +// initialize file with empty hooks +void pagecache_file_set_null_hooks(PAGECACHE_FILE*); #define flush_pagecache_blocks(A,B,C) \ flush_pagecache_blocks_with_filter(A,B,C,NULL,NULL) diff --git a/storage/maria/ma_pagecrc.c b/storage/maria/ma_pagecrc.c index d3522fa4e88..940feb8576b 100644 --- a/storage/maria/ma_pagecrc.c +++ b/storage/maria/ma_pagecrc.c @@ -128,11 +128,11 @@ static my_bool maria_page_crc_check(uchar *page, @retval 0 OK */ -my_bool maria_page_crc_set_normal(uchar *page, - pgcache_page_no_t page_no, - uchar *data_ptr) +my_bool maria_page_crc_set_normal(PAGECACHE_IO_HOOK_ARGS *args) { - MARIA_SHARE *share= (MARIA_SHARE *)data_ptr; + uchar *page= args->page; + pgcache_page_no_t page_no= args->pageno; + MARIA_SHARE *share= (MARIA_SHARE *)args->data; int data_length= share->block_size - CRC_SIZE; uint32 crc= maria_page_crc((uint32) page_no, page, data_length); DBUG_ENTER("maria_page_crc_set_normal"); @@ -154,11 +154,11 @@ my_bool maria_page_crc_set_normal(uchar *page, @retval 0 OK */ -my_bool maria_page_crc_set_index(uchar *page, - pgcache_page_no_t page_no, - uchar *data_ptr) +my_bool maria_page_crc_set_index(PAGECACHE_IO_HOOK_ARGS *args) { - MARIA_SHARE *share= (MARIA_SHARE *)data_ptr; + uchar *page= args->page; + pgcache_page_no_t page_no= args->pageno; + MARIA_SHARE *share= (MARIA_SHARE *)args->data; int data_length= _ma_get_page_used(share, page); uint32 crc= maria_page_crc((uint32) page_no, page, data_length); DBUG_ENTER("maria_page_crc_set_index"); @@ -185,11 +185,16 @@ my_bool maria_page_crc_set_index(uchar *page, @retval 1 Error */ -my_bool maria_page_crc_check_data(uchar *page, - pgcache_page_no_t page_no, - uchar *data_ptr) +my_bool maria_page_crc_check_data(int res, PAGECACHE_IO_HOOK_ARGS *args) { - MARIA_SHARE *share= (MARIA_SHARE *)data_ptr; + uchar *page= args->page; + pgcache_page_no_t page_no= args->pageno; + MARIA_SHARE *share= (MARIA_SHARE *)args->data; + if (res) + { + return 1; + } + return (maria_page_crc_check(page, (uint32) page_no, share, MARIA_NO_CRC_NORMAL_PAGE, share->block_size - CRC_SIZE)); @@ -207,11 +212,15 @@ my_bool maria_page_crc_check_data(uchar *page, @retval 1 Error */ -my_bool maria_page_crc_check_bitmap(uchar *page, - pgcache_page_no_t page_no, - uchar *data_ptr) +my_bool maria_page_crc_check_bitmap(int res, PAGECACHE_IO_HOOK_ARGS *args) { - MARIA_SHARE *share= (MARIA_SHARE *)data_ptr; + uchar *page= args->page; + pgcache_page_no_t page_no= args->pageno; + MARIA_SHARE *share= (MARIA_SHARE *)args->data; + if (res) + { + return 1; + } return (maria_page_crc_check(page, (uint32) page_no, share, MARIA_NO_CRC_BITMAP_PAGE, share->block_size - CRC_SIZE)); @@ -229,12 +238,16 @@ my_bool maria_page_crc_check_bitmap(uchar *page, @retval 1 Error */ -my_bool maria_page_crc_check_index(uchar *page, - pgcache_page_no_t page_no, - uchar *data_ptr) +my_bool maria_page_crc_check_index(int res, PAGECACHE_IO_HOOK_ARGS *args) { - MARIA_SHARE *share= (MARIA_SHARE *)data_ptr; + uchar *page= args->page; + pgcache_page_no_t page_no= args->pageno; + MARIA_SHARE *share= (MARIA_SHARE *)args->data; uint length= _ma_get_page_used(share, page); + if (res) + { + return 1; + } if (length > share->block_size - CRC_SIZE) { DBUG_PRINT("error", ("Wrong page length: %u", length)); @@ -253,12 +266,11 @@ my_bool maria_page_crc_check_index(uchar *page, @retval 1 Error */ -my_bool maria_page_crc_check_none(uchar *page __attribute__((unused)), - pgcache_page_no_t page_no - __attribute__((unused)), - uchar *data_ptr __attribute__((unused))) +my_bool maria_page_crc_check_none(int res, + PAGECACHE_IO_HOOK_ARGS *args + __attribute__((unused))) { - return 0; + return res != 0; } @@ -272,14 +284,16 @@ my_bool maria_page_crc_check_none(uchar *page __attribute__((unused)), @retval 0 OK */ -my_bool maria_page_filler_set_normal(uchar *page, - pgcache_page_no_t page_no - __attribute__((unused)), - uchar *data_ptr) +my_bool maria_page_filler_set_normal(PAGECACHE_IO_HOOK_ARGS *args) { + uchar *page= args->page; +#ifndef DBUG_OFF + pgcache_page_no_t page_no= args->pageno; +#endif + MARIA_SHARE *share= (MARIA_SHARE *)args->data; DBUG_ENTER("maria_page_filler_set_normal"); DBUG_ASSERT(page_no != 0); /* Catches some simple bugs */ - int4store_aligned(page + ((MARIA_SHARE *)data_ptr)->block_size - CRC_SIZE, + int4store_aligned(page + share->block_size - CRC_SIZE, MARIA_NO_CRC_NORMAL_PAGE); DBUG_RETURN(0); } @@ -295,13 +309,12 @@ my_bool maria_page_filler_set_normal(uchar *page, @retval 0 OK */ -my_bool maria_page_filler_set_bitmap(uchar *page, - pgcache_page_no_t page_no - __attribute__((unused)), - uchar *data_ptr) +my_bool maria_page_filler_set_bitmap(PAGECACHE_IO_HOOK_ARGS *args) { + uchar *page= args->page; + MARIA_SHARE *share= (MARIA_SHARE *)args->data; DBUG_ENTER("maria_page_filler_set_bitmap"); - int4store_aligned(page + ((MARIA_SHARE *)data_ptr)->block_size - CRC_SIZE, + int4store_aligned(page + share->block_size - CRC_SIZE, MARIA_NO_CRC_BITMAP_PAGE); DBUG_RETURN(0); } @@ -313,13 +326,13 @@ my_bool maria_page_filler_set_bitmap(uchar *page, @retval 0 OK */ -my_bool maria_page_filler_set_none(uchar *page __attribute__((unused)), - pgcache_page_no_t page_no - __attribute__((unused)), - uchar *data_ptr __attribute__((unused))) +my_bool maria_page_filler_set_none(PAGECACHE_IO_HOOK_ARGS *args + __attribute__((unused))) { #ifdef HAVE_valgrind - int4store_aligned(page + ((MARIA_SHARE *)data_ptr)->block_size - CRC_SIZE, + uchar *page= args->page; + MARIA_SHARE *share= (MARIA_SHARE *)args->data; + int4store_aligned(page + share->block_size - CRC_SIZE, 0); #endif return 0; @@ -332,9 +345,10 @@ my_bool maria_page_filler_set_none(uchar *page __attribute__((unused)), @param data_ptr Write callback data pointer (pointer to MARIA_SHARE) */ -void maria_page_write_failure(uchar* data_ptr) +void maria_page_write_failure(int error, PAGECACHE_IO_HOOK_ARGS *args) { - maria_mark_crashed_share((MARIA_SHARE *)data_ptr); + if (error) + maria_mark_crashed_share((MARIA_SHARE *)args->data); } @@ -349,13 +363,11 @@ void maria_page_write_failure(uchar* data_ptr) @retval 1 error */ -my_bool maria_flush_log_for_page(uchar *page, - pgcache_page_no_t page_no - __attribute__((unused)), - uchar *data_ptr __attribute__((unused))) +my_bool maria_flush_log_for_page(PAGECACHE_IO_HOOK_ARGS *args) { LSN lsn; - MARIA_SHARE *share= (MARIA_SHARE*) data_ptr; + uchar *page= args->page; + MARIA_SHARE *share= (MARIA_SHARE *)args->data; DBUG_ENTER("maria_flush_log_for_page"); /* share is 0 here only in unittest */ DBUG_ASSERT(!share || share->page_type == PAGECACHE_LSN_PAGE); @@ -372,10 +384,14 @@ my_bool maria_flush_log_for_page(uchar *page, } -my_bool maria_flush_log_for_page_none(uchar *page __attribute__((unused)), - pgcache_page_no_t page_no - __attribute__((unused)), - uchar *data_ptr __attribute__((unused))) +my_bool maria_flush_log_for_page_none(PAGECACHE_IO_HOOK_ARGS *args + __attribute__((unused))) +{ + return 0; +} + +my_bool maria_page_null_pre_read_hook(PAGECACHE_IO_HOOK_ARGS *args + __attribute__((unused))) { return 0; } diff --git a/storage/maria/ma_range.c b/storage/maria/ma_range.c index 7747df6415a..7216480abd9 100644 --- a/storage/maria/ma_range.c +++ b/storage/maria/ma_range.c @@ -209,14 +209,13 @@ static double _ma_search_pos(MARIA_HA *info, MARIA_KEY *key, uint32 nextflag, my_off_t pos) { int flag; - uint keynr, max_keynr; + uint keynr, UNINIT_VAR(max_keynr); my_bool after_key; uchar *keypos; double offset; MARIA_KEYDEF *keyinfo= key->keyinfo; MARIA_PAGE page; DBUG_ENTER("_ma_search_pos"); - LINT_INIT(max_keynr); if (pos == HA_OFFSET_ERROR) DBUG_RETURN(0.0); diff --git a/storage/maria/ma_rt_index.c b/storage/maria/ma_rt_index.c index 0d187c81692..320418c15ed 100644 --- a/storage/maria/ma_rt_index.c +++ b/storage/maria/ma_rt_index.c @@ -523,19 +523,15 @@ static const uchar *maria_rtree_pick_key(const MARIA_KEY *key, const MARIA_PAGE *page) { double increase; - double best_incr; + double UNINIT_VAR(best_incr); double perimeter; - double best_perimeter; + double UNINIT_VAR(best_perimeter); uchar *best_key= NULL; const MARIA_HA *info= page->info; uchar *k= rt_PAGE_FIRST_KEY(info->s, page->buf, page->node); uchar *last= rt_PAGE_END(info, page); - LINT_INIT(best_perimeter); - LINT_INIT(best_key); - LINT_INIT(best_incr); - for (; k < last; k= rt_PAGE_NEXT_KEY(k, key->data_length, nod_flag)) { if ((increase= maria_rtree_perimeter_increase(keyinfo->seg, k, key, @@ -563,13 +559,11 @@ static const uchar *maria_rtree_pick_key(const MARIA_KEY *key, double increase; double best_incr= DBL_MAX; double area; - double best_area; + double UNINIT_VAR(best_area); const uchar *best_key= NULL; const uchar *k= rt_PAGE_FIRST_KEY(share, page->buff, page->node); const uchar *last= rt_PAGE_END(page); - LINT_INIT(best_area); - for (; k < last; k= rt_PAGE_NEXT_KEY(share, k, key->data_length, page->node)) { diff --git a/storage/maria/ma_search.c b/storage/maria/ma_search.c index d38bc7af26c..14ff084332e 100644 --- a/storage/maria/ma_search.c +++ b/storage/maria/ma_search.c @@ -269,7 +269,7 @@ int _ma_bin_search(const MARIA_KEY *key, const MARIA_PAGE *ma_page, uint32 comp_flag, uchar **ret_pos, uchar *buff __attribute__((unused)), my_bool *last_key) { - int flag; + int UNINIT_VAR(flag); uint page_flag; uint start, mid, end, save_end, totlength, nod_flag; uint not_used[2]; @@ -278,8 +278,6 @@ int _ma_bin_search(const MARIA_KEY *key, const MARIA_PAGE *ma_page, uchar *page; DBUG_ENTER("_ma_bin_search"); - LINT_INIT(flag); - page_flag= ma_page->flag; if (page_flag & KEYPAGE_FLAG_HAS_TRANSID) { diff --git a/storage/maria/ma_sort.c b/storage/maria/ma_sort.c index 8593264b212..ac166cf4084 100644 --- a/storage/maria/ma_sort.c +++ b/storage/maria/ma_sort.c @@ -131,7 +131,7 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages, uint sort_length, maxbuffer; size_t memavl, old_memavl; DYNAMIC_ARRAY buffpek; - ha_rows records, keys; + ha_rows records, UNINIT_VAR(keys); uchar **sort_keys; IO_CACHE tempfile, tempfile_for_exceptions; DBUG_ENTER("_ma_create_index_by_sort"); @@ -150,7 +150,6 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages, memavl=MY_MAX(sortbuff_size,MIN_SORT_MEMORY); records= info->sort_info->max_records; sort_length= info->key_length; - LINT_INIT(keys); while (memavl >= MIN_SORT_MEMORY) { @@ -374,7 +373,7 @@ static my_bool _ma_thr_find_all_keys_exec(MARIA_SORT_PARAM* sort_param) uint sort_length; uint maxbuffer; uchar **sort_keys= NULL; - DBUG_ENTER("_ma_thr_find_all_keys"); + DBUG_ENTER("_ma_thr_find_all_keys_exec"); DBUG_PRINT("enter", ("master: %d", sort_param->master)); if (sort_param->sort_info->got_error) @@ -924,9 +923,8 @@ static my_off_t read_to_buffer(IO_CACHE *fromfile, BUFFPEK *buffpek, if ((count= (ha_keys) MY_MIN((ha_rows) buffpek->max_keys,buffpek->count))) { - if (mysql_file_pread(fromfile->file, (uchar*) buffpek->base, - (length= sort_length * count), - buffpek->file_pos, MYF_RW)) + if (my_b_pread(fromfile, (uchar*) buffpek->base, + (length= sort_length * count), buffpek->file_pos)) return(HA_OFFSET_ERROR); /* purecov: inspected */ buffpek->key=buffpek->base; buffpek->file_pos+= length; /* New filepos */ @@ -951,12 +949,12 @@ static my_off_t read_to_buffer_varlen(IO_CACHE *fromfile, BUFFPEK *buffpek, for (idx=1;idx<=count;idx++) { uint16 length_of_key; - if (mysql_file_pread(fromfile->file,(uchar*)&length_of_key,sizeof(length_of_key), - buffpek->file_pos,MYF_RW)) + if (my_b_pread(fromfile, (uchar*)&length_of_key, + sizeof(length_of_key), buffpek->file_pos)) return(HA_OFFSET_ERROR); buffpek->file_pos+=sizeof(length_of_key); - if (mysql_file_pread(fromfile->file, buffp, length_of_key, - buffpek->file_pos,MYF_RW)) + if (my_b_pread(fromfile, (uchar*) buffp, + length_of_key, buffpek->file_pos)) return((uint) -1); buffpek->file_pos+=length_of_key; buffp = buffp + sort_length; @@ -1021,7 +1019,6 @@ merge_buffers(MARIA_SORT_PARAM *info, ha_keys keys, IO_CACHE *from_file, count= 0; maxcount= keys/((uint) (Tb-Fb) +1); DBUG_ASSERT(maxcount > 0); - LINT_INIT(to_start_filepos); if (to_file) to_start_filepos=my_b_tell(to_file); strpos= (uchar*) sort_keys; diff --git a/storage/maria/ma_static.c b/storage/maria/ma_static.c index 35ad7d5a96a..2877f05c8dc 100644 --- a/storage/maria/ma_static.c +++ b/storage/maria/ma_static.c @@ -40,6 +40,7 @@ my_bool maria_in_ha_maria= FALSE; /* If used from ha_maria or not */ my_bool maria_recovery_changed_data= 0, maria_recovery_verbose= 0; my_bool maria_assert_if_crashed_table= 0; my_bool maria_checkpoint_disabled= 0; +my_bool maria_encrypt_tables= 0; mysql_mutex_t THR_LOCK_maria; #ifdef DONT_USE_RW_LOCKS diff --git a/storage/maria/ma_test1.c b/storage/maria/ma_test1.c index 901a7ef06e3..a32ed77e437 100644 --- a/storage/maria/ma_test1.c +++ b/storage/maria/ma_test1.c @@ -309,7 +309,7 @@ static int run_test(const char *filename) } /* Read through all rows and update them */ - assert(maria_scan_init(file) == 0); + maria_scan_init(file); found=0; while ((error= maria_scan(file,read_record)) == 0) diff --git a/storage/maria/ma_test2.c b/storage/maria/ma_test2.c index 709a190c1a7..e5e461261b5 100644 --- a/storage/maria/ma_test2.c +++ b/storage/maria/ma_test2.c @@ -665,7 +665,7 @@ int main(int argc, char *argv[]) if (!silent) puts("- Test if: Read rrnd - same"); DBUG_PRINT("progpos",("Read rrnd - same")); - assert(maria_scan_init(file) == 0); + maria_scan_init(file); for (i=0 ; i < write_count ; i++) { int tmp; @@ -818,7 +818,7 @@ int main(int argc, char *argv[]) } } ant=0; - assert(maria_scan_init(file) == 0); + maria_scan_init(file); while ((error= maria_scan(file,record)) != HA_ERR_END_OF_FILE && ant < write_count + 10) ant+= error ? 0 : 1; diff --git a/storage/maria/ma_update.c b/storage/maria/ma_update.c index e0e804ca655..4385f2d306c 100644 --- a/storage/maria/ma_update.c +++ b/storage/maria/ma_update.c @@ -26,14 +26,12 @@ int maria_update(register MARIA_HA *info, const uchar *oldrec, uchar *newrec) int flag,key_changed,save_errno; reg3 my_off_t pos; uint i; - uchar old_key_buff[MARIA_MAX_KEY_BUFF],*new_key_buff; + uchar old_key_buff[MARIA_MAX_KEY_BUFF], *UNINIT_VAR(new_key_buff); my_bool auto_key_changed= 0; - ulonglong changed; + ulonglong UNINIT_VAR(changed); MARIA_SHARE *share= info->s; MARIA_KEYDEF *keyinfo; DBUG_ENTER("maria_update"); - LINT_INIT(new_key_buff); - LINT_INIT(changed); DBUG_EXECUTE_IF("maria_pretend_crashed_table_on_usage", maria_print_error(info->s, HA_ERR_CRASHED); diff --git a/storage/maria/ma_write.c b/storage/maria/ma_write.c index 629b774706e..f57c462e7c0 100644 --- a/storage/maria/ma_write.c +++ b/storage/maria/ma_write.c @@ -392,8 +392,12 @@ err: else fatal_error= 1; - if ((*share->write_record_abort)(info)) - fatal_error= 1; + if (filepos != HA_OFFSET_ERROR) + { + if ((*share->write_record_abort)(info)) + fatal_error= 1; + } + if (fatal_error) { maria_print_error(info->s, HA_ERR_CRASHED); @@ -970,9 +974,10 @@ int _ma_split_page(MARIA_HA *info, MARIA_KEY *key, MARIA_PAGE *split_page, int move_length, uchar *key_buff, my_bool insert_last_key) { + uint keynr; uint length,a_length,key_ref_length,t_length,nod_flag,key_length; uint page_length, split_length, page_flag; - uchar *key_pos,*pos, *after_key; + uchar *key_pos, *pos, *UNINIT_VAR(after_key); MARIA_KEY_PARAM s_temp; MARIA_PINNED_PAGE tmp_page_link, *page_link= &tmp_page_link; MARIA_SHARE *share= info->s; @@ -982,7 +987,6 @@ int _ma_split_page(MARIA_HA *info, MARIA_KEY *key, MARIA_PAGE *split_page, int res; DBUG_ENTER("_ma_split_page"); - LINT_INIT(after_key); DBUG_DUMP("buff", split_page->buff, split_page->size); info->page_changed=1; /* Info->buff is used */ @@ -1045,10 +1049,8 @@ int _ma_split_page(MARIA_HA *info, MARIA_KEY *key, MARIA_PAGE *split_page, page_store_info(share, &new_page); /* Copy key number */ - new_page.buff[share->keypage_header - KEYPAGE_USED_SIZE - - KEYPAGE_KEYID_SIZE - KEYPAGE_FLAG_SIZE]= - split_page->buff[share->keypage_header - KEYPAGE_USED_SIZE - - KEYPAGE_KEYID_SIZE - KEYPAGE_FLAG_SIZE]; + keynr= _ma_get_keynr(share, split_page->buff); + _ma_store_keynr(share, new_page.buff, keynr); res= 2; /* Middle key up */ if (share->now_transactional && _ma_log_new(&new_page, 0)) @@ -1317,7 +1319,7 @@ static int _ma_balance_page(MARIA_HA *info, MARIA_KEYDEF *keyinfo, pos= right_page->buff + share->keypage_header + length; memcpy(father_key_pos, pos, (size_t) k_length); bmove(right_page->buff + share->keypage_header, - pos + k_length, new_right_length); + pos + k_length, new_right_length - share->keypage_header); if (share->now_transactional) { @@ -1494,8 +1496,7 @@ static int _ma_balance_page(MARIA_HA *info, MARIA_KEYDEF *keyinfo, page_store_info(share, &extra_page); /* Copy key number */ - extra_buff[share->keypage_header - KEYPAGE_USED_SIZE - KEYPAGE_KEYID_SIZE - - KEYPAGE_FLAG_SIZE]= keyinfo->key_nr; + _ma_store_keynr(share, extra_buff, keyinfo->key_nr); /* move first largest keys to new page */ pos= right_page->buff + right_length-extra_length; @@ -2051,7 +2052,7 @@ static my_bool _ma_log_split(MARIA_PAGE *ma_page, /* Store keypage_flag */ *log_pos++= KEY_OP_SET_PAGEFLAG; - *log_pos++= ma_page->buff[KEYPAGE_TRANSFLAG_OFFSET]; + *log_pos++= _ma_get_keypage_flag(info->s, ma_page->buff); if (new_length <= offset || !key_pos) { @@ -2218,7 +2219,7 @@ static my_bool _ma_log_del_prefix(MARIA_PAGE *ma_page, /* Store keypage_flag */ *log_pos++= KEY_OP_SET_PAGEFLAG; - *log_pos++= ma_page->buff[KEYPAGE_TRANSFLAG_OFFSET]; + *log_pos++= _ma_get_keypage_flag(info->s, ma_page->buff); if (offset < diff_length + info->s->keypage_header) { @@ -2342,7 +2343,7 @@ static my_bool _ma_log_key_middle(MARIA_PAGE *ma_page, /* Store keypage_flag */ *log_pos++= KEY_OP_SET_PAGEFLAG; - *log_pos++= ma_page->buff[KEYPAGE_TRANSFLAG_OFFSET]; + *log_pos++= _ma_get_keypage_flag(info->s, ma_page->buff); log_pos[0]= KEY_OP_DEL_SUFFIX; int2store(log_pos+1, data_deleted_last); diff --git a/storage/maria/maria_def.h b/storage/maria/maria_def.h index a1fff2e0e43..a4fac8c088a 100644 --- a/storage/maria/maria_def.h +++ b/storage/maria/maria_def.h @@ -236,6 +236,10 @@ typedef struct st_maria_state_info MARIA_MAX_POINTER_LENGTH) #define MARIA_DELETE_KEY_NR 255 /* keynr for deleted blocks */ + /* extra options */ +#define MA_EXTRA_OPTIONS_ENCRYPTED (1 << 0) +#define MA_EXTRA_OPTIONS_INSERT_ORDER (1 << 1) + /* Basic information of the Maria table. This is stored on disk and not changed (unless we do DLL changes). @@ -318,6 +322,7 @@ typedef struct st_maria_pack typedef struct st_maria_file_bitmap { + struct st_maria_share *share; uchar *map; pgcache_page_no_t page; /* Page number for current bitmap */ pgcache_page_no_t last_bitmap_page; /* Last possible bitmap page */ @@ -346,6 +351,8 @@ typedef struct st_maria_file_bitmap #define MARIA_CHECKPOINT_SHOULD_FREE_ME 2 #define MARIA_CHECKPOINT_SEEN_IN_LOOP 4 +typedef struct st_maria_crypt_data MARIA_CRYPT_DATA; + typedef struct st_maria_share { /* Shared between opens */ MARIA_STATE_INFO state; @@ -506,6 +513,18 @@ typedef struct st_maria_share MARIA_FILE_BITMAP bitmap; mysql_rwlock_t mmap_lock; LSN lsn_of_file_id; /**< LSN of its last LOGREC_FILE_ID */ + + /** + Crypt data + */ + uint crypt_page_header_space; + MARIA_CRYPT_DATA *crypt_data; + + /** + Keep of track of last insert page, used to implement insert order + */ + uint last_insert_page; + pgcache_page_no_t last_insert_bitmap; } MARIA_SHARE; @@ -724,14 +743,13 @@ struct st_maria_handler #define KEYPAGE_USED_SIZE 2 #define KEYPAGE_KEYID_SIZE 1 #define KEYPAGE_FLAG_SIZE 1 +#define KEYPAGE_KEY_VERSION_SIZE 4 /* encryption */ #define KEYPAGE_CHECKSUM_SIZE 4 #define MAX_KEYPAGE_HEADER_SIZE (LSN_STORE_SIZE + KEYPAGE_USED_SIZE + \ KEYPAGE_KEYID_SIZE + KEYPAGE_FLAG_SIZE + \ - TRANSID_SIZE) + TRANSID_SIZE + KEYPAGE_KEY_VERSION_SIZE) #define KEYPAGE_FLAG_ISNOD 1 #define KEYPAGE_FLAG_HAS_TRANSID 2 -/* Position to KEYPAGE_FLAG for transactional tables */ -#define KEYPAGE_TRANSFLAG_OFFSET LSN_STORE_SIZE + TRANSID_SIZE + KEYPAGE_KEYID_SIZE #define _ma_get_page_used(share,x) \ ((uint) mi_uint2korr((x) + (share)->keypage_header - KEYPAGE_USED_SIZE)) @@ -752,6 +770,18 @@ struct st_maria_handler (page)->flag|= KEYPAGE_FLAG_HAS_TRANSID; \ (page)->buff[(share)->keypage_header - KEYPAGE_USED_SIZE - KEYPAGE_FLAG_SIZE]= (page)->flag; +#define KEYPAGE_KEY_VERSION(share, x) ((x) + \ + (share)->keypage_header - \ + (KEYPAGE_USED_SIZE + \ + KEYPAGE_FLAG_SIZE + \ + KEYPAGE_KEYID_SIZE + \ + KEYPAGE_KEY_VERSION_SIZE)) + +#define _ma_get_key_version(share,x) \ + ((uint) uint4korr(KEYPAGE_KEY_VERSION((share), (x)))) + +#define _ma_store_key_version(share,x,kv) \ + int4store(KEYPAGE_KEY_VERSION((share), (x)), (kv)) /* TODO: write int4store_aligned as *((uint32 *) (T))= (uint32) (A) for @@ -918,6 +948,8 @@ extern PSI_mutex_key key_SHARE_BITMAP_lock, key_SORT_INFO_mutex, key_SERVICE_THREAD_CONTROL_lock, key_PAGECACHE_cache_lock; +extern PSI_mutex_key key_CRYPT_DATA_lock; + extern PSI_cond_key key_SHARE_key_del_cond, key_SERVICE_THREAD_CONTROL_cond, key_SORT_INFO_cond, key_SHARE_BITMAP_cond, key_COND_soft_sync, key_TRANSLOG_BUFFER_waiting_filling_buffer, @@ -1321,7 +1353,8 @@ void _ma_remap_file(MARIA_HA *info, my_off_t size); MARIA_RECORD_POS _ma_write_init_default(MARIA_HA *info, const uchar *record); my_bool _ma_write_abort_default(MARIA_HA *info); -int maria_delete_table_files(const char *name, myf sync_dir); +int maria_delete_table_files(const char *name, my_bool temporary, + myf sync_dir); /* This cannot be in my_base.h as it clashes with HA_SPATIAL. @@ -1380,40 +1413,19 @@ void _ma_unpin_all_pages(MARIA_HA *info, LSN undo_lsn); #define MARIA_NO_CRC_NORMAL_PAGE 0xffffffff #define MARIA_NO_CRC_BITMAP_PAGE 0xfffffffe -extern my_bool maria_page_crc_set_index(uchar *page, - pgcache_page_no_t page_no, - uchar *data_ptr); -extern my_bool maria_page_crc_set_normal(uchar *page, - pgcache_page_no_t page_no, - uchar *data_ptr); -extern my_bool maria_page_crc_check_bitmap(uchar *page, - pgcache_page_no_t page_no, - uchar *data_ptr); -extern my_bool maria_page_crc_check_data(uchar *page, - pgcache_page_no_t page_no, - uchar *data_ptr); -extern my_bool maria_page_crc_check_index(uchar *page, - pgcache_page_no_t page_no, - uchar *data_ptr); -extern my_bool maria_page_crc_check_none(uchar *page, - pgcache_page_no_t page_no, - uchar *data_ptr); -extern my_bool maria_page_filler_set_bitmap(uchar *page, - pgcache_page_no_t page_no, - uchar *data_ptr); -extern my_bool maria_page_filler_set_normal(uchar *page, - pgcache_page_no_t page_no, - uchar *data_ptr); -extern my_bool maria_page_filler_set_none(uchar *page, - pgcache_page_no_t page_no, - uchar *data_ptr); -extern void maria_page_write_failure(uchar* data_ptr); -extern my_bool maria_flush_log_for_page(uchar *page, - pgcache_page_no_t page_no, - uchar *data_ptr); -extern my_bool maria_flush_log_for_page_none(uchar *page, - pgcache_page_no_t page_no, - uchar *data_ptr); +extern my_bool maria_page_crc_set_index(PAGECACHE_IO_HOOK_ARGS *args); +extern my_bool maria_page_crc_set_normal(PAGECACHE_IO_HOOK_ARGS *args); +extern my_bool maria_page_crc_check_bitmap(int, PAGECACHE_IO_HOOK_ARGS *args); +extern my_bool maria_page_crc_check_data(int, PAGECACHE_IO_HOOK_ARGS *args); +extern my_bool maria_page_crc_check_index(int, PAGECACHE_IO_HOOK_ARGS *args); +extern my_bool maria_page_crc_check_none(int, PAGECACHE_IO_HOOK_ARGS *args); +extern my_bool maria_page_filler_set_bitmap(PAGECACHE_IO_HOOK_ARGS *args); +extern my_bool maria_page_filler_set_normal(PAGECACHE_IO_HOOK_ARGS *args); +extern my_bool maria_page_filler_set_none(PAGECACHE_IO_HOOK_ARGS *args); +extern void maria_page_write_failure(int error, PAGECACHE_IO_HOOK_ARGS *args); +extern my_bool maria_flush_log_for_page(PAGECACHE_IO_HOOK_ARGS *args); +extern my_bool maria_flush_log_for_page_none(PAGECACHE_IO_HOOK_ARGS *args); + extern PAGECACHE *maria_log_pagecache; extern void ma_set_index_cond_func(MARIA_HA *info, index_cond_func_t func, void *func_arg); diff --git a/storage/maria/maria_pack.c b/storage/maria/maria_pack.c index 7eca9e14e93..280c5ff8f0a 100644 --- a/storage/maria/maria_pack.c +++ b/storage/maria/maria_pack.c @@ -861,7 +861,7 @@ static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts) reclength= mrg->file[0]->s->base.reclength; null_bytes= mrg->file[0]->s->base.null_bytes; - record=(uchar*) my_safe_alloca(reclength, MARIA_MAX_RECORD_ON_STACK); + record=(uchar*) my_safe_alloca(reclength); end_count=huff_counts+mrg->file[0]->s->base.fields; record_count=0; glob_crc=0; max_blob_length=0; @@ -1145,7 +1145,7 @@ static int get_statistic(PACK_MRG_INFO *mrg,HUFF_COUNTS *huff_counts) mrg->records=record_count; mrg->max_blob_length=max_blob_length; - my_safe_afree(record, reclength, MARIA_MAX_RECORD_ON_STACK); + my_safe_afree(record, reclength); DBUG_RETURN(error != HA_ERR_END_OF_FILE); } @@ -2415,8 +2415,7 @@ static int compress_maria_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) DBUG_ENTER("compress_maria_file"); /* Allocate a buffer for the records (excluding blobs). */ - if (!(record=(uchar*) my_safe_alloca(isam_file->s->base.reclength, - MARIA_MAX_RECORD_ON_STACK))) + if (!(record=(uchar*) my_safe_alloca(isam_file->s->base.reclength))) return -1; end_count=huff_counts+isam_file->s->base.fields; @@ -2779,8 +2778,7 @@ static int compress_maria_file(PACK_MRG_INFO *mrg, HUFF_COUNTS *huff_counts) if (verbose >= 2) printf("wrote %s records.\n", llstr((longlong) record_count, llbuf)); - my_safe_afree(record, isam_file->s->base.reclength, - MARIA_MAX_RECORD_ON_STACK); + my_safe_afree(record, isam_file->s->base.reclength); mrg->ref_length=max_pack_length; mrg->min_pack_length=max_record_length ? min_record_length : 0; mrg->max_pack_length=max_record_length; diff --git a/storage/maria/maria_read_log.c b/storage/maria/maria_read_log.c index 8fa6533bc46..1087889b5b7 100644 --- a/storage/maria/maria_read_log.c +++ b/storage/maria/maria_read_log.c @@ -46,6 +46,7 @@ int main(int argc, char **argv) MY_INIT(argv[0]); maria_data_root= (char *)"."; + sf_leaking_memory=1; /* don't report memory leaks on early exits */ load_defaults("my", load_default_groups, &argc, &argv); default_argv= argv; get_options(&argc, &argv); @@ -151,6 +152,7 @@ end: free_tmpdir(&maria_chk_tmpdir); free_defaults(default_argv); my_end(0); + sf_leaking_memory=0; exit(0); return 0; /* No compiler warning */ diff --git a/storage/maria/trnman.c b/storage/maria/trnman.c index daccf3550c2..3ada502988a 100644 --- a/storage/maria/trnman.c +++ b/storage/maria/trnman.c @@ -60,7 +60,6 @@ static LF_HASH trid_to_trn; static TRN **short_trid_to_active_trn; /* locks for short_trid_to_active_trn and pool */ -static my_atomic_rwlock_t LOCK_short_trid_to_trn, LOCK_pool; static my_bool default_trnman_end_trans_hook(TRN *, my_bool, my_bool); static void trnman_free_trn(TRN *); @@ -191,8 +190,6 @@ int trnman_init(TrID initial_trid) 0, 0, trn_get_hash_key, 0); DBUG_PRINT("info", ("mysql_mutex_init LOCK_trn_list")); mysql_mutex_init(key_LOCK_trn_list, &LOCK_trn_list, MY_MUTEX_INIT_FAST); - my_atomic_rwlock_init(&LOCK_short_trid_to_trn); - my_atomic_rwlock_init(&LOCK_pool); DBUG_RETURN(0); } @@ -226,8 +223,6 @@ void trnman_destroy() lf_hash_destroy(&trid_to_trn); DBUG_PRINT("info", ("mysql_mutex_destroy LOCK_trn_list")); mysql_mutex_destroy(&LOCK_trn_list); - my_atomic_rwlock_destroy(&LOCK_short_trid_to_trn); - my_atomic_rwlock_destroy(&LOCK_pool); my_free(short_trid_to_active_trn+1); short_trid_to_active_trn= NULL; @@ -257,7 +252,6 @@ static uint get_short_trid(TRN *trn) for ( ; !res ; i= 1) { - my_atomic_rwlock_wrlock(&LOCK_short_trid_to_trn); for ( ; i <= SHORT_TRID_MAX; i++) /* the range is [1..SHORT_TRID_MAX] */ { void *tmp= NULL; @@ -268,7 +262,6 @@ static uint get_short_trid(TRN *trn) break; } } - my_atomic_rwlock_wrunlock(&LOCK_short_trid_to_trn); } return res; } @@ -306,11 +299,9 @@ TRN *trnman_new_trn(WT_THD *wt) Popping an unused TRN from the pool (ABA isn't possible, we're behind a mutex */ - my_atomic_rwlock_wrlock(&LOCK_pool); while (tmp.trn && !my_atomic_casptr((void **)(char*) &pool, &tmp.v, (void *)tmp.trn->next)) /* no-op */; - my_atomic_rwlock_wrunlock(&LOCK_pool); /* Nothing in the pool ? Allocate a new one */ if (!(trn= tmp.trn)) @@ -493,9 +484,7 @@ my_bool trnman_end_trn(TRN *trn, my_bool commit) note that we don't own trn anymore, it may be in a shared list now. Thus, we cannot dereference it, and must use cached_short_id below. */ - my_atomic_rwlock_rdlock(&LOCK_short_trid_to_trn); my_atomic_storeptr((void **)&short_trid_to_active_trn[cached_short_id], 0); - my_atomic_rwlock_rdunlock(&LOCK_short_trid_to_trn); /* we, under the mutex, removed going-in-free_me transactions from the @@ -545,7 +534,6 @@ static void trnman_free_trn(TRN *trn) tmp.trn= pool; - my_atomic_rwlock_wrlock(&LOCK_pool); do { /* @@ -554,7 +542,6 @@ static void trnman_free_trn(TRN *trn) */ *(TRN * volatile *)&(trn->next)= tmp.trn; } while (!my_atomic_casptr((void **)(char*)&pool, &tmp.v, trn)); - my_atomic_rwlock_wrunlock(&LOCK_pool); } /* diff --git a/storage/maria/unittest/ma_pagecache_consist.c b/storage/maria/unittest/ma_pagecache_consist.c index 5f0e25b5bf4..2c505428dab 100644 --- a/storage/maria/unittest/ma_pagecache_consist.c +++ b/storage/maria/unittest/ma_pagecache_consist.c @@ -77,30 +77,6 @@ static uint flush_divider= 1000; #endif /*TEST_HIGH_CONCURENCY*/ -/** - @brief Dummy pagecache callback. -*/ - -static my_bool -dummy_callback(uchar *page __attribute__((unused)), - pgcache_page_no_t page_no __attribute__((unused)), - uchar* data_ptr __attribute__((unused))) -{ - return 0; -} - - -/** - @brief Dummy pagecache callback. -*/ - -static void -dummy_fail_callback(uchar* data_ptr __attribute__((unused))) -{ - return; -} - - /* Get pseudo-random length of the field in (0;limit) @@ -392,8 +368,8 @@ int main(int argc __attribute__((unused)), errno); exit(1); } - pagecache_file_init(file1, &dummy_callback, &dummy_callback, - &dummy_fail_callback, &dummy_callback, NULL); + + pagecache_file_set_null_hooks(&file1); DBUG_PRINT("info", ("file1: %d", file1.file)); if (my_chmod(file1_name, 0777, MYF(MY_WME))) exit(1); diff --git a/storage/maria/unittest/ma_pagecache_rwconsist.c b/storage/maria/unittest/ma_pagecache_rwconsist.c index 1a268db6ad5..dbeb3a98052 100644 --- a/storage/maria/unittest/ma_pagecache_rwconsist.c +++ b/storage/maria/unittest/ma_pagecache_rwconsist.c @@ -49,30 +49,6 @@ static uint read_sleep_limit= 3; static uint report_divisor= 50; /** - @brief Dummy pagecache callback. -*/ - -static my_bool -dummy_callback(uchar *page __attribute__((unused)), - pgcache_page_no_t page_no __attribute__((unused)), - uchar* data_ptr __attribute__((unused))) -{ - return 0; -} - - -/** - @brief Dummy pagecache callback. -*/ - -static void -dummy_fail_callback(uchar* data_ptr __attribute__((unused))) -{ - return; -} - - -/** @brief Checks page consistency @param buff pointer to the page content @@ -262,8 +238,7 @@ int main(int argc __attribute__((unused)), errno); exit(1); } - pagecache_file_init(file1, &dummy_callback, &dummy_callback, - &dummy_fail_callback, &dummy_callback, NULL); + pagecache_file_set_null_hooks(&file1); DBUG_PRINT("info", ("file1: %d", file1.file)); if (my_chmod(file1_name, 0777, MYF(MY_WME))) exit(1); diff --git a/storage/maria/unittest/ma_pagecache_rwconsist2.c b/storage/maria/unittest/ma_pagecache_rwconsist2.c index 751c045a879..c06395d0fb3 100644 --- a/storage/maria/unittest/ma_pagecache_rwconsist2.c +++ b/storage/maria/unittest/ma_pagecache_rwconsist2.c @@ -55,30 +55,6 @@ static uint number_of_write_tests= 1000; static uint report_divisor= 50; /** - @brief Dummy pagecache callback. -*/ - -static my_bool -dummy_callback(uchar *page __attribute__((unused)), - pgcache_page_no_t page_no __attribute__((unused)), - uchar* data_ptr __attribute__((unused))) -{ - return 0; -} - - -/** - @brief Dummy pagecache callback. -*/ - -static void -dummy_fail_callback(uchar* data_ptr __attribute__((unused))) -{ - return; -} - - -/** @brief Checks page consistency @param buff pointer to the page content @@ -258,8 +234,7 @@ int main(int argc __attribute__((unused)), errno); exit(1); } - pagecache_file_init(file1, &dummy_callback, &dummy_callback, - &dummy_fail_callback, &dummy_callback, NULL); + pagecache_file_set_null_hooks(&file1); DBUG_PRINT("info", ("file1: %d", file1.file)); if (my_chmod(file1_name, 0777, MYF(MY_WME))) exit(1); diff --git a/storage/maria/unittest/ma_pagecache_single.c b/storage/maria/unittest/ma_pagecache_single.c index 64f6782f20f..6ae6f5b87a4 100644 --- a/storage/maria/unittest/ma_pagecache_single.c +++ b/storage/maria/unittest/ma_pagecache_single.c @@ -97,30 +97,6 @@ static struct file_desc simple_delete_flush_test_file[]= }; -/** - @brief Dummy pagecache callback. -*/ - -static my_bool -dummy_callback(uchar *page __attribute__((unused)), - pgcache_page_no_t page_no __attribute__((unused)), - uchar* data_ptr __attribute__((unused))) -{ - return 0; -} - - -/** - @brief Dummy pagecache callback. -*/ - -static void -dummy_fail_callback(uchar* data_ptr __attribute__((unused))) -{ - return; -} - - /* Recreate and reopen a file for test @@ -786,8 +762,7 @@ int main(int argc __attribute__((unused)), errno); exit(1); } - pagecache_file_init(file1, &dummy_callback, &dummy_callback, - &dummy_fail_callback, &dummy_callback, NULL); + pagecache_file_set_null_hooks(&file1); my_close(tmp_file, MYF(0)); my_delete(file2_name, MYF(0)); diff --git a/storage/maria/unittest/ma_test_loghandler_pagecache-t.c b/storage/maria/unittest/ma_test_loghandler_pagecache-t.c index 0e76f0d6042..365d15c69b8 100644 --- a/storage/maria/unittest/ma_test_loghandler_pagecache-t.c +++ b/storage/maria/unittest/ma_test_loghandler_pagecache-t.c @@ -38,30 +38,6 @@ static char file1_name[FN_REFLEN], first_translog_file[FN_REFLEN]; static PAGECACHE_FILE file1; -/** - @brief Dummy pagecache callback. -*/ - -static my_bool -dummy_callback(uchar *page __attribute__((unused)), - pgcache_page_no_t page_no __attribute__((unused)), - uchar* data_ptr __attribute__((unused))) -{ - return 0; -} - - -/** - @brief Dummy pagecache callback. -*/ - -static void -dummy_fail_callback(uchar* data_ptr __attribute__((unused))) -{ - return; -} - - int main(int argc __attribute__((unused)), char *argv[]) { uchar long_tr_id[6]; @@ -151,8 +127,9 @@ int main(int argc __attribute__((unused)), char *argv[]) errno); exit(1); } - pagecache_file_init(file1, &dummy_callback, &dummy_callback, - &dummy_fail_callback, maria_flush_log_for_page, NULL); + pagecache_file_set_null_hooks(&file1); + file1.flush_log_callback= maria_flush_log_for_page; + if (my_chmod(file1_name, 0777, MYF(MY_WME))) exit(1); |