diff options
37 files changed, 1988 insertions, 861 deletions
diff --git a/.bzrignore b/.bzrignore index 18b01f4714a..5bdb136c0c6 100644 --- a/.bzrignore +++ b/.bzrignore @@ -3060,3 +3060,6 @@ libmysql_r/client_settings.h libmysqld/scheduler.cc libmysqld/sql_connect.cc libmysqld/sql_tablespace.cc +sql/link_sources +ylwrap +libmysql_r/link_sources diff --git a/client/mysqltest.c b/client/mysqltest.c index fdee137fa98..558763f7cff 100644 --- a/client/mysqltest.c +++ b/client/mysqltest.c @@ -2018,9 +2018,9 @@ void var_set_query_get_value(struct st_command *command, VAR *var) static DYNAMIC_STRING ds_col; static DYNAMIC_STRING ds_row; const struct command_arg query_get_value_args[] = { - "query", ARG_STRING, TRUE, &ds_query, "Query to run", - "column name", ARG_STRING, TRUE, &ds_col, "Name of column", - "row number", ARG_STRING, TRUE, &ds_row, "Number for row", + {"query", ARG_STRING, TRUE, &ds_query, "Query to run"}, + {"column name", ARG_STRING, TRUE, &ds_col, "Name of column"}, + {"row number", ARG_STRING, TRUE, &ds_row, "Number for row"}, }; DBUG_ENTER("var_set_query_get_value"); @@ -5007,7 +5007,7 @@ static struct my_option my_long_options[] = "Don't use the memory allocation checking.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, {"sleep", 'T', "Sleep always this many seconds on sleep commands.", - (uchar**) &opt_sleep, (uchar**) &opt_sleep, 0, GET_INT, REQUIRED_ARG, -1, 0, 0, + (uchar**) &opt_sleep, (uchar**) &opt_sleep, 0, GET_INT, REQUIRED_ARG, -1, -1, 0, 0, 0, 0}, {"socket", 'S', "Socket file to use for connection.", (uchar**) &unix_sock, (uchar**) &unix_sock, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, diff --git a/dbug/dbug_add_tags.pl b/dbug/dbug_add_tags.pl index 141a2ed85f1..3e51a54c707 100755 --- a/dbug/dbug_add_tags.pl +++ b/dbug/dbug_add_tags.pl @@ -7,7 +7,7 @@ $ctags="exctags -x -f - --c-types=f -u"; sub get_tag { local $.; local $_=<TAGS>; ($symbol, $line)= /^(.*\S)\s+function\s+(\d+)/; - $symbol=$1 if /\s(\S+)\s*\(/; + $symbol=$1 if /[\s*]([^\s*]+)\s*\(/; $line=1e50 unless $line; } @@ -51,7 +51,7 @@ while($src=shift) $skip=!$semicolon; $semicolon= /;\s*$/; print && next if $skip || - (/^\s+\w+((::\w+)?|<\w+>)\s+\**\w+/ && !/^\s*return/); + (/^\s+\w+((::\w+)?|<\w+>)\s+\**\w+/ && !/^\s*return\b/); last if /DBUG_ENTER/; print "$tab DBUG_ENTER(\"$symbol\");\n"; print "\n" unless $_ eq "\n"; diff --git a/include/m_string.h b/include/m_string.h index 036eb8fe4d6..c24bfd7aa6c 100644 --- a/include/m_string.h +++ b/include/m_string.h @@ -203,6 +203,7 @@ double my_strtod(const char *str, char **end, int *error); double my_atof(const char *nptr); extern char *llstr(longlong value,char *buff); +extern char *ullstr(longlong value,char *buff); #ifndef HAVE_STRTOUL extern long strtol(const char *str, char **ptr, int base); extern ulong strtoul(const char *str, char **ptr, int base); diff --git a/include/my_sys.h b/include/my_sys.h index c2d6cabf78a..44f7731c9de 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -283,7 +283,13 @@ enum flush_type FLUSH_IGNORE_CHANGED, /* remove block from the cache */ /* as my_disable_flush_pagecache_blocks is always 0, it is strictly equivalent to FLUSH_KEEP */ - FLUSH_FORCE_WRITE + FLUSH_FORCE_WRITE, + /** + @brief like FLUSH_KEEP but return immediately if file is already being + flushed (even partially) by another thread; only for page cache, + forbidden for key cache. + */ + FLUSH_KEEP_LAZY }; typedef struct st_record_cache /* Used when cacheing records */ diff --git a/mysql-test/r/maria.result b/mysql-test/r/maria.result index 75c9ae99159..7fc6ae39224 100644 --- a/mysql-test/r/maria.result +++ b/mysql-test/r/maria.result @@ -2044,3 +2044,23 @@ check table t1 extended; Table Op Msg_type Msg_text test.t1 check status OK drop table t1; +show variables like 'maria%'; +Variable_name Value +maria_block_size 8192 +maria_checkpoint_interval 30 +maria_max_sort_file_size 9223372036853727232 +maria_pagecache_age_threshold 300 +maria_pagecache_buffer_size 8384512 +maria_pagecache_division_limit 100 +maria_repair_threads 1 +maria_sort_buffer_size 8388608 +maria_stats_method nulls_unequal +show status like 'maria%'; +Variable_name Value +Maria_pagecache_blocks_not_flushed # +Maria_pagecache_blocks_unused # +Maria_pagecache_blocks_used # +Maria_pagecache_read_requests # +Maria_pagecache_reads # +Maria_pagecache_write_requests # +Maria_pagecache_writes # diff --git a/mysql-test/t/maria.test b/mysql-test/t/maria.test index 9436be57e57..944ad68d7ba 100644 --- a/mysql-test/t/maria.test +++ b/mysql-test/t/maria.test @@ -1300,6 +1300,10 @@ update t1 set s1=2 where seq=1; check table t1 extended; drop table t1; +show variables like 'maria%'; +--replace_column 2 # +show status like 'maria%'; + # End of 5.2 tests --disable_result_log diff --git a/mysql-test/t/variables.test b/mysql-test/t/variables.test index 81db143b518..e1ed89cea4e 100644 --- a/mysql-test/t/variables.test +++ b/mysql-test/t/variables.test @@ -141,9 +141,9 @@ set GLOBAL myisam_max_sort_file_size=2000000; show global variables like 'myisam_max_sort_file_size'; select * from information_schema.global_variables where variable_name like 'myisam_max_sort_file_size'; set GLOBAL myisam_max_sort_file_size=default; ---replace_result 2147483647 FILE_SIZE 9223372036854775807 FILE_SIZE +--replace_result 2146435072 FILE_SIZE 9223372036853727232 FILE_SIZE show variables like 'myisam_max_sort_file_size'; ---replace_result 2147483647 FILE_SIZE 9223372036854775807 FILE_SIZE +--replace_result 2146435072 FILE_SIZE 9223372036853727232 FILE_SIZE select * from information_schema.session_variables where variable_name like 'myisam_max_sort_file_size'; set global net_retry_count=10, session net_retry_count=10; diff --git a/mysys/lf_alloc-pin.c b/mysys/lf_alloc-pin.c index 51c4df7c94a..a847d722023 100644 --- a/mysys/lf_alloc-pin.c +++ b/mysys/lf_alloc-pin.c @@ -318,9 +318,9 @@ static int match_pins(LF_PINS *el, void *addr) } #if STACK_DIRECTION < 0 -#define available_stack_size(END,CUR) (long) ((char*)(CUR) - (char*)(END)) +#define available_stack_size(CUR,END) (long) ((char*)(CUR) - (char*)(END)) #else -#define available_stack_size(END,CUR) (long) ((char*)(END) - (char*)(CUR)) +#define available_stack_size(CUR,END) (long) ((char*)(END) - (char*)(CUR)) #endif /* @@ -413,16 +413,20 @@ LF_REQUIRE_PINS(1); first->el->el->....->el->last. Use first==last to free only one element. */ static void alloc_free(struct st_lf_alloc_node *first, - struct st_lf_alloc_node *last, + struct st_lf_alloc_node volatile *last, LF_ALLOCATOR *allocator) { - struct st_lf_alloc_node *tmp; - tmp= allocator->top; + /* + we need a union here to access type-punned pointer reliably. + otherwise gcc -fstrict-aliasing will not see 'tmp' changed in the loop + */ + union { struct st_lf_alloc_node * node; void *ptr; } tmp; + tmp.node= allocator->top; do { - last->next= tmp; - } while (!my_atomic_casptr((void **)&allocator->top, (void **)&tmp, first) && - LF_BACKOFF); + last->next= tmp.node; + } while (!my_atomic_casptr((void **)(char *)&allocator->top, + (void **)&tmp.ptr, first) && LF_BACKOFF); } /* @@ -494,12 +498,13 @@ void *_lf_alloc_new(LF_PINS *pins) { node= (void *)my_malloc(allocator->element_size, MYF(MY_WME)); #ifdef MY_LF_EXTRA_DEBUG - if (likely(node)) + if (likely(node != 0)) my_atomic_add32(&allocator->mallocs, 1); #endif break; } - if (my_atomic_casptr((void **)&allocator->top, (void *)&node, node->next)) + if (my_atomic_casptr((void **)(char *)&allocator->top, + (void *)&node, node->next)) break; } _lf_unpin(pins, 0); diff --git a/mysys/mf_keycache.c b/mysys/mf_keycache.c index ea05ea6e127..8b1f3ad0540 100644 --- a/mysys/mf_keycache.c +++ b/mysys/mf_keycache.c @@ -3557,10 +3557,11 @@ static int flush_key_blocks_int(KEY_CACHE *keycache, file, keycache->blocks_used, keycache->blocks_changed)); #if !defined(DBUG_OFF) && defined(EXTRA_DEBUG) - DBUG_EXECUTE("check_keycache", - test_key_cache(keycache, "start of flush_key_blocks", 0);); + DBUG_EXECUTE("check_keycache", + test_key_cache(keycache, "start of flush_key_blocks", 0);); #endif + DBUG_ASSERT(type != FLUSH_KEEP_LAZY); cache= cache_buff; if (keycache->disk_blocks > 0 && (!my_disable_flush_key_blocks || type != FLUSH_KEEP)) diff --git a/mysys/my_getopt.c b/mysys/my_getopt.c index 3a5b130e067..218d9dce1f4 100644 --- a/mysys/my_getopt.c +++ b/mysys/my_getopt.c @@ -23,19 +23,14 @@ static void default_reporter(enum loglevel level, const char *format, ...); my_error_reporter my_getopt_error_reporter= &default_reporter; -static int findopt(char *optpat, uint length, - const struct my_option **opt_res, - char **ffname); -my_bool getopt_compare_strings(const char *s, - const char *t, - uint length); +static int findopt(char *, uint, const struct my_option **, char **); +my_bool getopt_compare_strings(const char *, const char *, uint); static longlong getopt_ll(char *arg, const struct my_option *optp, int *err); -static ulonglong getopt_ull(char *arg, const struct my_option *optp, - int *err); +static longlong getopt_ll_limit_value(longlong, const struct my_option *); +static ulonglong getopt_ull(char *, const struct my_option *, int *); static double getopt_double(char *arg, const struct my_option *optp, int *err); static void init_variables(const struct my_option *options); -static int setval(const struct my_option *opts, uchar* *value, char *argument, - my_bool set_maximum_value); +static int setval(const struct my_option *, uchar **, char *, my_bool); static char *check_struct_option(char *cur_arg, char *key_name); /* @@ -599,13 +594,17 @@ static int setval(const struct my_option *opts, uchar* *value, char *argument, *((my_bool*) result_pos)= (my_bool) atoi(argument) != 0; break; case GET_INT: - case GET_UINT: /* fall through */ *((int*) result_pos)= (int) getopt_ll(argument, opts, &err); break; + case GET_UINT: + *((uint*) result_pos)= (uint) getopt_ull(argument, opts, &err); + break; case GET_LONG: - case GET_ULONG: /* fall through */ *((long*) result_pos)= (long) getopt_ll(argument, opts, &err); break; + case GET_ULONG: + *((long*) result_pos)= (long) getopt_ull(argument, opts, &err); + break; case GET_LL: *((longlong*) result_pos)= getopt_ll(argument, opts, &err); break; @@ -748,7 +747,7 @@ static longlong eval_num_suffix(char *argument, int *error, char *option_name) return num; } -/* +/* function: getopt_ll Evaluates and returns the value that user gave as an argument @@ -761,16 +760,34 @@ static longlong eval_num_suffix(char *argument, int *error, char *option_name) static longlong getopt_ll(char *arg, const struct my_option *optp, int *err) { - longlong num; + longlong num=eval_num_suffix(arg, err, (char*) optp->name); + return getopt_ll_limit_value(num, optp); +} + +/* + function: getopt_ll_limit_value + + Applies min/max/block_size to a numeric value of an option. + Returns "fixed" value. +*/ + +static longlong getopt_ll_limit_value(longlong num, + const struct my_option *optp) +{ ulonglong block_size= (optp->block_size ? (ulonglong) optp->block_size : 1L); - - num= eval_num_suffix(arg, err, (char*) optp->name); + longlong old= num; + char buf1[255] __attribute__((unused)), buf2[255] __attribute__((unused)); + if (num > 0 && (ulonglong) num > (ulonglong) optp->max_value && optp->max_value) /* if max value is not set -> no upper limit */ num= (ulonglong) optp->max_value; num= ((num - optp->sub_size) / block_size); num= (longlong) (num * block_size); - return max(num, optp->min_value); + num= max(num, optp->min_value); + if (num != old) + DBUG_PRINT("options", ("option '%s' adjusted %s -> %s", + optp->name, llstr(old, buf1), llstr(num, buf2))); + return num; } /* @@ -782,15 +799,16 @@ static longlong getopt_ll(char *arg, const struct my_option *optp, int *err) static ulonglong getopt_ull(char *arg, const struct my_option *optp, int *err) { - ulonglong num; - - num= eval_num_suffix(arg, err, (char*) optp->name); + ulonglong num= eval_num_suffix(arg, err, (char*) optp->name); return getopt_ull_limit_value(num, optp); } ulonglong getopt_ull_limit_value(ulonglong num, const struct my_option *optp) { + ulonglong old= num; + char buf1[255] __attribute__((unused)), buf2[255] __attribute__((unused)); + if ((ulonglong) num > (ulonglong) optp->max_value && optp->max_value) /* if max value is not set -> no upper limit */ num= (ulonglong) optp->max_value; @@ -801,6 +819,9 @@ ulonglong getopt_ull_limit_value(ulonglong num, const struct my_option *optp) } if (num < (ulonglong) optp->min_value) num= (ulonglong) optp->min_value; + if (num != old) + DBUG_PRINT("options", ("option '%s' adjusted %s -> %s", + optp->name, ullstr(old, buf1), ullstr(num, buf2))); return num; } @@ -841,35 +862,39 @@ static double getopt_double(char *arg, const struct my_option *optp, int *err) SYNOPSIS init_one_value() - option Option to initialize - value Pointer to variable + optp Option to initialize + value Pointer to variable */ -static void init_one_value(const struct my_option *option, uchar* *variable, +static void init_one_value(const struct my_option *optp, uchar* *variable, longlong value) { DBUG_ENTER("init_one_value"); - switch ((option->var_type & GET_TYPE_MASK)) { + switch ((optp->var_type & GET_TYPE_MASK)) { case GET_BOOL: *((my_bool*) variable)= (my_bool) value; break; case GET_INT: - *((int*) variable)= (int) value; + *((int*) variable)= (int) getopt_ll_limit_value(value, optp); break; case GET_UINT: + *((uint*) variable)= (uint) getopt_ull_limit_value(value, optp); + break; case GET_ENUM: *((uint*) variable)= (uint) value; break; case GET_LONG: - *((long*) variable)= (long) value; + *((long*) variable)= (long) getopt_ll_limit_value(value, optp); break; case GET_ULONG: - *((ulong*) variable)= (ulong) value; + *((ulong*) variable)= (ulong) getopt_ull_limit_value(value, optp); break; case GET_LL: - *((longlong*) variable)= (longlong) value; + *((longlong*) variable)= (longlong) getopt_ll_limit_value(value, optp); break; case GET_ULL: + *((ulonglong*) variable)= (ulonglong) getopt_ull_limit_value(value, optp); + break; case GET_SET: *((ulonglong*) variable)= (ulonglong) value; break; @@ -906,7 +931,7 @@ static void init_one_value(const struct my_option *option, uchar* *variable, } -/* +/* initialize all variables to their default values SYNOPSIS diff --git a/sql/handler.cc b/sql/handler.cc index 2839b8f6e6f..2fe04c5dba3 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -28,10 +28,6 @@ #include <myisampack.h> #include <errno.h> -#ifdef WITH_MARIA_STORAGE_ENGINE -#include <maria.h> -#endif - #ifdef WITH_PARTITION_STORAGE_ENGINE #include "ha_partition.h" #endif @@ -2772,97 +2768,6 @@ int ha_change_key_cache(KEY_CACHE *old_key_cache, } -/***************************************************************************** - pagecache handling. - - This code is only relevant for maria tables - - pagecache->cache may be 0 only in the case where a key cache is not - initialized or when we where not able to init the key cache in a previous - call to ha_init_pagecache() (probably out of memory) -*****************************************************************************/ - - -#ifdef WITH_MARIA_STORAGE_ENGINE - -/* Init a pagecache if it has not been initied before */ - -int ha_init_pagecache(const char *name, PAGECACHE *pagecache) -{ - DBUG_ENTER("ha_init_pagecache"); - - if (!pagecache->inited) - { - pthread_mutex_lock(&LOCK_global_system_variables); - long tmp_buff_size= (long) pagecache->param_buff_size; - uint division_limit= pagecache->param_division_limit; - uint age_threshold= pagecache->param_age_threshold; - pthread_mutex_unlock(&LOCK_global_system_variables); - DBUG_RETURN(!init_pagecache(pagecache, - tmp_buff_size, division_limit, age_threshold, - MARIA_KEY_BLOCK_LENGTH)); - } - DBUG_RETURN(0); -} - - -/* Resize key cache */ -/* -TODO: uncomment when resize will be implemented -int ha_resize_pagecache(PAGECACHE *pagecache) -{ - DBUG_ENTER("ha_resize_pagecache"); - - if (pagecache->inited) - { - pthread_mutex_lock(&LOCK_global_system_variables); - long tmp_buff_size= (long) pagecache->param_buff_size; - long tmp_block_size= (long) pagecache->param_block_size; - uint division_limit= pagecache->param_division_limit; - uint age_threshold= pagecache->param_age_threshold; - pthread_mutex_unlock(&LOCK_global_system_variables); - DBUG_RETURN(!resize_pagecache(pagecache, tmp_block_size, - tmp_buff_size, - division_limit, age_threshold)); - } - DBUG_RETURN(0); -} -*/ - - -/* Change parameters for key cache (like size) */ - -int ha_change_pagecache_param(PAGECACHE *pagecache) -{ - if (pagecache->inited) - { - pthread_mutex_lock(&LOCK_global_system_variables); - uint division_limit= pagecache->param_division_limit; - uint age_threshold= pagecache->param_age_threshold; - pthread_mutex_unlock(&LOCK_global_system_variables); - change_pagecache_param(pagecache, division_limit, age_threshold); - } - return 0; -} - -/* Free memory allocated by a key cache */ - -int ha_end_pagecache(PAGECACHE *pagecache) -{ - end_pagecache(pagecache, 1); // Can never fail - return 0; -} - -/* Move all tables from one key cache to another one */ - -int ha_change_pagecache(PAGECACHE *old_pagecache, - PAGECACHE *new_pagecache) -{ - maria_change_pagecache(old_pagecache, new_pagecache); - return 0; -} - -#endif /* WITH_MARIA_STORAGE_ENGINE */ /** @brief Try to discover one table from handler(s) diff --git a/sql/handler.h b/sql/handler.h index 55062b54532..5bbb68b6913 100644 --- a/sql/handler.h +++ b/sql/handler.h @@ -23,9 +23,6 @@ #include <my_handler.h> #include <ft_global.h> #include <keycache.h> -#ifdef WITH_MARIA_STORAGE_ENGINE -#include "../storage/maria/ma_pagecache.h" -#endif #ifndef NO_HASH #define NO_HASH /* Not yet implemented */ @@ -276,7 +273,6 @@ enum legacy_db_type DB_TYPE_TABLE_FUNCTION, DB_TYPE_MEMCACHE, DB_TYPE_FALCON, - DB_TYPE_MARIA, DB_TYPE_FIRST_DYNAMIC=42, DB_TYPE_DEFAULT=127 // Must be last }; @@ -900,9 +896,6 @@ typedef struct st_ha_check_opt uint flags; /* isam layer flags (e.g. for myisamchk) */ uint sql_flags; /* sql layer flags - for something myisamchk cannot do */ KEY_CACHE *key_cache; /* new key cache when changing key cache */ -#ifdef WITH_MARIA_STORAGE_ENGINE - PAGECACHE *pagecache; /* new pagecache when changing pagecache */ -#endif void init(); } HA_CHECK_OPT; @@ -1896,17 +1889,6 @@ int ha_resize_key_cache(KEY_CACHE *key_cache); int ha_change_key_cache_param(KEY_CACHE *key_cache); int ha_change_key_cache(KEY_CACHE *old_key_cache, KEY_CACHE *new_key_cache); int ha_end_key_cache(KEY_CACHE *key_cache); -#ifdef WITH_MARIA_STORAGE_ENGINE -/* pagecache */ -int ha_init_pagecache(const char *name, PAGECACHE *pagecache); -/* -TODO: uncomment when resizing will be implemented -int ha_resize_pagecache(PAGECACHE *pagecache); -*/ -int ha_change_pagecache_param(PAGECACHE *pagecache); -int ha_change_pagecache(PAGECACHE *old_pagecache, PAGECACHE *new_pagecache); -int ha_end_pagecache(PAGECACHE *pagecache); -#endif /* report to InnoDB that control passes to the client */ int ha_release_temporary_latches(THD *thd); diff --git a/sql/mysql_priv.h b/sql/mysql_priv.h index 1400fe3e1bb..f7738e44404 100644 --- a/sql/mysql_priv.h +++ b/sql/mysql_priv.h @@ -1853,9 +1853,6 @@ extern pthread_cond_t COND_global_read_lock; extern pthread_attr_t connection_attrib; extern I_List<THD> threads; extern I_List<NAMED_LIST> key_caches; -#ifdef WITH_MARIA_STORAGE_ENGINE -extern I_List<NAMED_LIST> pagecaches; -#endif /* WITH_MARIA_STORAGE_ENGINE */ extern MY_BITMAP temp_pool; extern String my_empty_string; extern const String my_null_string; @@ -1869,6 +1866,9 @@ extern struct system_variables max_system_variables; extern struct system_status_var global_status_var; extern struct my_rnd_struct sql_rand; +extern handlerton *maria_hton; /* @todo remove, make it static in ha_maria.cc + currently it's needed for sql_delete.cc */ + extern const char *opt_date_time_formats[]; extern KNOWN_DATE_TIME_FORMAT known_date_time_formats[]; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 19138c6cda3..ea4473e5b1d 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -28,9 +28,6 @@ #include "events.h" #include "../storage/myisam/ha_myisam.h" -#ifdef WITH_MARIA_STORAGE_ENGINE -#include "../storage/maria/ha_maria.h" -#endif #include "rpl_injector.h" @@ -528,9 +525,6 @@ FILE *stderror_file=0; I_List<THD> threads; I_List<NAMED_LIST> key_caches; -#ifdef WITH_MARIA_STORAGE_ENGINE -I_List<NAMED_LIST> pagecaches; -#endif /* WITH_MARIA_STORAGE_ENGINE */ Rpl_filter* rpl_filter; Rpl_filter* binlog_filter; @@ -1214,13 +1208,7 @@ void clean_up(bool print_message) tc_log->close(); xid_cache_free(); delete_elements(&key_caches, (void (*)(const char*, uchar*)) free_key_cache); -#ifdef WITH_MARIA_STORAGE_ENGINE - delete_elements(&pagecaches, (void (*)(const char*, uchar*)) free_pagecache); -#endif /* WITH_MARIA_STORAGE_ENGINE */ multi_keycache_free(); -#ifdef WITH_MARIA_STORAGE_ENGINE - multi_pagecache_free(); -#endif free_status_vars(); end_thr_alarm(1); /* Free allocated memory */ my_free_open_file_info(); @@ -2209,10 +2197,6 @@ the problem, but since we have already crashed, something is definitely wrong\n\ and this may fail.\n\n"); fprintf(stderr, "key_buffer_size=%lu\n", (ulong) dflt_key_cache->key_cache_mem_size); -#ifdef WITH_MARIA_STORAGE_ENGINE - fprintf(stderr, "page_buffer_size=%lu\n", - (ulong) maria_pagecache->mem_size); -#endif /* WITH_MARIA_STORAGE_ENGINE */ fprintf(stderr, "read_buffer_size=%ld\n", (long) global_system_variables.read_buff_size); fprintf(stderr, "max_used_connections=%lu\n", max_used_connections); fprintf(stderr, "max_threads=%u\n", thread_scheduler.max_threads); @@ -2220,9 +2204,6 @@ and this may fail.\n\n"); fprintf(stderr, "It is possible that mysqld could use up to \n\ key_buffer_size + (read_buffer_size + sort_buffer_size)*max_threads = %lu K\n\ bytes of memory\n", ((ulong) dflt_key_cache->key_cache_mem_size + -#ifdef WITH_MARIA_STORAGE_ENGINE - (ulong) maria_pagecache->mem_size + -#endif /* WITH_MARIA_STORAGE_ENGINE */ (global_system_variables.read_buff_size + global_system_variables.sortbuff_size) * thread_scheduler.max_threads + @@ -3419,14 +3400,6 @@ server."); /* call ha_init_key_cache() on all key caches to init them */ process_key_caches(&ha_init_key_cache); - /* - Maria's pagecache needs to be ready before Maria engine (Recovery uses - pagecache, and Checkpoint may happen at startup). Maria engine is taken up - in plugin_init(). - */ -#ifdef WITH_MARIA_STORAGE_ENGINE - process_pagecaches(&ha_init_pagecache); -#endif /* WITH_MARIA_STORAGE_ENGINE */ /* Allow storage engine to give real error messages */ if (ha_init_errors()) @@ -6045,7 +6018,7 @@ log and this option does nothing anymore.", "Max packetlength to send/receive from to server.", (uchar**) &global_system_variables.max_allowed_packet, (uchar**) &max_system_variables.max_allowed_packet, 0, GET_ULONG, - REQUIRED_ARG, 1024*1024L, 1024, 1024L*1024L*1024L, MALLOC_OVERHEAD, 1024, 0}, + REQUIRED_ARG, 1024*1024L, 1024, 1024L*1024L*1024L, 0, 1024, 0}, {"max_binlog_cache_size", OPT_MAX_BINLOG_CACHE_SIZE, "Can be used to restrict the total size used to cache a multi-transaction query.", (uchar**) &max_binlog_cache_size, (uchar**) &max_binlog_cache_size, 0, @@ -6123,7 +6096,7 @@ The minimum value for this variable is 4096.", {"max_user_connections", OPT_MAX_USER_CONNECTIONS, "The maximum number of active connections for a single user (0 = no limit).", (uchar**) &max_user_connections, (uchar**) &max_user_connections, 0, GET_UINT, - REQUIRED_ARG, 0, 1, ~0, 0, 1, 0}, + REQUIRED_ARG, 0, 0, ~0, 0, 1, 0}, {"max_write_lock_count", OPT_MAX_WRITE_LOCK_COUNT, "After this many write locks, allow some read locks to run in between.", (uchar**) &max_write_lock_count, (uchar**) &max_write_lock_count, 0, GET_ULONG, @@ -6214,24 +6187,6 @@ The minimum value for this variable is 4096.", (uchar**) &global_system_variables.optimizer_search_depth, (uchar**) &max_system_variables.optimizer_search_depth, 0, GET_ULONG, OPT_ARG, MAX_TABLES+1, 0, MAX_TABLES+2, 0, 1, 0}, -#ifdef WITH_MARIA_STORAGE_ENGINE - {"pagecache_age_threshold", OPT_PAGECACHE_AGE_THRESHOLD, - "This characterizes the number of hits a hot block has to be untouched until it is considered aged enough to be downgraded to a warm block. This specifies the percentage ratio of that number of hits to the total number of blocks in key cache", - (uchar**) &maria_pagecache_var.param_age_threshold, - (uchar**) 0, 0, (GET_ULONG | GET_ASK_ADDR), REQUIRED_ARG, - 300, 100, ~0L, 0, 100, 0}, - {"pagecache_buffer_size", OPT_PAGECACHE_BUFFER_SIZE, - "The size of the buffer used for index blocks for Maria tables. Increase this to get better index handling (for all reads and multiple writes) to as much as you can afford; 64M on a 256M machine that mainly runs MySQL is quite common.", - (uchar**) &maria_pagecache_var.param_buff_size, - (uchar**) 0, 0, (GET_ULL | GET_ASK_ADDR), REQUIRED_ARG, - KEY_CACHE_SIZE, MALLOC_OVERHEAD, ~(ulong) 0, MALLOC_OVERHEAD, IO_SIZE, 0}, - {"pagecache_division_limit", OPT_PAGECACHE_DIVISION_LIMIT, - "The minimum percentage of warm blocks in key cache", - (uchar**) &maria_pagecache_var.param_division_limit, - (uchar**) 0, - 0, (GET_ULONG | GET_ASK_ADDR) , REQUIRED_ARG, 100, - 1, 100, 0, 1, 0}, -#endif /* WITH_MARIA_STORAGE_ENGINE */ {"plugin_dir", OPT_PLUGIN_DIR, "Directory for plugins.", (uchar**) &opt_plugin_dir_ptr, (uchar**) &opt_plugin_dir_ptr, 0, @@ -6288,7 +6243,7 @@ The minimum value for this variable is 4096.", "Allocation block size for storing ranges during optimization", (uchar**) &global_system_variables.range_alloc_block_size, (uchar**) &max_system_variables.range_alloc_block_size, 0, GET_ULONG, - REQUIRED_ARG, RANGE_ALLOC_BLOCK_SIZE, 4096, ~0L, 0, 1024, 0}, + REQUIRED_ARG, RANGE_ALLOC_BLOCK_SIZE, RANGE_ALLOC_BLOCK_SIZE, ~0L, 0, 1024, 0}, {"read_buffer_size", OPT_RECORD_BUFFER, "Each thread that does a sequential scan allocates a buffer of this size for each table it scans. If you do many sequential scans, you may want to increase this value.", (uchar**) &global_system_variables.read_buff_size, @@ -7160,22 +7115,12 @@ static void mysql_init_variables(void) threads.empty(); thread_cache.empty(); key_caches.empty(); -#ifdef WITH_MARIA_STORAGE_ENGINE - pagecaches.empty(); -#endif /* WITH_MARIA_STORAGE_ENGINE */ if (!(dflt_key_cache= get_or_create_key_cache(default_key_cache_base.str, default_key_cache_base.length))) exit(1); /* set key_cache_hash.default_value = dflt_key_cache */ multi_keycache_init(); -#ifdef WITH_MARIA_STORAGE_ENGINE - if (!(maria_pagecache= get_or_create_pagecache(maria_pagecache_base.str, - maria_pagecache_base.length))) - exit(1); - /* set pagecache_hash.default_value = maria_pagecache */ - multi_pagecache_init(); -#endif /* Set directory paths */ strmake(language, LANGUAGE, sizeof(language)-1); @@ -7842,24 +7787,6 @@ mysql_getopt_value(const char *keyname, uint key_length, return (uchar**) &key_cache->param_age_threshold; } } -#ifdef WITH_MARIA_STORAGE_ENGINE - case OPT_PAGECACHE_BUFFER_SIZE: - case OPT_PAGECACHE_DIVISION_LIMIT: - case OPT_PAGECACHE_AGE_THRESHOLD: - { - PAGECACHE *pagecache; - if (!(pagecache= get_or_create_pagecache(keyname, key_length))) - exit(1); - switch (option->id) { - case OPT_PAGECACHE_BUFFER_SIZE: - return (uchar**) &pagecache->param_buff_size; - case OPT_PAGECACHE_DIVISION_LIMIT: - return (uchar**) &pagecache->param_division_limit; - case OPT_PAGECACHE_AGE_THRESHOLD: - return (uchar**) &pagecache->param_age_threshold; - } - } -#endif } return option->value; } @@ -8263,9 +8190,6 @@ void refresh_status(THD *thd) /* Reset the counters of all key caches (default and named). */ process_key_caches(reset_key_cache_counters); -#ifdef WITH_MARIA_STORAGE_ENGINE - process_pagecaches(reset_pagecache_counters); -#endif /* WITH_MARIA_STORAGE_ENGINE */ pthread_mutex_unlock(&LOCK_status); /* diff --git a/sql/set_var.cc b/sql/set_var.cc index 95bdc54a5e9..0815c7df755 100644 --- a/sql/set_var.cc +++ b/sql/set_var.cc @@ -54,9 +54,6 @@ #include <thr_alarm.h> #include <myisam.h> #include <my_dir.h> -#ifdef WITH_MARIA_STORAGE_ENGINE -#include <maria.h> -#endif #include "events.h" @@ -124,9 +121,6 @@ static void fix_thd_mem_root(THD *thd, enum_var_type type); static void fix_trans_mem_root(THD *thd, enum_var_type type); static void fix_server_id(THD *thd, enum_var_type type); static KEY_CACHE *create_key_cache(const char *name, uint length); -#ifdef WITH_MARIA_STORAGE_ENGINE -static PAGECACHE *create_pagecache(const char *name, uint length); -#endif /* WITH_MARIA_STORAGE_ENGINE */ void fix_sql_mode_var(THD *thd, enum_var_type type); static uchar *get_error_count(THD *thd); static uchar *get_warning_count(THD *thd); @@ -244,14 +238,6 @@ static sys_var_key_cache_long sys_key_cache_division_limit(&vars, "key_cache_div static sys_var_key_cache_long sys_key_cache_age_threshold(&vars, "key_cache_age_threshold", offsetof(KEY_CACHE, param_age_threshold)); -#ifdef WITH_MARIA_STORAGE_ENGINE -sys_var_pagecache_long sys_pagecache_division_limit("pagecache_division_limit", - offsetof(PAGECACHE, - param_division_limit)); -sys_var_pagecache_long sys_pagecache_age_threshold("pagecache_age_threshold", - offsetof(KEY_CACHE, - param_age_threshold)); -#endif /* WITH_MARIA_STORAGE_ENGINE */ static sys_var_bool_ptr sys_local_infile(&vars, "local_infile", &opt_local_infile); static sys_var_trust_routine_creators @@ -1919,10 +1905,6 @@ LEX_STRING default_key_cache_base= {(char *) "default", 7 }; static KEY_CACHE zero_key_cache; -#ifdef WITH_MARIA_STORAGE_ENGINE -LEX_STRING maria_pagecache_base= {(char *) "default", 7 }; -static PAGECACHE zero_pagecache; -#endif /* WITH_MARIA_STORAGE_ENGINE */ KEY_CACHE *get_key_cache(LEX_STRING *cache_name) { @@ -1933,17 +1915,6 @@ KEY_CACHE *get_key_cache(LEX_STRING *cache_name) cache_name->str, cache_name->length, 0)); } -#ifdef WITH_MARIA_STORAGE_ENGINE -PAGECACHE *get_pagecache(LEX_STRING *cache_name) -{ - safe_mutex_assert_owner(&LOCK_global_system_variables); - if (!cache_name || ! cache_name->length) - cache_name= &default_key_cache_base; - return ((PAGECACHE*) find_named(&pagecaches, - cache_name->str, cache_name->length, 0)); -} -#endif /* WITH_MARIA_STORAGE_ENGINE */ - uchar *sys_var_key_cache_param::value_ptr(THD *thd, enum_var_type type, LEX_STRING *base) { @@ -1954,18 +1925,6 @@ uchar *sys_var_key_cache_param::value_ptr(THD *thd, enum_var_type type, } -#ifdef WITH_MARIA_STORAGE_ENGINE -uchar *sys_var_pagecache_param::value_ptr(THD *thd, enum_var_type type, - LEX_STRING *base) -{ - PAGECACHE *pagecache= get_pagecache(base); - if (!pagecache) - pagecache= &zero_pagecache; - return (uchar*) pagecache + offset ; -} -#endif /* WITH_MARIA_STORAGE_ENGINE */ - - bool sys_var_key_buffer_size::update(THD *thd, set_var *var) { ulonglong tmp= var->save_result.ulonglong_value; @@ -2102,60 +2061,6 @@ end: } -#ifdef WITH_MARIA_STORAGE_ENGINE -bool sys_var_pagecache_long::update(THD *thd, set_var *var) -{ - ulong tmp= (ulong) var->value->val_int(); - LEX_STRING *base_name= &var->base; - bool error= 0; - - if (!base_name->length) - base_name= &maria_pagecache_base; - - pthread_mutex_lock(&LOCK_global_system_variables); - PAGECACHE *pagecache= get_pagecache(base_name); - - if (!pagecache && !(pagecache= create_pagecache(base_name->str, - base_name->length))) - { - error= 1; - goto end; - } - - /* - Abort if some other thread is changing the key cache - TODO: This should be changed so that we wait until the previous - assignment is done and then do the new assign - */ - if (pagecache->in_init) - goto end; - - *((ulong*) (((char*) pagecache) + offset))= - (ulong) getopt_ull_limit_value(tmp, option_limits); - - /* - Don't create a new key cache if it didn't exist - (pagecaches are created only when the user sets block_size) - */ - pagecache->in_init= 1; - - pthread_mutex_unlock(&LOCK_global_system_variables); - - /* - TODO: uncomment whan it will be implemented - error= (bool) (ha_resize_pagecache(pagecache)); - */ - - pthread_mutex_lock(&LOCK_global_system_variables); - pagecache->in_init= 0; - -end: - pthread_mutex_unlock(&LOCK_global_system_variables); - return error; -} -#endif /* WITH_MARIA_STORAGE_ENGINE */ - - bool sys_var_log_state::update(THD *thd, set_var *var) { bool res; @@ -3694,78 +3599,6 @@ bool process_key_caches(process_key_cache_t func) } -#ifdef WITH_MARIA_STORAGE_ENGINE - -static PAGECACHE *create_pagecache(const char *name, uint length) -{ - PAGECACHE *pagecache; - DBUG_ENTER("create_pagecache"); - DBUG_PRINT("enter",("name: %.*s", length, name)); - - if ((pagecache= (PAGECACHE*) my_malloc(sizeof(PAGECACHE), - MYF(MY_ZEROFILL | MY_WME)))) - { - if (!new NAMED_LIST(&pagecaches, name, length, (uchar*) pagecache)) - { - my_free((char*) pagecache, MYF(0)); - pagecache= 0; - } - else - { - /* - Set default values for a key cache - The values in maria_pagecache_var is set by my_getopt() at startup - We don't set 'buff_size' as this is used to enable the key cache - */ - pagecache->param_buff_size= (maria_pagecache_var.param_buff_size ? - maria_pagecache_var.param_buff_size: - KEY_CACHE_SIZE); - pagecache->param_division_limit= maria_pagecache_var.param_division_limit; - pagecache->param_age_threshold= maria_pagecache_var.param_age_threshold; - } - } - DBUG_RETURN(pagecache); -} - - -PAGECACHE *get_or_create_pagecache(const char *name, uint length) -{ - LEX_STRING pagecache_name; - PAGECACHE *pagecache; - - pagecache_name.str= (char *) name; - pagecache_name.length= length; - pthread_mutex_lock(&LOCK_global_system_variables); - if (!(pagecache= get_pagecache(&pagecache_name))) - pagecache= create_pagecache(name, length); - pthread_mutex_unlock(&LOCK_global_system_variables); - return pagecache; -} - - -void free_pagecache(const char *name, PAGECACHE *pagecache) -{ - ha_end_pagecache(pagecache); - my_free((char*) pagecache, MYF(0)); -} - - -bool process_pagecaches(int (* func) (const char *name, PAGECACHE *)) -{ - I_List_iterator<NAMED_LIST> it(pagecaches); - NAMED_LIST *element; - - while ((element= it++)) - { - PAGECACHE *pagecache= (PAGECACHE *) element->data; - func(element->name, pagecache); - } - return 0; -} - -#endif /* WITH_MARIA_STORAGE_ENGINE */ - - void sys_var_trust_routine_creators::warn_deprecated(THD *thd) { WARN_DEPRECATED(thd, "5.2", "log_bin_trust_routine_creators", diff --git a/sql/set_var.h b/sql/set_var.h index b3ef4389038..eb2c893c89e 100644 --- a/sql/set_var.h +++ b/sql/set_var.h @@ -734,33 +734,6 @@ public: }; -#ifdef WITH_MARIA_STORAGE_ENGINE -class sys_var_pagecache_param :public sys_var -{ -protected: - size_t offset; -public: - sys_var_pagecache_param(const char *name_arg, size_t offset_arg) - :sys_var(name_arg), offset(offset_arg) - {} - uchar *value_ptr(THD *thd, enum_var_type type, LEX_STRING *base); - bool check_default(enum_var_type type) { return 1; } - bool is_struct() { return 1; } -}; - - -class sys_var_pagecache_long :public sys_var_pagecache_param -{ -public: - sys_var_pagecache_long(const char *name_arg, size_t offset_arg) - :sys_var_pagecache_param(name_arg, offset_arg) - {} - bool update(THD *thd, set_var *var); - SHOW_TYPE type() { return SHOW_LONG; } -}; -#endif /* WITH_MARIA_STORAGE_ENGINE */ - - class sys_var_thd_date_time_format :public sys_var_thd { DATE_TIME_FORMAT *SV::*offset; @@ -1185,10 +1158,6 @@ public: my_free((uchar*) name, MYF(0)); } friend bool process_key_caches(process_key_cache_t func); -#ifdef WITH_MARIA_STORAGE_ENGINE - friend bool process_pagecaches(int (* func) (const char *name, - PAGECACHE *)); -#endif /* WITH_MARIA_STORAGE_ENGINE */ friend void delete_elements(I_List<NAMED_LIST> *list, void (*free_element)(const char*, uchar*)); }; @@ -1198,9 +1167,6 @@ public: extern sys_var_thd_bool sys_old_alter_table; extern sys_var_thd_bool sys_old_passwords; extern LEX_STRING default_key_cache_base; -#ifdef WITH_MARIA_STORAGE_ENGINE -extern LEX_STRING maria_pagecache_base; -#endif /* WITH_MARIA_STORAGE_ENGINE */ /* For sql_yacc */ struct sys_var_with_base @@ -1242,8 +1208,3 @@ void free_key_cache(const char *name, KEY_CACHE *key_cache); bool process_key_caches(process_key_cache_t func); void delete_elements(I_List<NAMED_LIST> *list, void (*free_element)(const char*, uchar*)); -#ifdef WITH_MARIA_STORAGE_ENGINE -PAGECACHE *get_or_create_pagecache(const char *name, uint length); -void free_pagecache(const char *name, PAGECACHE *pagecache); -bool process_pagecaches(int (* func) (const char *name, PAGECACHE *)); -#endif /* WITH_MARIA_STORAGE_ENGINE */ diff --git a/sql/sql_delete.cc b/sql/sql_delete.cc index f53d01aa30b..87739b7fb2d 100644 --- a/sql/sql_delete.cc +++ b/sql/sql_delete.cc @@ -939,7 +939,7 @@ bool mysql_truncate(THD *thd, TABLE_LIST *table_list, bool dont_send_ok) is correctly created as non-transactional but then, when truncated, is recreated as transactional. */ - if (table_type->db_type == DB_TYPE_MARIA) + if (table_type == maria_hton) create_info.transactional= HA_CHOICE_NO; #endif close_temporary_table(thd, table, 0, 0); // Don't free share diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index 42bfe679aa7..1d36c05ee4c 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -44,6 +44,9 @@ C_MODE_END #define trans_register_ha(A, B, C) do { /* nothing */ } while(0) #endif +ulong pagecache_division_limit, pagecache_age_threshold; +ulonglong pagecache_buffer_size; + /** @todo For now there is no way for a user to set a different value of maria_recover_options, i.e. auto-check-and-repair is always disabled. @@ -54,7 +57,7 @@ C_MODE_END still corrupted. */ ulong maria_recover_options= HA_RECOVER_NONE; -static handlerton *maria_hton; +handlerton *maria_hton; /* bits in maria_recover_options */ const char *maria_recover_names[]= @@ -78,9 +81,11 @@ TYPELIB maria_stats_method_typelib= maria_stats_method_names, NULL }; -static void update_checkpoint_frequency(MYSQL_THD thd, - struct st_mysql_sys_var *var, - void *var_ptr, void *save); +/** @brief Interval between background checkpoints in seconds */ +static ulong checkpoint_interval; +static void update_checkpoint_interval(MYSQL_THD thd, + struct st_mysql_sys_var *var, + void *var_ptr, void *save); static MYSQL_SYSVAR_ULONG(block_size, maria_block_size, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, @@ -88,11 +93,11 @@ static MYSQL_SYSVAR_ULONG(block_size, maria_block_size, MARIA_KEY_BLOCK_LENGTH, MARIA_MIN_KEY_BLOCK_LENGTH, MARIA_MAX_KEY_BLOCK_LENGTH, MARIA_MIN_KEY_BLOCK_LENGTH); -static MYSQL_SYSVAR_ULONG(checkpoint_frequency, maria_checkpoint_frequency, +static MYSQL_SYSVAR_ULONG(checkpoint_interval, checkpoint_interval, PLUGIN_VAR_RQCMDARG, - "Frequency of automatic checkpoints, in seconds;" - " 0 means 'no checkpoints'.", - NULL, update_checkpoint_frequency, 30, 0, UINT_MAX, 1); + "Interval between automatic checkpoints, in seconds;" + " 0 means 'no automatic checkpoints'.", + NULL, update_checkpoint_interval, 30, 0, UINT_MAX, 1); static MYSQL_SYSVAR_ULONGLONG(max_sort_file_size, maria_max_temp_length, PLUGIN_VAR_RQCMDARG, @@ -100,6 +105,26 @@ static MYSQL_SYSVAR_ULONGLONG(max_sort_file_size, "temporary file would get bigger than this.", 0, 0, MAX_FILE_SIZE, 0, MAX_FILE_SIZE, 1024*1024); +static MYSQL_SYSVAR_ULONG(pagecache_age_threshold, + pagecache_age_threshold, PLUGIN_VAR_RQCMDARG, + "This characterizes the number of hits a hot block has to be untouched " + "until it is considered aged enough to be downgraded to a warm block. " + "This specifies the percentage ratio of that number of hits to the " + "total number of blocks in the page cache.", 0, 0, + 300, 100, ~0L, 100); + +static MYSQL_SYSVAR_ULONGLONG(pagecache_buffer_size, pagecache_buffer_size, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "The size of the buffer used for index blocks for Maria tables. " + "Increase this to get better index handling (for all reads and multiple " + "writes) to as much as you can afford.", 0, 0, + KEY_CACHE_SIZE, MALLOC_OVERHEAD, ~(ulong) 0, IO_SIZE); + +static MYSQL_SYSVAR_ULONG(pagecache_division_limit, pagecache_division_limit, + PLUGIN_VAR_RQCMDARG, + "The minimum percentage of warm blocks in key cache", 0, 0, + 100, 1, 100, 1); + static MYSQL_THDVAR_ULONG(repair_threads, PLUGIN_VAR_RQCMDARG, "Number of threads to use when repairing maria tables. The value of 1 " "disables parallel repair.", @@ -739,7 +764,7 @@ int ha_maria::open(const char *name, int mode, uint test_if_locked) growing files. Using an open_flag instead of calling ma_extra(... HA_EXTRA_MMAP ...) after maxs_open() has the advantage that the mapping is not repeated for every open, but just done on the initial - open, when the MyISAM share is created. Everytime the server + open, when the MyISAM share is created. Every time the server requires to open a new instance of a table it calls this method. We will always supply HA_OPEN_MMAP for a permanent table. However, the Maria storage engine will ignore this flag if this is a secondary @@ -1300,6 +1325,7 @@ int ha_maria::repair(THD *thd, HA_CHECK ¶m, bool do_optimize) int ha_maria::assign_to_keycache(THD * thd, HA_CHECK_OPT *check_opt) { +#if 0 && NOT_IMPLEMENTED PAGECACHE *new_pagecache= check_opt->pagecache; const char *errmsg= 0; int error= HA_ADMIN_OK; @@ -1307,8 +1333,6 @@ int ha_maria::assign_to_keycache(THD * thd, HA_CHECK_OPT *check_opt) TABLE_LIST *table_list= table->pos_in_table_list; DBUG_ENTER("ha_maria::assign_to_keycache"); - /* for now, it is disabled */ - DBUG_RETURN(HA_ADMIN_NOT_IMPLEMENTED); table->keys_in_use_for_query.clear_all(); @@ -1341,6 +1365,9 @@ int ha_maria::assign_to_keycache(THD * thd, HA_CHECK_OPT *check_opt) _ma_check_print_error(¶m, errmsg); } DBUG_RETURN(error); +#else + return HA_ADMIN_NOT_IMPLEMENTED; +#endif } @@ -2319,8 +2346,9 @@ bool ha_maria::check_if_incompatible_data(HA_CREATE_INFO *info, static int maria_hton_panic(handlerton *hton, ha_panic_function flag) { - ma_checkpoint_execute(CHECKPOINT_FULL, FALSE); /* can't catch error */ - return maria_panic(flag); + /* If no background checkpoints, we need to do one now */ + return ((checkpoint_interval == 0) ? + ma_checkpoint_execute(CHECKPOINT_FULL, FALSE) : 0) | maria_panic(flag); } @@ -2363,7 +2391,7 @@ static int ha_maria_init(void *p) int res; maria_hton= (handlerton *)p; maria_hton->state= SHOW_OPTION_YES; - maria_hton->db_type= DB_TYPE_MARIA; + maria_hton->db_type= DB_TYPE_UNKNOWN; maria_hton->create= maria_create_handler; maria_hton->panic= maria_hton_panic; maria_hton->commit= maria_commit; @@ -2373,6 +2401,9 @@ static int ha_maria_init(void *p) bzero(maria_log_pagecache, sizeof(*maria_log_pagecache)); maria_data_root= mysql_real_data_home; res= maria_init() || ma_control_file_create_or_open() || + (init_pagecache(maria_pagecache, + pagecache_buffer_size, pagecache_division_limit, + pagecache_age_threshold, MARIA_KEY_BLOCK_LENGTH) == 0) || (init_pagecache(maria_log_pagecache, TRANSLOG_PAGECACHE_SIZE, 0, 0, TRANSLOG_PAGE_SIZE) == 0) || @@ -2380,7 +2411,7 @@ static int ha_maria_init(void *p) MYSQL_VERSION_ID, server_id, maria_log_pagecache, TRANSLOG_DEFAULT_FLAGS) || maria_recover() || - ma_checkpoint_init(TRUE); + ma_checkpoint_init(checkpoint_interval); maria_multi_threaded= TRUE; return res; } @@ -2404,7 +2435,7 @@ static int ha_maria_init(void *p) @return The error code. The engine_data and engine_callback will be set to 0. @retval TRUE Success - @retval FALSE An error occured + @retval FALSE An error occurred */ my_bool ha_maria::register_query_cache_table(THD *thd, char *table_name, @@ -2463,8 +2494,11 @@ my_bool ha_maria::register_query_cache_table(THD *thd, char *table_name, static struct st_mysql_sys_var* system_variables[]= { MYSQL_SYSVAR(block_size), - MYSQL_SYSVAR(checkpoint_frequency), + MYSQL_SYSVAR(checkpoint_interval), MYSQL_SYSVAR(max_sort_file_size), + MYSQL_SYSVAR(pagecache_age_threshold), + MYSQL_SYSVAR(pagecache_buffer_size), + MYSQL_SYSVAR(pagecache_division_limit), MYSQL_SYSVAR(repair_threads), MYSQL_SYSVAR(sort_buffer_size), MYSQL_SYSVAR(stats_method), @@ -2473,25 +2507,27 @@ static struct st_mysql_sys_var* system_variables[]= { /** - @brief Updates the checkpoint frequency and restarts the background thread. - - Background thread has a loop which correctness depends on a constant - checkpoint frequency. So when the user wants to modify it, we stop and - restart the thread. + @brief Updates the checkpoint interval and restarts the background thread. */ -static void update_checkpoint_frequency(MYSQL_THD thd, + +static void update_checkpoint_interval(MYSQL_THD thd, struct st_mysql_sys_var *var, void *var_ptr, void *save) { - ulong new_value= (ulong)(*(long *)save), *dest= (ulong *)var_ptr; - if (new_value != *dest) /* it's actually a change */ - { - ma_checkpoint_end(); - *dest= new_value; - ma_checkpoint_init(TRUE); - } + ma_checkpoint_end(); + ma_checkpoint_init(*(ulong *)var_ptr= (ulong)(*(long *)save)); } +static SHOW_VAR status_variables[]= { + {"Maria_pagecache_blocks_not_flushed", (char*) &maria_pagecache_var.global_blocks_changed, SHOW_LONG}, + {"Maria_pagecache_blocks_unused", (char*) &maria_pagecache_var.blocks_unused, SHOW_LONG}, + {"Maria_pagecache_blocks_used", (char*) &maria_pagecache_var.blocks_used, SHOW_LONG}, + {"Maria_pagecache_read_requests", (char*) &maria_pagecache_var.global_cache_r_requests, SHOW_LONGLONG}, + {"Maria_pagecache_reads", (char*) &maria_pagecache_var.global_cache_read, SHOW_LONGLONG}, + {"Maria_pagecache_write_requests", (char*) &maria_pagecache_var.global_cache_w_requests, SHOW_LONGLONG}, + {"Maria_pagecache_writes", (char*) &maria_pagecache_var.global_cache_write, SHOW_LONGLONG}, + {NullS, NullS, SHOW_LONG} +}; struct st_mysql_storage_engine maria_storage_engine= { MYSQL_HANDLERTON_INTERFACE_VERSION }; @@ -2507,7 +2543,7 @@ mysql_declare_plugin(maria) ha_maria_init, /* Plugin Init */ NULL, /* Plugin Deinit */ 0x0100, /* 1.0 */ - NULL, /* status variables */ + status_variables, /* status variables */ system_variables, /* system variables */ NULL } diff --git a/storage/maria/ha_maria.h b/storage/maria/ha_maria.h index 21b08979bb3..78d88cf52c6 100644 --- a/storage/maria/ha_maria.h +++ b/storage/maria/ha_maria.h @@ -18,7 +18,7 @@ #pragma interface /* gcc class implementation */ #endif -/* class for the the maria handler */ +/* class for the maria handler */ #include <maria.h> diff --git a/storage/maria/ma_bitmap.c b/storage/maria/ma_bitmap.c index 261ca1a50e7..fa2b32395b1 100644 --- a/storage/maria/ma_bitmap.c +++ b/storage/maria/ma_bitmap.c @@ -1231,6 +1231,7 @@ static my_bool find_blob(MARIA_HA *info, ulong length) MARIA_BITMAP_BLOCK *first_block= 0; DBUG_ENTER("find_blob"); DBUG_PRINT("enter", ("length: %lu", length)); + LINT_INIT(first_block_pos); pages= length / full_page_size; rest_length= (uint) (length - pages * full_page_size); @@ -2141,6 +2142,9 @@ my_bool _ma_check_bitmap_data(MARIA_HA *info, case BLOB_PAGE: bits= FULL_TAIL_PAGE; break; + default: + bits= 0; /* to satisfy compiler */ + DBUG_ASSERT(0); } return (*bitmap_pattern= get_page_bits(info, &info->s->bitmap, page)) != bits; diff --git a/storage/maria/ma_blockrec.c b/storage/maria/ma_blockrec.c index 6881078d36b..c418d1e71b2 100644 --- a/storage/maria/ma_blockrec.c +++ b/storage/maria/ma_blockrec.c @@ -1610,6 +1610,7 @@ static my_bool write_full_pages(MARIA_HA *info, KEYPAGE_CHECKSUM_SIZE, (uchar) 255); DBUG_ASSERT(share->pagecache->block_size == block_size); + /** @todo RECOVERY BUG the page does not get a rec_lsn with this! */ if (pagecache_write(share->pagecache, &info->dfile, page, 0, buff, share->page_type, @@ -5952,6 +5953,7 @@ my_bool _ma_apply_undo_row_update(MARIA_HA *info, LSN undo_lsn, MARIA_RECORD_POS record_pos; ha_checksum checksum_delta; DBUG_ENTER("_ma_apply_undo_row_update"); + LINT_INIT(checksum_delta); page= page_korr(header); header+= PAGE_STORE_SIZE; diff --git a/storage/maria/ma_checkpoint.c b/storage/maria/ma_checkpoint.c index 60318e60102..76f1723b053 100644 --- a/storage/maria/ma_checkpoint.c +++ b/storage/maria/ma_checkpoint.c @@ -20,11 +20,7 @@ /* Here is the implementation of this module */ -/** - @todo RECOVERY BUG this is unreviewed code, but used in safe conditions: - ha_maria takes a checkpoint at end of recovery and one at clean shutdown, - that's all. So there never are open tables, dirty pages, transactions. -*/ +/** @todo RECOVERY BUG this is unreviewed code */ /* Summary: checkpoints are done either by a background thread (checkpoint every Nth @@ -42,25 +38,6 @@ #include "ma_loghandler_lsn.h" -/** @brief Frequency of background checkpoints, in seconds */ -ulong maria_checkpoint_frequency; -/* - Checkpoints currently happen only at ha_maria's startup (after recovery) and - at shutdown, always when there is no open tables. - Background page flushing is not used. - So, needed pagecache functions for doing this flushing are not yet pushed. -*/ -#define flush_pagecache_blocks_with_filter(A,B,C,D,E) (int)(((ulong)D) * 0) -/** - filter has to return 0, 1 or 2: 0 means "don't flush this page", 1 means - "flush it", 2 means "don't flush this page and following pages". - Will move to ma_pagecache.h -*/ -typedef int (*PAGECACHE_FILTER)(enum pagecache_page_type type, - pgcache_page_no_t page, - LSN rec_lsn, void *arg); - - /** @brief type of checkpoint currently running */ static CHECKPOINT_LEVEL checkpoint_in_progress= CHECKPOINT_NONE; /** @brief protects checkpoint_in_progress */ @@ -89,18 +66,22 @@ struct st_filter_param uint max_pages; /**< stop after flushing this number pages */ }; /**< information to determine which dirty pages should be flushed */ -static int filter_flush_data_file_medium(enum pagecache_page_type type, - pgcache_page_no_t page, - LSN rec_lsn, void *arg); -static int filter_flush_data_file_full(enum pagecache_page_type type, - pgcache_page_no_t page, - LSN rec_lsn, void *arg); -static int filter_flush_data_file_indirect(enum pagecache_page_type type, - pgcache_page_no_t page, - LSN rec_lsn, void *arg); -static int filter_flush_data_file_evenly(enum pagecache_page_type type, - pgcache_page_no_t pageno, - LSN rec_lsn, void *arg); +static enum pagecache_flush_filter_result +filter_flush_data_file_medium(enum pagecache_page_type type, + pgcache_page_no_t page, + LSN rec_lsn, void *arg); +static enum pagecache_flush_filter_result +filter_flush_data_file_full(enum pagecache_page_type type, + pgcache_page_no_t page, + LSN rec_lsn, void *arg); +static enum pagecache_flush_filter_result +filter_flush_data_file_indirect(enum pagecache_page_type type, + pgcache_page_no_t page, + LSN rec_lsn, void *arg); +static enum pagecache_flush_filter_result +filter_flush_data_file_evenly(enum pagecache_page_type type, + pgcache_page_no_t pageno, + LSN rec_lsn, void *arg); static int really_execute_checkpoint(void); pthread_handler_t ma_checkpoint_background(void *arg); static int collect_tables(LEX_STRING *str, LSN checkpoint_start_log_horizon); @@ -191,13 +172,6 @@ static int really_execute_checkpoint(void) rules, the log's lock is a mutex). "Horizon" is a lower bound of the LSN of the next log record. */ - /** - @todo RECOVERY BUG - this is an horizon, but it is used as a LSN (REDO phase may start from - there! probably log handler would refuse to read then; - Sanja proposed to make a loghandler's function which finds the LSN after - this horizon. - */ checkpoint_start_log_horizon= translog_get_horizon(); DBUG_PRINT("info",("checkpoint_start_log_horizon (%lu,0x%lx)", LSN_IN_PARTS(checkpoint_start_log_horizon))); @@ -263,7 +237,6 @@ static int really_execute_checkpoint(void) log_array[TRANSLOG_INTERNAL_PARTS + 1 + i]= record_pieces[i]; total_rec_length+= record_pieces[i].length; } - if (unlikely(translog_write_record(&lsn, LOGREC_CHECKPOINT, &dummy_transaction_object, NULL, total_rec_length, @@ -271,7 +244,6 @@ static int really_execute_checkpoint(void) log_array, NULL, NULL) || translog_flush(lsn))) goto err; - translog_lock(); /* This cannot be done as a inwrite_rec_hook of LOGREC_CHECKPOINT, because @@ -336,33 +308,39 @@ end: /** @brief Initializes the checkpoint module - @param create_background_thread If one wants the module to now create a - thread which will periodically do - checkpoints, and flush dirty pages, in the - background. + @param interval If one wants the module to create a + thread which will periodically do + checkpoints, and flush dirty pages, in the + background, it should specify a non-zero + interval in seconds. The thread will then be + created and will take checkpoints separated by + approximately 'interval' second. + + @note A checkpoint is taken only if there has been some significant + activity since the previous checkpoint. Between checkpoint N and N+1 the + thread flushes all dirty pages which were already dirty at the time of + checkpoint N. @return Operation status @retval 0 ok @retval !=0 error */ -int ma_checkpoint_init(my_bool create_background_thread) +int ma_checkpoint_init(ulong interval) { pthread_t th; int res= 0; DBUG_ENTER("ma_checkpoint_init"); checkpoint_inited= TRUE; checkpoint_thread_die= 2; /* not yet born == dead */ - /* Background thread will be enabled in a later changeset */ - create_background_thread= FALSE; - if (maria_checkpoint_frequency == 0) - create_background_thread= FALSE; if (pthread_mutex_init(&LOCK_checkpoint, MY_MUTEX_INIT_SLOW) || pthread_cond_init(&COND_checkpoint, 0)) res= 1; - else if (create_background_thread) + else if (interval > 0) { - if (!(res= pthread_create(&th, NULL, ma_checkpoint_background, NULL))) + compile_time_assert(sizeof(void *) >= sizeof(ulong)); + if (!(res= pthread_create(&th, NULL, ma_checkpoint_background, + (void *)interval))) checkpoint_thread_die= 0; /* thread lives, will have to be killed */ } DBUG_RETURN(res); @@ -417,15 +395,12 @@ void ma_checkpoint_end(void) @param pageno Page's number @param rec_lsn Page's rec_lsn @param arg filter_param - - @return Operation status - @retval 0 don't flush the page - @retval 1 flush the page */ -static int filter_flush_data_file_medium(enum pagecache_page_type type, - pgcache_page_no_t pageno, - LSN rec_lsn, void *arg) +static enum pagecache_flush_filter_result +filter_flush_data_file_medium(enum pagecache_page_type type, + pgcache_page_no_t pageno, + LSN rec_lsn, void *arg) { struct st_filter_param *param= (struct st_filter_param *)arg; return ((type == PAGECACHE_LSN_PAGE) && @@ -444,17 +419,13 @@ static int filter_flush_data_file_medium(enum pagecache_page_type type, @param pageno Page's number @param rec_lsn Page's rec_lsn @param arg filter_param - - @return Operation status - @retval 0 don't flush the page - @retval 1 flush the page */ -static int filter_flush_data_file_full(enum pagecache_page_type type, - pgcache_page_no_t pageno, - LSN rec_lsn - __attribute__ ((unused)), - void *arg) +static enum pagecache_flush_filter_result +filter_flush_data_file_full(enum pagecache_page_type type, + pgcache_page_no_t pageno, + LSN rec_lsn __attribute__ ((unused)), + void *arg) { struct st_filter_param *param= (struct st_filter_param *)arg; return (type == PAGECACHE_LSN_PAGE) || @@ -472,18 +443,14 @@ static int filter_flush_data_file_full(enum pagecache_page_type type, @param pageno Page's number @param rec_lsn Page's rec_lsn @param arg filter_param - - @return Operation status - @retval 0 don't flush the page - @retval 1 flush the page */ -static int filter_flush_data_file_indirect(enum pagecache_page_type type - __attribute__ ((unused)), - pgcache_page_no_t pageno, - LSN rec_lsn - __attribute__ ((unused)), - void *arg) +static enum pagecache_flush_filter_result +filter_flush_data_file_indirect(enum pagecache_page_type type + __attribute__ ((unused)), + pgcache_page_no_t pageno, + LSN rec_lsn __attribute__ ((unused)), + void *arg) { struct st_filter_param *param= (struct st_filter_param *)arg; return @@ -505,40 +472,33 @@ static int filter_flush_data_file_indirect(enum pagecache_page_type type @param pageno Page's number @param rec_lsn Page's rec_lsn @param arg filter_param - - @return Operation status - @retval 0 don't flush the page - @retval 1 flush the page - @retval 2 don't flush the page and following pages */ -static int filter_flush_data_file_evenly(enum pagecache_page_type type, - pgcache_page_no_t pageno - __attribute__ ((unused)), - LSN rec_lsn, void *arg) +static enum pagecache_flush_filter_result +filter_flush_data_file_evenly(enum pagecache_page_type type, + pgcache_page_no_t pageno __attribute__ ((unused)), + LSN rec_lsn, void *arg) { struct st_filter_param *param= (struct st_filter_param *)arg; if (unlikely(param->max_pages == 0)) /* all flushed already */ - return 2; + return FLUSH_FILTER_SKIP_ALL; if ((type == PAGECACHE_LSN_PAGE) && (cmp_translog_addr(rec_lsn, param->up_to_lsn) <= 0)) { param->max_pages--; - return 1; + return FLUSH_FILTER_OK; } - return 0; + return FLUSH_FILTER_SKIP_TRY_NEXT; } /** @brief Background thread which does checkpoints and flushes periodically. - Takes a checkpoint every maria_checkpoint_frequency-th second. After taking - a checkpoint, all pages dirty at the time of that checkpoint are flushed - evenly until it is time to take another checkpoint - (maria_checkpoint_frequency seconds later). This ensures that the REDO - phase starts at earliest (in LSN time) at the next-to-last checkpoint - record ("two-checkpoint rule"). + Takes a checkpoint. After this, all pages dirty at the time of that + checkpoint are flushed evenly until it is time to take another checkpoint. + This ensures that the REDO phase starts at earliest (in LSN time) at the + next-to-last checkpoint record ("two-checkpoint rule"). @note MikaelR questioned why the same thread does two different jobs, the risk could be that while a checkpoint happens no LRD flushing happens. @@ -549,34 +509,46 @@ static int filter_flush_data_file_evenly(enum pagecache_page_type type, writing 2 MB. */ -pthread_handler_t ma_checkpoint_background(void *arg __attribute__((unused))) +pthread_handler_t ma_checkpoint_background(void *arg) { /** @brief At least this of log/page bytes written between checkpoints */ const uint checkpoint_min_activity= 2*1024*1024; - uint sleeps= 0; + /* + If the interval could be changed by the user while we are in this thread, + it could be annoying: for example it could cause "case 2" to be executed + right after "case 0", thus having 'dfile' unset. So the thread cares only + about the interval's value when it started. + */ + const ulong interval= (ulong)arg; + uint sleeps; + TRANSLOG_ADDRESS log_horizon_at_last_checkpoint= LSN_IMPOSSIBLE; + ulonglong pagecache_flushes_at_last_checkpoint= 0; + uint pages_bunch_size; + struct st_filter_param filter_param; + PAGECACHE_FILE *dfile; /**< data file currently being flushed */ + PAGECACHE_FILE *kfile; /**< index file currently being flushed */ + LINT_INIT(kfile); + LINT_INIT(dfile); + LINT_INIT(pages_bunch_size); my_thread_init(); DBUG_PRINT("info",("Maria background checkpoint thread starts")); - for(;;) + DBUG_ASSERT(interval > 0); + + /* + Recovery ended with all tables closed and a checkpoint: no need to take + one immediately. + */ + sleeps= 1; + pages_to_flush_before_next_checkpoint= 0; + + for(;;) /* iterations of checkpoints and dirty page flushing */ { #if 0 /* good for testing, to do a lot of checkpoints, finds a lot of bugs */ sleeps=0; #endif - uint pages_bunch_size; - struct st_filter_param filter_param; - PAGECACHE_FILE *dfile; /**< data file currently being flushed */ - PAGECACHE_FILE *kfile; /**< index file currently being flushed */ - TRANSLOG_ADDRESS log_horizon_at_last_checkpoint= LSN_IMPOSSIBLE; - ulonglong pagecache_flushes_at_last_checkpoint= 0; struct timespec abstime; - LINT_INIT(kfile); - LINT_INIT(dfile); - /* - If the frequency could be changed by the user while we are in this loop, - it could be annoying: for example it could cause "case 2" to be executed - right after "case 0", thus having 'dfile' unset. - */ - switch((sleeps++) % maria_checkpoint_frequency) + switch((sleeps++) % interval) { case 0: /* @@ -620,22 +592,28 @@ pthread_handler_t ma_checkpoint_background(void *arg __attribute__((unused))) case 1: /* set up parameters for background page flushing */ filter_param.up_to_lsn= last_checkpoint_lsn; - pages_bunch_size= pages_to_flush_before_next_checkpoint / - maria_checkpoint_frequency; + pages_bunch_size= pages_to_flush_before_next_checkpoint / interval; dfile= dfiles; kfile= kfiles; /* fall through */ default: if (pages_bunch_size > 0) { + DBUG_PRINT("info", ("Maria background checkpoint thread: %u pages", + pages_bunch_size)); /* flush a bunch of dirty pages */ filter_param.max_pages= pages_bunch_size; filter_param.is_data_file= TRUE; while (dfile != dfiles_end) { + /* + We use FLUSH_KEEP_LAZY: if a file is already in flush, it's + smarter to move to the next file than wait for this one to be + completely flushed, which may take long. + */ int res= flush_pagecache_blocks_with_filter(maria_pagecache, - dfile, FLUSH_KEEP, + dfile, FLUSH_KEEP_LAZY, filter_flush_data_file_evenly, &filter_param); /* note that it may just be a pinned page */ @@ -651,7 +629,7 @@ pthread_handler_t ma_checkpoint_background(void *arg __attribute__((unused))) { int res= flush_pagecache_blocks_with_filter(maria_pagecache, - dfile, FLUSH_KEEP, + dfile, FLUSH_KEEP_LAZY, filter_flush_data_file_evenly, &filter_param); if (unlikely(res)) @@ -682,22 +660,8 @@ pthread_handler_t ma_checkpoint_background(void *arg __attribute__((unused))) pthread_mutex_unlock(&LOCK_checkpoint); DBUG_PRINT("info",("Maria background checkpoint thread ends")); /* - A last checkpoint, now that all tables should be closed; to have instant - recovery later. We always do it, because the test above about number of - log records or flushed pages is only approximative. For example, some log - records may have been written while ma_checkpoint_execute() above was - running, or some pages may have been flushed during this time. Thus it - could be that, while nothing has changed since that checkpoint's *end*, if - we recovered from that checkpoint we would have a non-empty dirty pages - list, REDOs to execute, and we don't want that, we want a clean shutdown - to have an empty recovery (simplifies upgrade/backups: one can just do a - clean shutdown, copy its tables to another system without copying the log - or control file and it will work because recovery will not need those). - Another reason why it's approximative is that a log record may have been - written above between ma_checkpoint_execute() and the - tranlog_get_horizon() which follows. - So, we have at least two checkpoints per start/stop of the engine, and - only two if the engine stays idle. + That's the final one, which guarantees that a clean shutdown always ends + with a checkpoint. */ ma_checkpoint_execute(CHECKPOINT_FULL, FALSE); pthread_mutex_lock(&LOCK_checkpoint); @@ -823,7 +787,7 @@ static int collect_tables(LEX_STRING *str, LSN checkpoint_start_log_horizon) struct st_filter_param filter_param; /* only possible checkpointer, so can do the read below without mutex */ filter_param.up_to_lsn= last_checkpoint_lsn; - PAGECACHE_FILTER filter; + PAGECACHE_FLUSH_FILTER filter; switch(checkpoint_in_progress) { case CHECKPOINT_MEDIUM: @@ -871,6 +835,10 @@ static int collect_tables(LEX_STRING *str, LSN checkpoint_start_log_horizon) /* No need for a mutex to read the above, only us can write this flag */ continue; } + /** + @todo We should not look at tables which didn't change since last + checkpoint. + */ DBUG_PRINT("info",("looking at table '%s'", share->open_file_name)); if (state_copy == state_copies_end) /* we have no more cached states */ { @@ -1055,6 +1023,39 @@ static int collect_tables(LEX_STRING *str, LSN checkpoint_start_log_horizon) only a little slower than CHECKPOINT_INDIRECT). */ + /* + PageCacheFlushConcurrencyBugs + Inside the page cache, calls to flush_pagecache_blocks_int() on the same + file are serialized. Examples of concurrency bugs which happened when we + didn't have this serialization: + - maria_chk_size() (via CHECK TABLE) happens concurrently with + Checkpoint: Checkpoint is flushing a page: it pins the page and is + pre-empted, maria_chk_size() wants to flush this page too so gets an + error because Checkpoint pinned this page. Such error makes + maria_chk_size() mark the table as corrupted. + - maria_close() happens concurrently with Checkpoint: + Checkpoint is flushing a page: it registers a request on the page, is + pre-empted ; maria_close() flushes this page too with FLUSH_RELEASE: + FLUSH_RELEASE will cause a free_block() which assumes the page is in the + LRU, but it is not (as Checkpoint registered a request). Crash. + - one thread is evicting a page of the file out of the LRU: it marks it + iPC_BLOCK_IN_SWITCH and is pre-empted. Then two other threads do flushes + of the same file concurrently (like above). Then one flusher sees the + page is in switch, removes it from changed_blocks[] and puts it in its + first_in_switch, so the other flusher will not see the page at all and + return too early. If it's maria_close() which returns too early, then + maria_close() may close the file descriptor, and the other flusher, and + the evicter will fail to write their page: corruption. + */ + + /* + We do NOT use FLUSH_KEEP_LAZY because we must be sure that bitmap pages + have been flushed. That's a condition of correctness of Recovery: data + pages may have been all flushed, if we write the checkpoint record + Recovery will start from after their REDOs. If bitmap page was not + flushed, as the REDOs about it will be skipped, it will wrongly not be + recovered. If bitmap pages had a rec_lsn it would be different. + */ /** @todo we ignore the error because it may be just due a pinned page; we should rather fix the function below to distinguish between diff --git a/storage/maria/ma_checkpoint.h b/storage/maria/ma_checkpoint.h index 86f3779ca7a..41e39126391 100644 --- a/storage/maria/ma_checkpoint.h +++ b/storage/maria/ma_checkpoint.h @@ -32,7 +32,7 @@ typedef enum enum_ma_checkpoint_level { } CHECKPOINT_LEVEL; C_MODE_START -int ma_checkpoint_init(my_bool create_background_thread); +int ma_checkpoint_init(ulong interval); void ma_checkpoint_end(void); int ma_checkpoint_execute(CHECKPOINT_LEVEL level, my_bool no_wait); C_MODE_END diff --git a/storage/maria/ma_delete_all.c b/storage/maria/ma_delete_all.c index 556ae5ad095..14dfe8cf7e0 100644 --- a/storage/maria/ma_delete_all.c +++ b/storage/maria/ma_delete_all.c @@ -67,6 +67,10 @@ int maria_delete_all_rows(MARIA_HA *info) log_array, log_data, NULL) || translog_flush(lsn))) goto err; + /* + If we fail in this function after this point, log and table will be + inconsistent. + */ } /* @@ -113,11 +117,6 @@ err: int save_errno=my_errno; VOID(_ma_writeinfo(info,WRITEINFO_UPDATE_KEYFILE)); info->update|=HA_STATE_WRITTEN; /* Buffer changed */ - /** - @todo RECOVERY if we come here, Recovery may later apply the REDO above, - which may be wrong. Not fixing it now, as anyway this way of deleting - rows will have to be re-examined when we have versioning. - */ allow_break(); /* Allow SIGHUP & SIGINT */ DBUG_RETURN(my_errno=save_errno); } diff --git a/storage/maria/ma_loghandler.c b/storage/maria/ma_loghandler.c index 372af0ea518..8f6a50bd93f 100644 --- a/storage/maria/ma_loghandler.c +++ b/storage/maria/ma_loghandler.c @@ -20,11 +20,10 @@ /** @file @brief Module which writes and reads to a transaction log - - @todo LOG: in functions where the log's lock is required, a - translog_assert_owner() could be added. */ +#define TRANSLOG_FILLER 0xFF + /* number of opened log files in the pagecache (should be at least 2) */ #define OPENED_FILES_NUM 3 @@ -62,6 +61,31 @@ #define COMPRESSED_LSN_MAX_STORE_SIZE (2 + LSN_STORE_SIZE) #define MAX_NUMBER_OF_LSNS_PER_RECORD 2 +#ifndef DBUG_OFF +static int translog_mutex_lock(pthread_mutex_t *M) +{ + int rc; + DBUG_PRINT("info", ("Going lock mutex 0x%lx...", (ulong)(M))); + rc= pthread_mutex_lock(M); + DBUG_PRINT("info", ("Mutex locked 0x%lx rc: %d", (ulong)(M), rc)); + return (rc); +} + +static int translog_mutex_unlock(pthread_mutex_t *M) +{ + int rc; + DBUG_PRINT("info", ("Going unlock mutex 0x%lx...", (ulong)(M))); + rc= pthread_mutex_unlock(M); + DBUG_PRINT("info", ("Mutex unlocked 0x%lx rc: %d", (ulong)(M), rc)); + return(rc); +} + +#else +#define translog_mutex_lock(M) pthread_mutex_lock(M) +#define translog_mutex_unlock(M) pthread_mutex_unlock(M) +#endif + + /* log write buffer descriptor */ struct st_translog_buffer { @@ -187,6 +211,7 @@ static struct st_translog_descriptor log_descriptor; /* Marker for end of log */ static uchar end_of_log= 0; +#define END_OF_LOG &end_of_log my_bool translog_inited= 0; @@ -211,6 +236,8 @@ static my_atomic_rwlock_t LOCK_id_to_share; static my_bool translog_page_validator(uchar *page_addr, uchar* data_ptr); +static my_bool translog_get_next_chunk(TRANSLOG_SCANNER_DATA *scanner); + /* Initialize log_record_type_descriptors @@ -735,13 +762,9 @@ static my_bool translog_write_file_header() /* file number */ int3store(page, LSN_FILE_NO(log_descriptor.horizon)); page+= 3; - /* - Here should be max lsn storing for current file (which is LSN_IPOSSIBLE): - lsn_store(page, LSN_IPOSSIBLE); + lsn_store(page, LSN_IMPOSSIBLE); page+= LSN_STORE_SIZE; - But it is zeros so we can rely on bzero() in this case - */ - bzero(page, sizeof(page_buff) - (page- page_buff)); + memset(page, TRANSLOG_FILLER, sizeof(page_buff) - (page- page_buff)); DBUG_RETURN(my_pwrite(log_descriptor.log_file_num[0], page_buff, sizeof(page_buff), 0, log_write_flags) != 0); @@ -877,7 +900,7 @@ static my_bool translog_set_lsn_for_files(uint32 from_file, uint32 to_file, translog_unlock(); /* Checks finished files if they are */ - pthread_mutex_lock(&log_descriptor.file_header_lock); + translog_mutex_lock(&log_descriptor.file_header_lock); for (file= from_file; file <= to_file; file++) { LOGHANDLER_FILE_INFO info; @@ -888,7 +911,7 @@ static my_bool translog_set_lsn_for_files(uint32 from_file, uint32 to_file, translog_max_lsn_to_header(fd, lsn))) DBUG_RETURN(1); } - pthread_mutex_unlock(&log_descriptor.file_header_lock); + translog_mutex_unlock(&log_descriptor.file_header_lock); DBUG_RETURN(0); } @@ -917,7 +940,7 @@ static void translog_mark_file_unfinished(uint32 file) DBUG_ENTER("translog_mark_file_unfinished"); DBUG_PRINT("enter", ("file: %lu", (ulong) file)); - pthread_mutex_lock(&log_descriptor.unfinished_files_lock); + translog_mutex_lock(&log_descriptor.unfinished_files_lock); if (log_descriptor.unfinished_files.elements == 0) { @@ -968,7 +991,7 @@ static void translog_mark_file_unfinished(uint32 file) place + 1, struct st_file_counter *), &fc, sizeof(struct st_file_counter)); end: - pthread_mutex_unlock(&log_descriptor.unfinished_files_lock); + translog_mutex_unlock(&log_descriptor.unfinished_files_lock); DBUG_VOID_RETURN; } @@ -989,7 +1012,7 @@ static void translog_mark_file_finished(uint32 file) LINT_INIT(fc_ptr); - pthread_mutex_lock(&log_descriptor.unfinished_files_lock); + translog_mutex_lock(&log_descriptor.unfinished_files_lock); DBUG_ASSERT(log_descriptor.unfinished_files.elements > 0); for (i= 0; @@ -1007,7 +1030,7 @@ static void translog_mark_file_finished(uint32 file) if (! --fc_ptr->counter) delete_dynamic_element(&log_descriptor.unfinished_files, i); - pthread_mutex_unlock(&log_descriptor.unfinished_files_lock); + translog_mutex_unlock(&log_descriptor.unfinished_files_lock); DBUG_VOID_RETURN; } @@ -1030,7 +1053,7 @@ LSN translog_get_file_max_lsn_stored(uint32 file) DBUG_PRINT("enter", ("file: %lu", (ulong)file)); DBUG_ASSERT(translog_inited == 1); - pthread_mutex_lock(&log_descriptor.unfinished_files_lock); + translog_mutex_lock(&log_descriptor.unfinished_files_lock); /* find file with minimum file number "in progress" */ if (log_descriptor.unfinished_files.elements > 0) @@ -1040,7 +1063,7 @@ LSN translog_get_file_max_lsn_stored(uint32 file) 0, struct st_file_counter *); limit= fc_ptr->file; /* minimal file number "in progress" */ } - pthread_mutex_unlock(&log_descriptor.unfinished_files_lock); + translog_mutex_unlock(&log_descriptor.unfinished_files_lock); /* if there is no "in progress file" then unfinished file is in progress @@ -1093,7 +1116,7 @@ static my_bool translog_buffer_init(struct st_translog_buffer *buffer) buffer->file= -1; buffer->overlay= 0; /* IO cache for current log */ - bzero(buffer->buffer, TRANSLOG_WRITE_BUFFER); + memset(buffer->buffer, TRANSLOG_FILLER, TRANSLOG_WRITE_BUFFER); /* Buffer size */ buffer->size= 0; /* cond of thread which is waiting for buffer filling */ @@ -1203,8 +1226,8 @@ static my_bool translog_buffer_lock(struct st_translog_buffer *buffer) ("Lock buffer #%u: (0x%lx) mutex: 0x%lx", (uint) buffer->buffer_no, (ulong) buffer, (ulong) &buffer->mutex)); - res= (pthread_mutex_lock(&buffer->mutex) != 0); - DBUG_RETURN(res); + res= (translog_mutex_lock(&buffer->mutex) != 0); + DBUG_RETURN(test(res)); } #else #define translog_buffer_lock(B) \ @@ -1234,7 +1257,7 @@ static my_bool translog_buffer_unlock(struct st_translog_buffer *buffer) (uint) buffer->buffer_no, (ulong) buffer, (ulong) &buffer->mutex)); - res= (pthread_mutex_unlock(&buffer->mutex) != 0); + res= (translog_mutex_unlock(&buffer->mutex) != 0); DBUG_PRINT("enter", ("Unlocked buffer... #%u: 0x%lx mutex: 0x%lx", (uint) buffer->buffer_no, (ulong) buffer, (ulong) &buffer->mutex)); @@ -1275,7 +1298,9 @@ static void translog_new_page_header(TRANSLOG_ADDRESS *horizon, /* File number */ int3store(ptr, LSN_FILE_NO(*horizon)); ptr+= 3; - *(ptr++)= (uchar) log_descriptor.flags; + DBUG_ASSERT(TRANSLOG_PAGE_FLAGS == (ptr - cursor->ptr)); + cursor->ptr[TRANSLOG_PAGE_FLAGS]= (uchar) log_descriptor.flags; + ptr++; if (log_descriptor.flags & TRANSLOG_PAGE_CRC) { #ifndef DBUG_OFF @@ -1406,7 +1431,7 @@ static void translog_finish_page(TRANSLOG_ADDRESS *horizon, struct st_buffer_cursor *cursor) { uint16 left= TRANSLOG_PAGE_SIZE - cursor->current_page_fill; - uchar *page= cursor->ptr -cursor->current_page_fill; + uchar *page= cursor->ptr - cursor->current_page_fill; DBUG_ENTER("translog_finish_page"); DBUG_PRINT("enter", ("Buffer: #%u 0x%lx " "Buffer addr: (%lu,0x%lx) " @@ -1433,7 +1458,7 @@ static void translog_finish_page(TRANSLOG_ADDRESS *horizon, if (left != 0) { DBUG_PRINT("info", ("left: %u", (uint) left)); - bzero(cursor->ptr, left); + memset(cursor->ptr, TRANSLOG_FILLER, left); cursor->ptr +=left; (*horizon)+= left; /* offset increasing */ if (!cursor->chaser) @@ -1447,14 +1472,19 @@ static void translog_finish_page(TRANSLOG_ADDRESS *horizon, (ulong) (cursor->ptr - cursor->buffer->buffer))); DBUG_EXECUTE("info", translog_check_cursor(cursor);); } - if (page[TRANSLOG_PAGE_FLAGS] & TRANSLOG_SECTOR_PROTECTION) + /* + When we are finishing the page other thread might not finish the page + header yet so we have to read log_descriptor.flags but not the flags + from the page + */ + if (log_descriptor.flags & TRANSLOG_SECTOR_PROTECTION) { translog_put_sector_protection(page, cursor); DBUG_PRINT("info", ("drop write_counter")); cursor->write_counter= 0; cursor->previous_offset= 0; } - if (page[TRANSLOG_PAGE_FLAGS] & TRANSLOG_PAGE_CRC) + if (log_descriptor.flags & TRANSLOG_PAGE_CRC) { uint32 crc= translog_crc(page + log_descriptor.page_overhead, TRANSLOG_PAGE_SIZE - @@ -1676,7 +1706,10 @@ static my_bool translog_buffer_next(TRANSLOG_ADDRESS *horizon, if (chasing) translog_cursor_init(cursor, new_buffer, new_buffer_no); else + { + translog_lock_assert_owner(); translog_start_buffer(new_buffer, cursor, new_buffer_no); + } log_descriptor.buffers[old_buffer_no].next_buffer_offset= new_buffer->offset; translog_new_page_header(horizon, cursor); DBUG_RETURN(0); @@ -1697,7 +1730,7 @@ static my_bool translog_buffer_next(TRANSLOG_ADDRESS *horizon, static void translog_set_sent_to_file(LSN lsn, TRANSLOG_ADDRESS in_buffers) { DBUG_ENTER("translog_set_sent_to_file"); - pthread_mutex_lock(&log_descriptor.sent_to_file_lock); + translog_mutex_lock(&log_descriptor.sent_to_file_lock); DBUG_PRINT("enter", ("lsn: (%lu,0x%lx) in_buffers: (%lu,0x%lx) " "in_buffers_only: (%lu,0x%lx)", LSN_IN_PARTS(lsn), @@ -1711,7 +1744,7 @@ static void translog_set_sent_to_file(LSN lsn, TRANSLOG_ADDRESS in_buffers) log_descriptor.in_buffers_only= in_buffers; DBUG_PRINT("info", ("set new in_buffers_only")); } - pthread_mutex_unlock(&log_descriptor.sent_to_file_lock); + translog_mutex_unlock(&log_descriptor.sent_to_file_lock); DBUG_VOID_RETURN; } @@ -1728,7 +1761,7 @@ static void translog_set_sent_to_file(LSN lsn, TRANSLOG_ADDRESS in_buffers) static void translog_set_only_in_buffers(TRANSLOG_ADDRESS in_buffers) { DBUG_ENTER("translog_set_only_in_buffers"); - pthread_mutex_lock(&log_descriptor.sent_to_file_lock); + translog_mutex_lock(&log_descriptor.sent_to_file_lock); DBUG_PRINT("enter", ("in_buffers: (%lu,0x%lx) " "in_buffers_only: (%lu,0x%lx)", LSN_IN_PARTS(in_buffers), @@ -1739,7 +1772,7 @@ static void translog_set_only_in_buffers(TRANSLOG_ADDRESS in_buffers) log_descriptor.in_buffers_only= in_buffers; DBUG_PRINT("info", ("set new in_buffers_only")); } - pthread_mutex_unlock(&log_descriptor.sent_to_file_lock); + translog_mutex_unlock(&log_descriptor.sent_to_file_lock); DBUG_VOID_RETURN; } @@ -1758,9 +1791,9 @@ static TRANSLOG_ADDRESS translog_only_in_buffers() { register TRANSLOG_ADDRESS addr; DBUG_ENTER("translog_only_in_buffers"); - pthread_mutex_lock(&log_descriptor.sent_to_file_lock); + translog_mutex_lock(&log_descriptor.sent_to_file_lock); addr= log_descriptor.in_buffers_only; - pthread_mutex_unlock(&log_descriptor.sent_to_file_lock); + translog_mutex_unlock(&log_descriptor.sent_to_file_lock); DBUG_RETURN(addr); } @@ -1779,9 +1812,9 @@ static LSN translog_get_sent_to_file() { register LSN lsn; DBUG_ENTER("translog_get_sent_to_file"); - pthread_mutex_lock(&log_descriptor.sent_to_file_lock); + translog_mutex_lock(&log_descriptor.sent_to_file_lock); lsn= log_descriptor.sent_to_file; - pthread_mutex_unlock(&log_descriptor.sent_to_file_lock); + translog_mutex_unlock(&log_descriptor.sent_to_file_lock); DBUG_RETURN(lsn); } @@ -2004,6 +2037,7 @@ static my_bool translog_buffer_flush(struct st_translog_buffer *buffer) buffer->file, LSN_IN_PARTS(buffer->offset), (ulong) buffer->size)); + translog_buffer_lock_assert_owner(buffer); DBUG_ASSERT(buffer->file != -1); @@ -2089,7 +2123,7 @@ static my_bool translog_recover_page_up_to_sector(uchar *page, uint16 offset) DBUG_PRINT("enter", ("offset: %u first chunk: %u", (uint) offset, (uint) chunk_offset)); - while (page[chunk_offset] != '\0' && chunk_offset < offset) + while (page[chunk_offset] != TRANSLOG_FILLER && chunk_offset < offset) { uint16 chunk_length; if ((chunk_length= @@ -2112,7 +2146,7 @@ static my_bool translog_recover_page_up_to_sector(uchar *page, uint16 offset) valid_chunk_end= chunk_offset; /* end of trusted area - sector parsing */ - while (page[chunk_offset] != '\0') + while (page[chunk_offset] != TRANSLOG_FILLER) { uint16 chunk_length; if ((chunk_length= @@ -2130,7 +2164,8 @@ static my_bool translog_recover_page_up_to_sector(uchar *page, uint16 offset) } DBUG_PRINT("info", ("valid chunk end offset: %u", (uint) valid_chunk_end)); - bzero(page + valid_chunk_end, TRANSLOG_PAGE_SIZE - valid_chunk_end); + memset(page + valid_chunk_end, TRANSLOG_FILLER, + TRANSLOG_PAGE_SIZE - valid_chunk_end); DBUG_RETURN(0); } @@ -2482,6 +2517,7 @@ static uchar *translog_get_page(TRANSLOG_VALIDATOR_DATA *data, uchar *buffer, DBUG_RETURN(buffer); } + /** @brief free direct log page link @@ -2501,6 +2537,7 @@ static void translog_free_link(PAGECACHE_BLOCK_LINK *direct_link) DBUG_VOID_RETURN; } + /* Finds last page of the given log file @@ -2631,6 +2668,73 @@ static uint16 translog_get_chunk_header_length(uchar *page, uint16 offset) } +/** + @brief Truncate the log to the given address + + @param addr new horizon + + @retval 0 OK + @retval 1 Error +*/ + +static my_bool translog_truncate_log(TRANSLOG_ADDRESS addr) +{ + uchar *page; + TRANSLOG_ADDRESS current_page; + uint32 next_page_offset, page_rest; + uint32 i; + File fd; + TRANSLOG_VALIDATOR_DATA data; + char path[FN_REFLEN]; + uchar page_buff[TRANSLOG_PAGE_SIZE]; + DBUG_ENTER("translog_truncate_log"); + /* TODO: write warning to the client */ + DBUG_PRINT("warning", ("removing all records from (%lx,0x%lx) " + "till (%lx,0x%lx)", + LSN_IN_PARTS(addr), + LSN_IN_PARTS(log_descriptor.horizon))); + DBUG_ASSERT(cmp_translog_addr(addr, log_descriptor.horizon) < 0); + /* remove files between the address and horizon */ + for (i= LSN_FILE_NO(addr) + 1; i <= LSN_FILE_NO(log_descriptor.horizon); i++) + if (my_delete(translog_filename_by_fileno(i, path), MYF(MY_WME))) + { + translog_unlock(); + DBUG_RETURN(1); + } + + /* truncate the last file up to the last page */ + next_page_offset= LSN_OFFSET(addr); + next_page_offset= (next_page_offset - + ((next_page_offset - 1) % TRANSLOG_PAGE_SIZE + 1) + + TRANSLOG_PAGE_SIZE); + page_rest= next_page_offset - LSN_OFFSET(addr); + memset(page_buff, TRANSLOG_FILLER, page_rest); + if ((fd= open_logfile_by_number_no_cache(LSN_FILE_NO(addr))) < 0 || + my_chsize(fd, next_page_offset, TRANSLOG_FILLER, MYF(MY_WME)) || + (page_rest && my_pwrite(fd, page_buff, page_rest, LSN_OFFSET(addr), + log_write_flags)) || + my_sync(fd, MYF(MY_WME)) || + my_close(fd, MYF(MY_WME))) + DBUG_RETURN(1); + /* fix the horizon */ + log_descriptor.horizon= addr; + /* fix the buffer data */ + current_page= MAKE_LSN(LSN_FILE_NO(addr), (next_page_offset - + TRANSLOG_PAGE_SIZE)); + data.addr= ¤t_page; + if ((page= translog_get_page(&data, log_descriptor.buffers->buffer, NULL)) == + NULL) + DBUG_RETURN(1); + if (page != log_descriptor.buffers->buffer) + memcpy(log_descriptor.buffers->buffer, page, TRANSLOG_PAGE_SIZE); + log_descriptor.bc.buffer->offset= current_page; + log_descriptor.bc.buffer->size= LSN_OFFSET(addr) - LSN_OFFSET(current_page); + log_descriptor.bc.ptr= + log_descriptor.buffers->buffer + log_descriptor.bc.buffer->size; + log_descriptor.bc.current_page_fill= log_descriptor.bc.buffer->size; + DBUG_RETURN(0); +} + /* Initialize transaction log @@ -2888,10 +2992,11 @@ my_bool translog_init(const char *directory, log_descriptor.horizon= last_valid_page; translog_start_buffer(log_descriptor.buffers, &log_descriptor.bc, 0); /* - Free space if filled with 0 and first uchar of - real chunk can't be 0 + Free space if filled with TRANSLOG_FILLER and first uchar of + real chunk can't be TRANSLOG_FILLER */ - while (chunk_offset < TRANSLOG_PAGE_SIZE && page[chunk_offset] != '\0') + while (chunk_offset < TRANSLOG_PAGE_SIZE && + page[chunk_offset] != TRANSLOG_FILLER) { uint16 chunk_length; if ((chunk_length= @@ -2985,7 +3090,143 @@ my_bool translog_init(const char *directory, if (unlikely(!id_to_share)) DBUG_RETURN(1); id_to_share--; /* min id is 1 */ + translog_inited= 1; + /* Check the last LSN record integrity */ + if (logs_found) + { + TRANSLOG_SCANNER_DATA scanner; + TRANSLOG_ADDRESS page_addr; + LSN last_lsn= LSN_IMPOSSIBLE; + /* + take very last page address and try to find LSN record on it + if it fail take address of previous page and so on + */ + page_addr= (log_descriptor.horizon - + ((log_descriptor.horizon - 1) % TRANSLOG_PAGE_SIZE + 1)); + if (translog_init_scanner(page_addr, 1, &scanner, 1)) + DBUG_RETURN(1); + scanner.page_offset= page_overhead[scanner.page[TRANSLOG_PAGE_FLAGS]]; + for (;;) + { + uint chunk_type; + chunk_type= scanner.page[scanner.page_offset] & TRANSLOG_CHUNK_TYPE; + DBUG_PRINT("info", ("type: %x byte: %x", (uint) chunk_type, + (uint) scanner.page[scanner.page_offset])); + while (chunk_type != TRANSLOG_CHUNK_LSN && + chunk_type != TRANSLOG_CHUNK_FIXED && + scanner.page != END_OF_LOG && + scanner.page[scanner.page_offset] != TRANSLOG_FILLER && + scanner.page_addr == page_addr) + { + if (translog_get_next_chunk(&scanner)) + { + translog_destroy_scanner(&scanner); + DBUG_RETURN(1); + } + if (scanner.page != END_OF_LOG) + { + chunk_type= scanner.page[scanner.page_offset] & TRANSLOG_CHUNK_TYPE; + DBUG_PRINT("info", ("type: %x byte: %x", (uint) chunk_type, + (uint) scanner.page[scanner.page_offset])); + } + } + if (chunk_type == TRANSLOG_CHUNK_LSN || + chunk_type == TRANSLOG_CHUNK_FIXED) + { + last_lsn= scanner.page_addr + scanner.page_offset; + if (translog_get_next_chunk(&scanner)) + { + translog_destroy_scanner(&scanner); + DBUG_RETURN(1); + } + if (scanner.page == END_OF_LOG) + break; /* it was the last record */ + chunk_type= scanner.page[scanner.page_offset] & TRANSLOG_CHUNK_TYPE; + DBUG_PRINT("info", ("type: %x byte: %x", (uint) chunk_type, + (uint) scanner.page[scanner.page_offset])); + continue; /* try to find other record on this page */ + } + + if (last_lsn != LSN_IMPOSSIBLE) + break; /* there is no more records on the page */ + + /* We have to make step back */ + if (unlikely(LSN_OFFSET(page_addr) == TRANSLOG_PAGE_SIZE)) + { + uint32 file_no= LSN_FILE_NO(page_addr); + bool last_page_ok; + /* it is beginning of the current file */ + if (unlikely(file_no == 1)) + { + /* + It is beginning of the log => there is no LSNs in the log => + There is no harm in leaving it "as-is". + */ + DBUG_RETURN(0); + } + file_no--; + page_addr= MAKE_LSN(file_no, TRANSLOG_PAGE_SIZE); + translog_get_last_page_addr(&page_addr, &last_page_ok); + /* page should be OK as it is not the last file */ + DBUG_ASSERT(last_page_ok); + } + else + { + page_addr-= TRANSLOG_PAGE_SIZE; + } + translog_destroy_scanner(&scanner); + if (translog_init_scanner(page_addr, 1, &scanner, 1)) + DBUG_RETURN(1); + scanner.page_offset= page_overhead[scanner.page[TRANSLOG_PAGE_FLAGS]]; + } + translog_destroy_scanner(&scanner); + + /* Now scanner points to the last LSN chunk, lets check it */ + { + TRANSLOG_HEADER_BUFFER rec; + translog_size_t rec_len; + int len; + uchar buffer[1]; + DBUG_PRINT("info", ("going to check the last found record (%lu,0x%lx)", + LSN_IN_PARTS(last_lsn))); + + len= + translog_read_record_header(last_lsn, &rec); + if (unlikely (len == RECHEADER_READ_ERROR || + len == RECHEADER_READ_EOF)) + { + DBUG_PRINT("error", ("unexpected end of log or record during " + "reading record header: (%lu,0x%lx) len: %d", + LSN_IN_PARTS(last_lsn), len)); + if (translog_truncate_log(last_lsn)) + DBUG_RETURN(1); + } + else + { + DBUG_ASSERT(last_lsn == rec.lsn); + if (likely(rec.record_length != 0)) + { + /* + Reading the last byte of record will trigger scanning all + record chunks for now + */ + rec_len= translog_read_record(rec.lsn, rec.record_length - 1, 1, + buffer, NULL); + if (rec_len != 1) + { + DBUG_PRINT("error", ("unexpected end of log or record during " + "reading record body: (%lu,0x%lx) len: %d", + LSN_IN_PARTS(rec.lsn), + len)); + if (translog_truncate_log(last_lsn)) + DBUG_RETURN(1); + } + } + } + } + } + DBUG_RETURN(0); } @@ -3017,7 +3258,9 @@ static void translog_buffer_destroy(struct st_translog_buffer *buffer) We ignore errors here, because we can't do something about it (it is shutting down) */ + translog_buffer_lock(buffer); translog_buffer_flush(buffer); + translog_buffer_unlock(buffer); } DBUG_PRINT("info", ("Destroy mutex: 0x%lx", (ulong) &buffer->mutex)); pthread_mutex_destroy(&buffer->mutex); @@ -5245,7 +5488,6 @@ my_bool translog_init_scanner(LSN lsn, DBUG_ENTER("translog_init_scanner"); DBUG_PRINT("enter", ("Scanner: 0x%lx LSN: (0x%lu,0x%lx)", (ulong) scanner, LSN_IN_PARTS(lsn))); - DBUG_ASSERT(LSN_OFFSET(lsn) % TRANSLOG_PAGE_SIZE != 0); DBUG_ASSERT(translog_inited == 1); data.addr= &scanner->page_addr; @@ -5349,7 +5591,7 @@ static my_bool translog_scanner_eop(TRANSLOG_SCANNER_DATA *scanner) { DBUG_ENTER("translog_scanner_eop"); DBUG_RETURN(scanner->page_offset >= TRANSLOG_PAGE_SIZE || - scanner->page[scanner->page_offset] == 0); + scanner->page[scanner->page_offset] == TRANSLOG_FILLER); } @@ -5403,14 +5645,16 @@ translog_get_next_chunk(TRANSLOG_SCANNER_DATA *scanner) uint16 len; DBUG_ENTER("translog_get_next_chunk"); - if ((len= translog_get_total_chunk_length(scanner->page, - scanner->page_offset)) == 0) + if (translog_scanner_eop(scanner)) + len= TRANSLOG_PAGE_SIZE - scanner->page_offset; + else if ((len= translog_get_total_chunk_length(scanner->page, + scanner->page_offset)) == 0) DBUG_RETURN(1); scanner->page_offset+= len; if (translog_scanner_eol(scanner)) { - scanner->page= &end_of_log; + scanner->page= END_OF_LOG; scanner->page_offset= 0; DBUG_RETURN(0); } @@ -5443,11 +5687,11 @@ translog_get_next_chunk(TRANSLOG_SCANNER_DATA *scanner) scanner->page_offset= translog_get_first_chunk_offset(scanner->page); if (translog_scanner_eol(scanner)) { - scanner->page= &end_of_log; + scanner->page= END_OF_LOG; scanner->page_offset= 0; DBUG_RETURN(0); } - DBUG_ASSERT(scanner->page[scanner->page_offset]); + DBUG_ASSERT(scanner->page[scanner->page_offset] != TRANSLOG_FILLER); } DBUG_RETURN(0); } @@ -5465,6 +5709,7 @@ translog_get_next_chunk(TRANSLOG_SCANNER_DATA *scanner) @return Length of header or operation status @retval RECHEADER_READ_ERROR error + @retval RECHEADER_READ_EOF End of the log reached during the read @retval # number of bytes in TRANSLOG_HEADER_BUFFER::header where stored decoded part of the header @@ -5571,7 +5816,17 @@ translog_variable_length_header(uchar *page, translog_size_t page_offset, DBUG_RETURN(RECHEADER_READ_ERROR); } if (translog_get_next_chunk(scanner)) + { + if (scanner == &internal_scanner) + translog_destroy_scanner(scanner); DBUG_RETURN(RECHEADER_READ_ERROR); + } + if (scanner->page == END_OF_LOG) + { + if (scanner == &internal_scanner) + translog_destroy_scanner(scanner); + DBUG_RETURN(RECHEADER_READ_EOF); + } page= scanner->page; page_offset= scanner->page_offset; src= page + page_offset + header_to_skip; @@ -5643,7 +5898,7 @@ translog_variable_length_header(uchar *page, translog_size_t page_offset, @return Length of header or operation status @retval RECHEADER_READ_ERROR error @retval # number of bytes in - TRANSLOG_HEADER_BUFFER::header where + TRANSLOG_HEADER_BUFFER::header where stored decoded part of the header */ @@ -5813,14 +6068,26 @@ int translog_read_next_record_header(TRANSLOG_SCANNER_DATA *scanner, { if (translog_get_next_chunk(scanner)) DBUG_RETURN(RECHEADER_READ_ERROR); + if (scanner->page == END_OF_LOG) + { + DBUG_PRINT("info", ("End of file from the scanner")); + /* Last record was read */ + buff->lsn= LSN_IMPOSSIBLE; + DBUG_RETURN(RECHEADER_READ_EOF); + } chunk_type= scanner->page[scanner->page_offset] & TRANSLOG_CHUNK_TYPE; - DBUG_PRINT("info", ("type: %x byte: %x", (uint) chunk_type, + DBUG_PRINT("info", ("Page: (%lu,0x%lx) offset: %lu type: %x byte: %x", + LSN_IN_PARTS(scanner->page_addr), + (ulong) scanner->page_offset, + (uint) chunk_type, (uint) scanner->page[scanner->page_offset])); - } while (chunk_type != TRANSLOG_CHUNK_LSN && chunk_type != - TRANSLOG_CHUNK_FIXED && scanner->page[scanner->page_offset] != 0); + } while (chunk_type != TRANSLOG_CHUNK_LSN && + chunk_type != TRANSLOG_CHUNK_FIXED && + scanner->page[scanner->page_offset] != TRANSLOG_FILLER); - if (scanner->page[scanner->page_offset] == 0) + if (scanner->page[scanner->page_offset] == TRANSLOG_FILLER) { + DBUG_PRINT("info", ("End of file")); /* Last record was read */ buff->lsn= LSN_IMPOSSIBLE; /* Return 'end of log' marker */ @@ -5876,6 +6143,14 @@ static my_bool translog_record_read_next_chunk(struct st_translog_reader_data data->current_chunk++; if (translog_get_next_chunk(&data->scanner)) DBUG_RETURN(1); + if (data->scanner.page == END_OF_LOG) + { + /* + Actually it should not happened, but we wont quite nice in case of + cut log + */ + DBUG_RETURN(1); + } } type= data->scanner.page[data->scanner.page_offset] & TRANSLOG_CHUNK_TYPE; @@ -6137,7 +6412,7 @@ static void translog_force_current_buffer_to_finish() new_buff_beginning-= log_descriptor.bc.current_page_fill; current_page_fill= log_descriptor.bc.current_page_fill; - bzero(log_descriptor.bc.ptr, left); + memset(log_descriptor.bc.ptr, TRANSLOG_FILLER, left); log_descriptor.bc.buffer->size+= left; DBUG_PRINT("info", ("Finish Page buffer #%u: 0x%lx " "Size: %lu", @@ -6186,8 +6461,7 @@ static void translog_force_current_buffer_to_finish() */ translog_wait_for_writers(old_buffer); - - if (data[TRANSLOG_PAGE_FLAGS] & TRANSLOG_SECTOR_PROTECTION) + if (log_descriptor.flags & TRANSLOG_SECTOR_PROTECTION) { translog_put_sector_protection(data, &log_descriptor.bc); if (left) @@ -6203,7 +6477,7 @@ static void translog_force_current_buffer_to_finish() } } - if (data[TRANSLOG_PAGE_FLAGS] & TRANSLOG_PAGE_CRC) + if (log_descriptor.flags & TRANSLOG_PAGE_CRC) { uint32 crc= translog_crc(data + log_descriptor.page_overhead, TRANSLOG_PAGE_SIZE - @@ -6269,7 +6543,7 @@ my_bool translog_flush(LSN lsn) DBUG_PRINT("enter", ("Flush up to LSN: (%lu,0x%lx)", LSN_IN_PARTS(lsn))); DBUG_ASSERT(translog_inited == 1); - pthread_mutex_lock(&log_descriptor.log_flush_lock); + translog_mutex_lock(&log_descriptor.log_flush_lock); translog_lock(); old_flushed= log_descriptor.flushed; for (;;) @@ -6352,7 +6626,7 @@ my_bool translog_flush(LSN lsn) /** @todo LOG decide if syncing of directory is needed */ rc|= my_sync(log_descriptor.directory_fd, MYF(MY_WME | MY_IGNORE_BADFD)); out: - pthread_mutex_unlock(&log_descriptor.log_flush_lock); + translog_mutex_unlock(&log_descriptor.log_flush_lock); DBUG_RETURN(rc); } @@ -6383,7 +6657,7 @@ int translog_assign_id_to_share(MARIA_HA *tbl_info, TRN *trn) */ DBUG_ASSERT(share->data_file_type == BLOCK_RECORD); /* re-check under mutex to avoid having 2 ids for the same share */ - pthread_mutex_lock(&share->intern_lock); + translog_mutex_lock(&share->intern_lock); if (likely(share->id == 0)) { /* Inspired by set_short_trid() of trnman.c */ @@ -6430,7 +6704,7 @@ int translog_assign_id_to_share(MARIA_HA *tbl_info, TRN *trn) log_array, log_data, NULL))) return 1; } - pthread_mutex_unlock(&share->intern_lock); + translog_mutex_unlock(&share->intern_lock); return 0; } @@ -6507,14 +6781,14 @@ static uint32 translog_first_file(TRANSLOG_ADDRESS horizon, int is_protected) uint min_file= 1, max_file; DBUG_ENTER("translog_first_file"); if (!is_protected) - pthread_mutex_lock(&log_descriptor.purger_lock); + translog_mutex_lock(&log_descriptor.purger_lock); if (log_descriptor.min_file_number && translog_is_file(log_descriptor.min_file_number)) { DBUG_PRINT("info", ("cached %lu", (ulong) log_descriptor.min_file_number)); if (!is_protected) - pthread_mutex_unlock(&log_descriptor.purger_lock); + translog_mutex_unlock(&log_descriptor.purger_lock); DBUG_RETURN(log_descriptor.min_file_number); } @@ -6541,7 +6815,7 @@ static uint32 translog_first_file(TRANSLOG_ADDRESS horizon, int is_protected) } log_descriptor.min_file_number= max_file; if (!is_protected) - pthread_mutex_unlock(&log_descriptor.purger_lock); + translog_mutex_unlock(&log_descriptor.purger_lock); DBUG_RETURN(max_file); } @@ -6571,29 +6845,73 @@ LSN translog_next_LSN(TRANSLOG_ADDRESS addr, TRANSLOG_ADDRESS horizon) DBUG_RETURN(LSN_IMPOSSIBLE); translog_init_scanner(addr, 0, &scanner, 1); + /* + addr can point not to a chunk beginning but page end so next + page beginning. + */ + if (addr % TRANSLOG_PAGE_SIZE == 0) + { + /* + We are emulating the page end which cased such horizon value to + trigger translog_scanner_eop(). + + We can't just increase addr on page header overhead because it + can be file end so we allow translog_get_next_chunk() to skip + to the next page in correct way + */ + scanner.page_addr-= TRANSLOG_PAGE_SIZE; + scanner.page_offset= TRANSLOG_PAGE_SIZE; +#ifndef DBUG_OFF + scanner.page= NULL; /* prevent using incorrect page content */ +#endif + } + /* addr can point not to a chunk beginning but to a page end */ + if (translog_scanner_eop(&scanner)) + { + if (translog_get_next_chunk(&scanner)) + { + result= LSN_ERROR; + goto out; + } + if (scanner.page == END_OF_LOG) + { + result= LSN_IMPOSSIBLE; + goto out; + } + } chunk_type= scanner.page[scanner.page_offset] & TRANSLOG_CHUNK_TYPE; DBUG_PRINT("info", ("type: %x byte: %x", (uint) chunk_type, (uint) scanner.page[scanner.page_offset])); while (chunk_type != TRANSLOG_CHUNK_LSN && chunk_type != TRANSLOG_CHUNK_FIXED && - scanner.page[scanner.page_offset] != 0) + scanner.page[scanner.page_offset] != TRANSLOG_FILLER) { if (translog_get_next_chunk(&scanner)) - DBUG_RETURN(LSN_ERROR); + { + result= LSN_ERROR; + goto out; + } + if (scanner.page == END_OF_LOG) + { + result= LSN_IMPOSSIBLE; + goto out; + } chunk_type= scanner.page[scanner.page_offset] & TRANSLOG_CHUNK_TYPE; DBUG_PRINT("info", ("type: %x byte: %x", (uint) chunk_type, (uint) scanner.page[scanner.page_offset])); } - if (scanner.page[scanner.page_offset] == 0) + if (scanner.page[scanner.page_offset] == TRANSLOG_FILLER) result= LSN_IMPOSSIBLE; /* reached page filler */ else result= scanner.page_addr + scanner.page_offset; +out: translog_destroy_scanner(&scanner); DBUG_RETURN(result); } + /** @brief returns the LSN of the first record starting in this log @@ -6687,7 +7005,7 @@ my_bool translog_purge(TRANSLOG_ADDRESS low) DBUG_PRINT("enter", ("low: (%lu,0x%lx)", LSN_IN_PARTS(low))); DBUG_ASSERT(translog_inited == 1); - pthread_mutex_lock(&log_descriptor.purger_lock); + translog_mutex_lock(&log_descriptor.purger_lock); if (LSN_FILE_NO(log_descriptor.last_lsn_checked) < last_need_file) { uint32 i; @@ -6715,6 +7033,6 @@ my_bool translog_purge(TRANSLOG_ADDRESS low) } } - pthread_mutex_unlock(&log_descriptor.purger_lock); + translog_mutex_unlock(&log_descriptor.purger_lock); DBUG_RETURN(rc); } diff --git a/storage/maria/ma_loghandler_lsn.h b/storage/maria/ma_loghandler_lsn.h index e019be16fd2..34edb1e294f 100644 --- a/storage/maria/ma_loghandler_lsn.h +++ b/storage/maria/ma_loghandler_lsn.h @@ -83,9 +83,9 @@ typedef LSN LSN_WITH_FLAGS; #define FILENO_IMPOSSIBLE 0 /**< log file's numbering starts at 1 */ #define LOG_OFFSET_IMPOSSIBLE 0 /**< log always has a header */ -#define LSN_IMPOSSIBLE 0 +#define LSN_IMPOSSIBLE ((LSN)0) /* following LSN also is impossible */ -#define LSN_ERROR 1 +#define LSN_ERROR ((LSN)1) /** @brief some impossible LSN serve as markers */ #define LSN_REPAIRED_BY_MARIA_CHK ((LSN)2) diff --git a/storage/maria/ma_pagecache.c b/storage/maria/ma_pagecache.c index f414b35c625..842acd2f0e8 100755 --- a/storage/maria/ma_pagecache.c +++ b/storage/maria/ma_pagecache.c @@ -115,11 +115,6 @@ /* TODO: put it to my_static.c */ my_bool my_disable_flush_pagecache_blocks= 0; -/** - when flushing pages of a file, it can happen that we take some dirty blocks - out of changed_blocks[]; Checkpoint must not run at this moment. -*/ -uint changed_blocks_is_incomplete= 0; #define STRUCT_PTR(TYPE, MEMBER, a) \ (TYPE *) ((char *) (a) - offsetof(TYPE, MEMBER)) @@ -320,6 +315,22 @@ struct st_pagecache_block_link LSN rec_lsn; }; +/** @brief information describing a run of flush_pagecache_blocks_int() */ +struct st_file_in_flush +{ + PAGECACHE_FILE file; + /** + @brief threads waiting for the thread currently flushing this file to be + done + */ + WQUEUE flush_queue; + /** + @brief if the thread currently flushing the file has a non-empty + first_in_switch list. + */ + my_bool first_in_switch; +}; + #ifndef DBUG_OFF /* debug checks */ @@ -678,9 +689,14 @@ ulong init_pagecache(PAGECACHE *pagecache, size_t use_mem, pagecache->disk_blocks= -1; if (! pagecache->inited) { + if (pthread_mutex_init(&pagecache->cache_lock, MY_MUTEX_INIT_FAST) || + hash_init(&pagecache->files_in_flush, &my_charset_bin, 32, + offsetof(struct st_file_in_flush, file), + sizeof(((struct st_file_in_flush *)NULL)->file), + NULL, NULL, 0)) + goto err; pagecache->inited= 1; pagecache->in_init= 0; - pthread_mutex_init(&pagecache->cache_lock, MY_MUTEX_INIT_FAST); pagecache->resize_queue.last_thread= NULL; } @@ -841,7 +857,7 @@ static int flush_all_key_blocks(PAGECACHE *pagecache) KEYCACHE_DBUG_ASSERT(cnt <= pagecache->blocks_used); #endif if (flush_pagecache_blocks_int(pagecache, &block->hash_link->file, - FLUSH_RELEASE)) + FLUSH_RELEASE, NULL, NULL)) return 1; break; } @@ -1074,6 +1090,7 @@ void end_pagecache(PAGECACHE *pagecache, my_bool cleanup) if (cleanup) { + hash_free(&pagecache->files_in_flush); pthread_mutex_destroy(&pagecache->cache_lock); pagecache->inited= pagecache->can_be_used= 0; PAGECACHE_DEBUG_CLOSE; @@ -3489,7 +3506,7 @@ static int flush_cached_blocks(PAGECACHE *pagecache, pagecache_pthread_mutex_unlock(&pagecache->cache_lock); /* As all blocks referred in 'cache' are marked by PCBLOCK_IN_FLUSH - we are guarantied no thread will change them + we are guaranteed that no thread will change them */ qsort((uchar*) cache, count, sizeof(*cache), (qsort_cmp) cmp_sec_link); @@ -3506,6 +3523,8 @@ static int flush_cached_blocks(PAGECACHE *pagecache, DBUG_PRINT("info", ("block: %u (0x%lx) pinned", PCBLOCK_NUMBER(pagecache, block), (ulong)block)); PCBLOCK_INFO(block); + /* undo the mark put by flush_pagecache_blocks_int(): */ + block->status&= ~PCBLOCK_IN_FLUSH; last_errno= -1; unreg_request(pagecache, block, 1); continue; @@ -3555,7 +3574,8 @@ static int flush_cached_blocks(PAGECACHE *pagecache, wqueue_release_queue(&block->wqueue[COND_FOR_SAVED]); #endif /* type will never be FLUSH_IGNORE_CHANGED here */ - if (! (type == FLUSH_KEEP || type == FLUSH_FORCE_WRITE)) + if (! (type == FLUSH_KEEP || type == FLUSH_KEEP_LAZY || + type == FLUSH_FORCE_WRITE)) { pagecache->blocks_changed--; pagecache->global_blocks_changed--; @@ -3573,17 +3593,28 @@ static int flush_cached_blocks(PAGECACHE *pagecache, /** - @brief flush all key blocks for a file to disk but don't do any mutex locks + @brief flush all blocks for a file to disk but don't do any mutex locks @param pagecache pointer to a pagecache data structure @param file handler for the file to flush to @param flush_type type of the flush + @param filter optional function which tells what blocks to flush; + can be non-NULL only if FLUSH_KEEP, FLUSH_KEEP_LAZY + or FLUSH_FORCE_WRITE. + @param filter_arg an argument to pass to 'filter'. Information about + the block will be passed too. @note This function doesn't do any mutex locks because it needs to be called both from flush_pagecache_blocks and flush_all_key_blocks (the later one does the mutex lock in the resize_pagecache() function). + @note + This function can cause problems if two threads call it + concurrently on the same file (look for "PageCacheFlushConcurrencyBugs" + in ma_checkpoint.c); to avoid them, it has internal logic to serialize in + this situation. + @return Operation status @retval 0 OK @retval 1 Error @@ -3591,7 +3622,9 @@ static int flush_cached_blocks(PAGECACHE *pagecache, static int flush_pagecache_blocks_int(PAGECACHE *pagecache, PAGECACHE_FILE *file, - enum flush_type type) + enum flush_type type, + PAGECACHE_FLUSH_FILTER filter, + void *filter_arg) { PAGECACHE_BLOCK_LINK *cache_buff[FLUSH_CACHE],**cache; int last_errno= 0; @@ -3607,9 +3640,15 @@ static int flush_pagecache_blocks_int(PAGECACHE *pagecache, cache= cache_buff; if (pagecache->disk_blocks > 0 && - (!my_disable_flush_pagecache_blocks || type != FLUSH_KEEP)) + (!my_disable_flush_pagecache_blocks || + (type != FLUSH_KEEP && type != FLUSH_KEEP_LAZY))) { - /* Key cache exists and flush is not disabled */ + /* + Key cache exists. If my_disable_flush_pagecache_blocks is true it + disables the operation but only FLUSH_KEEP[_LAZY]: other flushes still + need to be allowed: FLUSH_RELEASE has to free blocks, and + FLUSH_FORCE_WRITE is to overrule my_disable_flush_pagecache_blocks. + */ int error= 0; uint count= 0; PAGECACHE_BLOCK_LINK **pos, **end; @@ -3618,13 +3657,66 @@ static int flush_pagecache_blocks_int(PAGECACHE *pagecache, #if defined(PAGECACHE_DEBUG) uint cnt= 0; #endif - uint8 changed_blocks_is_incomplete_incremented= 0; + +#ifdef THREAD + struct st_file_in_flush us_flusher, *other_flusher; + us_flusher.file= *file; + us_flusher.flush_queue.last_thread= NULL; + us_flusher.first_in_switch= FALSE; + while ((other_flusher= (struct st_file_in_flush *) + hash_search(&pagecache->files_in_flush, (uchar *)file, + sizeof(*file)))) + { + /* + File is in flush already: wait, unless FLUSH_KEEP_LAZY. "Flusher" + means "who can mark PCBLOCK_IN_FLUSH", i.e. caller of + flush_pagecache_blocks_int(). + */ + struct st_my_thread_var *thread; + if (type == FLUSH_KEEP_LAZY) + { + DBUG_PRINT("info",("FLUSH_KEEP_LAZY skips")); + DBUG_RETURN(0); + } + thread= my_thread_var; + wqueue_add_to_queue(&other_flusher->flush_queue, thread); + do + { + KEYCACHE_DBUG_PRINT("flush_pagecache_blocks_int: wait1", + ("suspend thread %ld", thread->id)); + pagecache_pthread_cond_wait(&thread->suspend, + &pagecache->cache_lock); + } + while (thread->next); + } + /* we are the only flusher of this file now */ + while (my_hash_insert(&pagecache->files_in_flush, (uchar *)&us_flusher)) + { + /* + Out of memory, wait for flushers to empty the hash and retry; should + rarely happen. Other threads are flushing the file; when done, they + are going to remove themselves from the hash, and thus memory will + appear again. However, this memory may be stolen by yet another thread + (for a purpose unrelated to page cache), before we retry + hash_insert(). So the loop may run for long. Only if the thread was + killed do we abort the loop, returning 1 (error) which can cause the + table to be marked as corrupted (cf maria_chk_size(), maria_close()) + and thus require a table check. + */ + DBUG_ASSERT(0); + pagecache_pthread_mutex_unlock(&pagecache->cache_lock); + if (my_thread_var->abort) + DBUG_RETURN(1); /* End if aborted by user */ + sleep(10); + pagecache_pthread_mutex_lock(&pagecache->cache_lock); + } +#endif if (type != FLUSH_IGNORE_CHANGED) { /* - Count how many key blocks we have to cache to be able - to flush all dirty pages with minimum seek moves + Count how many key blocks we have to cache to be able + to flush all dirty pages with minimum seek moves. */ for (block= pagecache->changed_blocks[FILE_HASH(*file)] ; block; @@ -3659,7 +3751,19 @@ restart: KEYCACHE_DBUG_ASSERT(cnt <= pagecache->blocks_used); #endif next= block->next_changed; - if (block->hash_link->file.file == file->file) + if (block->hash_link->file.file != file->file) + continue; + if (filter != NULL) + { + int filter_res= (*filter)(block->type, block->hash_link->pageno, + block->rec_lsn, filter_arg); + DBUG_PRINT("info",("filter returned %d", filter_res)); + if (filter_res == FLUSH_FILTER_SKIP_TRY_NEXT) + continue; + if (filter_res == FLUSH_FILTER_SKIP_ALL) + break; + DBUG_ASSERT(filter_res == FLUSH_FILTER_OK); + } { /* Mark the block with BLOCK_IN_FLUSH in order not to let @@ -3705,34 +3809,15 @@ restart: free_block(pagecache, block); } } - else + else if (type != FLUSH_KEEP_LAZY) { - /* Link the block into a list of blocks 'in switch' */ - unlink_changed(block); - link_changed(block, &first_in_switch); /* - We have just removed a page from the list of dirty pages - ("changed_blocks") though it's still dirty (the flush by another - thread has not yet happened). Checkpoint will miss the page and so - must be blocked until that flush has happened. - Note that if there are two concurrent - flush_pagecache_blocks_int() on this file, then the first one may - move the block into its first_in_switch, and the second one would - just not see the block and wrongly consider its job done. - @todo RECOVERY Maria does protect such flushes with intern_lock, - but Checkpoint does not (Checkpoint makes sure that - changed_blocks_is_incomplete is 0 when it starts, but as - flush_cached_blocks() releases mutex, this may change... - */ - /** - @todo RECOVERY: check all places where we remove a page from the - list of dirty pages + Link the block into a list of blocks 'in switch', and then we will + wait for this list to be empty, which means they have been flushed */ - if (unlikely(!changed_blocks_is_incomplete_incremented)) - { - changed_blocks_is_incomplete_incremented= 1; - changed_blocks_is_incomplete++; - } + unlink_changed(block); + link_changed(block, &first_in_switch); + us_flusher.first_in_switch= TRUE; } } } @@ -3754,7 +3839,7 @@ restart: wqueue_add_to_queue(&block->wqueue[COND_FOR_SAVED], thread); do { - KEYCACHE_DBUG_PRINT("flush_pagecache_blocks_int: wait", + KEYCACHE_DBUG_PRINT("flush_pagecache_blocks_int: wait2", ("suspend thread %ld", thread->id)); pagecache_pthread_cond_wait(&thread->suspend, &pagecache->cache_lock); @@ -3770,11 +3855,16 @@ restart: KEYCACHE_DBUG_ASSERT(cnt <= pagecache->blocks_used); #endif } - changed_blocks_is_incomplete-= - changed_blocks_is_incomplete_incremented; + us_flusher.first_in_switch= FALSE; /* The following happens very seldom */ - if (! (type == FLUSH_KEEP || type == FLUSH_FORCE_WRITE)) + if (! (type == FLUSH_KEEP || type == FLUSH_KEEP_LAZY || + type == FLUSH_FORCE_WRITE)) { + /* + this code would free all blocks while filter maybe handled only a + few, that is not possible. + */ + DBUG_ASSERT(filter == NULL); #if defined(PAGECACHE_DEBUG) cnt=0; #endif @@ -3796,6 +3886,12 @@ restart: } } } +#ifdef THREAD + /* wake up others waiting to flush this file */ + hash_delete(&pagecache->files_in_flush, (uchar *)&us_flusher); + if (us_flusher.flush_queue.last_thread) + wqueue_release_queue(&us_flusher.flush_queue); +#endif } #ifndef DBUG_OFF @@ -3810,23 +3906,28 @@ restart: } -/* - Flush all blocks for a file to disk - - SYNOPSIS +/** + @brief flush all blocks for a file to disk - flush_pagecache_blocks() - pagecache pointer to a page cache data structure - file handler for the file to flush to - flush_type type of the flush + @param pagecache pointer to a pagecache data structure + @param file handler for the file to flush to + @param flush_type type of the flush + @param filter optional function which tells what blocks to flush; + can be non-NULL only if FLUSH_KEEP, FLUSH_KEEP_LAZY + or FLUSH_FORCE_WRITE. + @param filter_arg an argument to pass to 'filter'. Information about + the block will be passed too. - RETURN - 0 OK - 1 error + @return Operation status + @retval 0 OK + @retval 1 Error */ -int flush_pagecache_blocks(PAGECACHE *pagecache, - PAGECACHE_FILE *file, enum flush_type type) +int flush_pagecache_blocks_with_filter(PAGECACHE *pagecache, + PAGECACHE_FILE *file, + enum flush_type type, + PAGECACHE_FLUSH_FILTER filter, + void *filter_arg) { int res; DBUG_ENTER("flush_pagecache_blocks"); @@ -3836,7 +3937,7 @@ int flush_pagecache_blocks(PAGECACHE *pagecache, DBUG_RETURN(0); pagecache_pthread_mutex_lock(&pagecache->cache_lock); inc_counter_for_resize_op(pagecache); - res= flush_pagecache_blocks_int(pagecache, file, type); + res= flush_pagecache_blocks_int(pagecache, file, type, filter, filter_arg); dec_counter_for_resize_op(pagecache); pagecache_pthread_mutex_unlock(&pagecache->cache_lock); DBUG_RETURN(res); @@ -3916,16 +4017,42 @@ my_bool pagecache_collect_changed_blocks_with_lsn(PAGECACHE *pagecache, of memory at most. */ pagecache_pthread_mutex_lock(&pagecache->cache_lock); - while (changed_blocks_is_incomplete > 0) +#ifdef THREAD + for (;;) { + struct st_file_in_flush *other_flusher; + for (file_hash= 0; + (other_flusher= (struct st_file_in_flush *) + hash_element(&pagecache->files_in_flush, file_hash)) != NULL && + !other_flusher->first_in_switch; + file_hash++) + {} + if (other_flusher == NULL) + break; /* - Some pages are more recent in memory than on disk (=dirty) and are not - in "changed_blocks" so we cannot know them. Wait. + other_flusher.first_in_switch is true: some thread is flushing a file + and has removed dirty blocks from changed_blocks[] while they were still + dirty (they were being evicted (=>flushed) by yet another thread, which + may not have flushed the block yet so it may still be dirty). + If Checkpoint proceeds now, it will not see the page. If there is a + crash right after writing the checkpoint record, before the page is + flushed, at recovery the page will be wrongly ignored because it won't + be in the dirty pages list in the checkpoint record. So wait. */ - pagecache_pthread_mutex_unlock(&pagecache->cache_lock); - sleep(1); - pagecache_pthread_mutex_lock(&pagecache->cache_lock); + { + struct st_my_thread_var *thread= my_thread_var; + wqueue_add_to_queue(&other_flusher->flush_queue, thread); + do + { + KEYCACHE_DBUG_PRINT("pagecache_collect_çhanged_blocks_with_lsn: wait", + ("suspend thread %ld", thread->id)); + pagecache_pthread_cond_wait(&thread->suspend, + &pagecache->cache_lock); + } + while (thread->next); + } } +#endif /* Count how many dirty pages are interesting */ for (file_hash= 0; file_hash < PAGECACHE_CHANGED_BLOCKS_HASH; file_hash++) @@ -3941,8 +4068,18 @@ my_bool pagecache_collect_changed_blocks_with_lsn(PAGECACHE *pagecache, */ DBUG_ASSERT(block->hash_link != NULL); DBUG_ASSERT(block->status & PCBLOCK_CHANGED); + /** + @todo RECOVERY BUG + REDO phase uses PAGECACHE_PLAIN_PAGE, so the lines below would + confuse the indirect Checkpoint taken at the end of the REDO phase. + So we below collect even dirty pages of temporary tables as a result + :( Soon we should have the MARIA_SHARE accessible from the + pagecache's block and then we can test born_transactional. + */ +#ifdef TRANS_TABLES_ALWAYS_USE_LSN_PAGE if (block->type != PAGECACHE_LSN_PAGE) continue; /* no need to store it */ +#endif stored_list_size++; } } @@ -3967,8 +4104,10 @@ my_bool pagecache_collect_changed_blocks_with_lsn(PAGECACHE *pagecache, block; block= block->next_changed) { +#ifdef TRANS_TABLES_ALWAYS_USE_LSN_PAGE if (block->type != PAGECACHE_LSN_PAGE) continue; /* no need to store it in the checkpoint record */ +#endif compile_time_assert(sizeof(block->hash_link->file.file) <= 4); compile_time_assert(sizeof(block->hash_link->pageno) <= 4); int4store(ptr, block->hash_link->file.file); diff --git a/storage/maria/ma_pagecache.h b/storage/maria/ma_pagecache.h index 23a443a5b97..64935a0fa36 100644 --- a/storage/maria/ma_pagecache.h +++ b/storage/maria/ma_pagecache.h @@ -22,6 +22,7 @@ C_MODE_START #include "ma_loghandler_lsn.h" #include <m_string.h> +#include <hash.h> /* Type of the page */ enum pagecache_page_type @@ -159,8 +160,21 @@ typedef struct st_pagecache my_bool resize_in_flush; /* true during flush of resize operation */ my_bool can_be_used; /* usage of cache for read/write is allowed */ my_bool in_init; /* Set to 1 in MySQL during init/resize */ + HASH files_in_flush; /**< files in flush_pagecache_blocks_int() */ } PAGECACHE; +/** @brief Return values for PAGECACHE_FLUSH_FILTER */ +enum pagecache_flush_filter_result +{ + FLUSH_FILTER_SKIP_TRY_NEXT= 0,/**< skip page and move on to next one */ + FLUSH_FILTER_OK, /**< flush page and move on to next one */ + FLUSH_FILTER_SKIP_ALL /**< skip page and all next ones */ +}; +/** @brief a filter function type for flush_pagecache_blocks_with_filter() */ +typedef enum pagecache_flush_filter_result +(*PAGECACHE_FLUSH_FILTER)(enum pagecache_page_type type, pgcache_page_no_t page, + LSN rec_lsn, void *arg); + /* The default key cache */ extern PAGECACHE dflt_pagecache_var, *dflt_pagecache; @@ -228,9 +242,13 @@ extern void pagecache_unpin(PAGECACHE *pagecache, extern void pagecache_unpin_by_link(PAGECACHE *pagecache, PAGECACHE_BLOCK_LINK *link, LSN lsn); -extern int flush_pagecache_blocks(PAGECACHE *keycache, - PAGECACHE_FILE *file, - enum flush_type type); +#define flush_pagecache_blocks(A,B,C) \ + flush_pagecache_blocks_with_filter(A,B,C,NULL,NULL) +extern int flush_pagecache_blocks_with_filter(PAGECACHE *keycache, + PAGECACHE_FILE *file, + enum flush_type type, + PAGECACHE_FLUSH_FILTER filter, + void *filter_arg); extern my_bool pagecache_delete(PAGECACHE *pagecache, PAGECACHE_FILE *file, pgcache_page_no_t pageno, diff --git a/storage/maria/ma_recovery.c b/storage/maria/ma_recovery.c index 5a293507a33..4d2c8dc7fdd 100644 --- a/storage/maria/ma_recovery.c +++ b/storage/maria/ma_recovery.c @@ -54,6 +54,7 @@ static my_bool skip_DDLs; /**< if REDO phase should skip DDL records */ /** @brief to avoid writing a checkpoint if recovery did nothing. */ static my_bool checkpoint_useful; static ulonglong now; /**< for tracking execution time of phases */ +static char preamble[]= "Maria engine: starting recovery; "; #define prototype_redo_exec_hook(R) \ static int exec_REDO_LOGREC_ ## R(const TRANSLOG_HEADER_BUFFER *rec) @@ -89,7 +90,7 @@ prototype_undo_exec_hook(UNDO_ROW_INSERT); prototype_undo_exec_hook(UNDO_ROW_DELETE); prototype_undo_exec_hook(UNDO_ROW_UPDATE); -static int run_redo_phase(LSN lsn, my_bool apply); +static int run_redo_phase(LSN lsn, enum maria_apply_log_way apply); static uint end_of_redo_phase(my_bool prepare_for_undo_phase); static int run_undo_phase(uint unfinished); static void display_record_position(const LOG_DESC *log_desc, @@ -126,20 +127,22 @@ static void enlarge_buffer(const TRANSLOG_HEADER_BUFFER *rec) MYF(MY_WME | MY_ALLOW_ZERO_PTR)); } } -static my_bool redo_phase_message_printed; +/** @brief Tells what kind of progress message was printed to the error log */ +static enum recovery_message_type +{ + REC_MSG_NONE= 0, REC_MSG_REDO, REC_MSG_UNDO, REC_MSG_FLUSH +} recovery_message_printed; /** @brief Prints to a trace file if it is not NULL */ void tprint(FILE *trace_file, const char *format, ...) ATTRIBUTE_FORMAT(printf, 2, 3); void tprint(FILE *trace_file __attribute__ ((unused)), const char *format __attribute__ ((unused)), ...) { -#ifdef EXTRA_DEBUG va_list args; va_start(args, format); if (trace_file != NULL) vfprintf(trace_file, format, args); va_end(args); -#endif } #define ALERT_USER() DBUG_ASSERT(0) @@ -174,7 +177,8 @@ int maria_recover(void) #endif tprint(trace_file, "TRACE of the last MARIA recovery from mysqld\n"); DBUG_ASSERT(maria_pagecache->inited); - res= maria_apply_log(LSN_IMPOSSIBLE, TRUE, trace_file, TRUE, TRUE, TRUE); + res= maria_apply_log(LSN_IMPOSSIBLE, MARIA_LOG_APPLY, trace_file, + TRUE, TRUE, TRUE); if (!res) tprint(trace_file, "SUCCESS\n"); if (trace_file) @@ -189,7 +193,7 @@ int maria_recover(void) @param from_lsn LSN from which log reading/applying should start; LSN_IMPOSSIBLE means "use last checkpoint" - @param apply if log records should be applied or not + @param apply how log records should be applied or not @param trace_file trace file where progress/debug messages will go @param skip_DDLs_arg Should DDL records (CREATE/RENAME/DROP/REPAIR) be skipped by the REDO phase or not @@ -204,15 +208,17 @@ int maria_recover(void) @retval !=0 Error */ -int maria_apply_log(LSN from_lsn, my_bool apply, FILE *trace_file, +int maria_apply_log(LSN from_lsn, enum maria_apply_log_way apply, + FILE *trace_file, my_bool should_run_undo_phase, my_bool skip_DDLs_arg, my_bool take_checkpoints) { int error= 0; uint unfinished_trans; + ulonglong old_now; DBUG_ENTER("maria_apply_log"); - DBUG_ASSERT(apply || !should_run_undo_phase); + DBUG_ASSERT(apply == MARIA_LOG_APPLY || !should_run_undo_phase); DBUG_ASSERT(!maria_multi_threaded); /* checkpoints can happen only if TRNs have been built */ DBUG_ASSERT(should_run_undo_phase || !take_checkpoints); @@ -225,10 +231,10 @@ int maria_apply_log(LSN from_lsn, my_bool apply, FILE *trace_file, if (!all_active_trans || !all_tables) goto err; - if (take_checkpoints && ma_checkpoint_init(FALSE)) + if (take_checkpoints && ma_checkpoint_init(0)) goto err; - redo_phase_message_printed= FALSE; + recovery_message_printed= REC_MSG_NONE; tracef= trace_file; if (!(skip_DDLs= skip_DDLs_arg)) { @@ -277,6 +283,7 @@ int maria_apply_log(LSN from_lsn, my_bool apply, FILE *trace_file, } } + now= my_getsystime(); if (run_redo_phase(from_lsn, apply)) goto err; @@ -284,6 +291,24 @@ int maria_apply_log(LSN from_lsn, my_bool apply, FILE *trace_file, end_of_redo_phase(should_run_undo_phase)) == (uint)-1) goto err; + old_now= now; + now= my_getsystime(); + if (recovery_message_printed == REC_MSG_REDO) + { + float phase_took= (now - old_now)/10000000.0; + /** @todo RECOVERY BUG all prints to stderr should go to error log */ + fprintf(stderr, " (%.1f seconds); ", phase_took); + } + + /** + REDO phase does not fill blocks' rec_lsn, so a checkpoint now would be + wrong: if a future recovery used it, the REDO phase would always + start from the checkpoint and never from before, wrongly skipping REDOs + (tested). + + @todo fix this. + */ +#if 0 if (take_checkpoints && checkpoint_useful) { /* @@ -294,6 +319,7 @@ int maria_apply_log(LSN from_lsn, my_bool apply, FILE *trace_file, if (ma_checkpoint_execute(CHECKPOINT_INDIRECT, FALSE)) goto err; } +#endif if (should_run_undo_phase) { @@ -304,6 +330,15 @@ int maria_apply_log(LSN from_lsn, my_bool apply, FILE *trace_file, tprint(tracef, "WARNING: %u unfinished transactions; some tables may be" " left inconsistent!\n", unfinished_trans); + old_now= now; + now= my_getsystime(); + if (recovery_message_printed == REC_MSG_UNDO) + { + float phase_took= (now - old_now)/10000000.0; + /** @todo RECOVERY BUG all prints to stderr should go to error log */ + fprintf(stderr, " (%.1f seconds); ", phase_took); + } + /* we don't use maria_panic() because it would maria_end(), and Recovery does not want that (we want to keep some modules initialized for runtime). @@ -311,6 +346,15 @@ int maria_apply_log(LSN from_lsn, my_bool apply, FILE *trace_file, if (close_all_tables()) goto err; + old_now= now; + now= my_getsystime(); + if (recovery_message_printed == REC_MSG_FLUSH) + { + float phase_took= (now - old_now)/10000000.0; + /** @todo RECOVERY BUG all prints to stderr should go to error log */ + fprintf(stderr, " (%.1f seconds); ", phase_took); + } + if (take_checkpoints && checkpoint_useful) { /* No dirty pages, all tables are closed, no active transactions, save: */ @@ -335,14 +379,10 @@ end: log_record_buffer.str= NULL; log_record_buffer.length= 0; ma_checkpoint_end(); - if (tracef != stdout && redo_phase_message_printed) + if (recovery_message_printed != REC_MSG_NONE) { - ulonglong old_now= now; - now= my_getsystime(); - float previous_phase_took= (now - old_now)/10000000.0; - /** @todo RECOVERY BUG all prints to stderr should go to error log */ /** @todo RECOVERY BUG all prints to stderr should go to error log */ - fprintf(stderr, " (%.1f seconds)\n", previous_phase_took); + fprintf(stderr, "%s.\n", error ? " failed" : "done"); } /* we don't cleanly close tables if we hit some error (may corrupt them) */ DBUG_RETURN(error); @@ -376,7 +416,7 @@ static int display_and_apply_record(const LOG_DESC *log_desc, return 1; } if ((error= (*log_desc->record_execute_in_redo_phase)(rec))) - tprint(tracef, "Got error when executing redo on record\n"); + tprint(tracef, "Got error when executing record\n"); return error; } @@ -529,10 +569,31 @@ prototype_redo_exec_hook(REDO_CREATE_TABLE) else /* one or two files absent, or header corrupted... */ tprint(tracef, " can't be opened, probably does not exist"); /* if does not exist, or is older, overwrite it */ - /** @todo symlinks */ ptr= name + strlen(name) + 1; if ((flags= ptr[0] ? HA_DONT_TOUCH_DATA : 0)) tprint(tracef, ", we will only touch index file"); + ptr++; + kfile_size_before_extension= uint2korr(ptr); + ptr+= 2; + keystart= uint2korr(ptr); + ptr+= 2; + uchar *kfile_header= ptr; + ptr+= kfile_size_before_extension; + /* set create_rename_lsn (for maria_read_log to be idempotent) */ + lsn_store(kfile_header + sizeof(info->s->state.header) + 2, rec->lsn); + /* we also set is_of_horizon, like maria_create() does */ + lsn_store(kfile_header + sizeof(info->s->state.header) + 2 + LSN_STORE_SIZE, + rec->lsn); + uchar *data_file_name= ptr; + ptr+= strlen(data_file_name) + 1; + uchar *index_file_name= ptr; + ptr+= strlen(index_file_name) + 1; + /** @todo handle symlinks */ + if (data_file_name[0] || index_file_name[0]) + { + tprint(tracef, ", DATA|INDEX DIRECTORY clauses are not handled\n"); + goto end; + } fn_format(filename, name, "", MARIA_NAME_IEXT, (MY_UNPACK_FILENAME | (flags & HA_DONT_TOUCH_DATA) ? MY_RETURN_REAL_PATH : 0) | @@ -546,17 +607,7 @@ prototype_redo_exec_hook(REDO_CREATE_TABLE) tprint(tracef, " Failed to create index file\n"); goto end; } - ptr++; - kfile_size_before_extension= uint2korr(ptr); - ptr+= 2; - keystart= uint2korr(ptr); - ptr+= 2; - /* set create_rename_lsn (for maria_read_log to be idempotent) */ - lsn_store(ptr + sizeof(info->s->state.header) + 2, rec->lsn); - /* we also set is_of_horizon, like maria_create() does */ - lsn_store(ptr + sizeof(info->s->state.header) + 2 + LSN_STORE_SIZE, - rec->lsn); - if (my_pwrite(kfile, ptr, + if (my_pwrite(kfile, kfile_header, kfile_size_before_extension, 0, MYF(MY_NABP|MY_WME)) || my_chsize(kfile, keystart, 0, MYF(MY_WME))) { @@ -953,10 +1004,21 @@ static int new_table(uint16 sid, const char *name, 0 (success): leave table open and return 0. */ int error= 1; + MARIA_HA *info; checkpoint_useful= TRUE; + if ((name == NULL) || (name[0] == 0)) + { + /* + we didn't use DBUG_ASSERT() because such record corruption could + silently pass in the "info == NULL" test below. + */ + tprint(tracef, ", record is corrupted"); + info= NULL; + goto end; + } tprint(tracef, "Table '%s', id %u", name, sid); - MARIA_HA *info= maria_open(name, O_RDWR, HA_OPEN_FOR_REPAIR); + info= maria_open(name, O_RDWR, HA_OPEN_FOR_REPAIR); if (info == NULL) { tprint(tracef, ", is absent (must have been dropped later?)" @@ -1578,7 +1640,7 @@ prototype_undo_exec_hook(UNDO_ROW_UPDATE) } -static int run_redo_phase(LSN lsn, my_bool apply) +static int run_redo_phase(LSN lsn, enum maria_apply_log_way apply) { TRANSLOG_HEADER_BUFFER rec; struct st_translog_scanner_data scanner; @@ -1627,7 +1689,6 @@ static int run_redo_phase(LSN lsn, my_bool apply) len= translog_read_record_header(lsn, &rec); - /** @todo EOF should be detected */ if (len == RECHEADER_READ_ERROR) { tprint(tracef, "Failed to read header of the first record.\n"); @@ -1693,7 +1754,22 @@ static int run_redo_phase(LSN lsn, my_bool apply) { const LOG_DESC *log_desc2= &log_record_type_descriptor[rec2.type]; display_record_position(log_desc2, &rec2, 0); - if (apply && display_and_apply_record(log_desc2, &rec2)) + if (apply == MARIA_LOG_CHECK) + { + translog_size_t read_len; + enlarge_buffer(&rec2); + read_len= + translog_read_record(rec2.lsn, 0, rec2.record_length, + log_record_buffer.str, NULL); + if (read_len != rec2.record_length) + { + tprint(tracef, "Cannot read record's body: read %u of" + " %u bytes\n", read_len, rec2.record_length); + goto err; + } + } + if (apply == MARIA_LOG_APPLY && + display_and_apply_record(log_desc2, &rec2)) { translog_destroy_scanner(&scanner2); goto err; @@ -1715,7 +1791,8 @@ static int run_redo_phase(LSN lsn, my_bool apply) translog_destroy_scanner(&scanner2); } } - if (apply && display_and_apply_record(log_desc, &rec)) + if (apply == MARIA_LOG_APPLY && + display_and_apply_record(log_desc, &rec)) goto err; } else /* record does not end group */ @@ -1744,6 +1821,11 @@ static int run_redo_phase(LSN lsn, my_bool apply) } translog_destroy_scanner(&scanner); translog_free_record_header(&rec); + if (recovery_message_printed == REC_MSG_REDO) + { + /** @todo RECOVERY BUG all prints to stderr should go to error log */ + fprintf(stderr, " 100%%"); + } return 0; err: @@ -1846,17 +1928,16 @@ static int run_undo_phase(uint unfinished) checkpoint_useful= TRUE; if (tracef != stdout) { - ulonglong old_now= now; - now= my_getsystime(); - float previous_phase_took= (now - old_now)/10000000.0; + if (recovery_message_printed == REC_MSG_NONE) + fprintf(stderr, preamble); /** @todo RECOVERY BUG all prints to stderr should go to error log */ - fprintf(stderr, " 100%% (%.1f seconds); transactions to roll back:", - previous_phase_took); + fprintf(stderr, "transactions to roll back:"); + recovery_message_printed= REC_MSG_UNDO; } tprint(tracef, "%u transactions will be rolled back\n", unfinished); for( ; ; ) { - if (tracef != stdout) + if (recovery_message_printed == REC_MSG_UNDO) fprintf(stderr, " %u", unfinished); if ((unfinished--) == 0) break; @@ -1917,7 +1998,10 @@ static void prepare_table_for_close(MARIA_HA *info, TRANSLOG_ADDRESS horizon) */ if (cmp_translog_addr(share->state.is_of_horizon, horizon) < 0 && cmp_translog_addr(share->lsn_of_file_id, horizon) < 0) + { share->state.is_of_horizon= horizon; + _ma_state_info_write_sub(share->kfile.file, &share->state, 1); + } _ma_reenable_logging_for_table(share); } @@ -1933,15 +2017,20 @@ static MARIA_HA *get_MARIA_HA_from_REDO_record(const print_redo_phase_progress(rec->lsn); sid= fileid_korr(rec->header); page= page_korr(rec->header + FILEID_STORE_SIZE); - /** - @todo RECOVERY BUG - - for REDO_FREE_BLOCKS, page is not at this pos - - for DELETE_ALL, record ends here! buffer overrun! - Solution: caller should pass a param enum { i_am_about_data_file, - i_am_about_index_file, none }. - */ - llstr(page, llbuf); - tprint(tracef, " For page %s of table of short id %u", llbuf, sid); + switch(rec->type) + { + /* not all REDO records have a page: */ + case LOGREC_REDO_INSERT_ROW_HEAD: + case LOGREC_REDO_INSERT_ROW_TAIL: + case LOGREC_REDO_PURGE_ROW_HEAD: + case LOGREC_REDO_PURGE_ROW_TAIL: + llstr(page, llbuf); + tprint(tracef, " For page %s of table of short id %u", llbuf, sid); + break; + /* other types could print their info here too */ + default: + break; + } info= all_tables[sid].info; if (info == NULL) { @@ -2116,8 +2205,8 @@ static LSN parse_checkpoint_record(LSN lsn) LSN first_log_write_lsn= lsn_korr(ptr); ptr+= LSN_STORE_SIZE; uint name_len= strlen(ptr) + 1; + strmake(name, ptr, sizeof(name)-1); ptr+= name_len; - strnmov(name, ptr, sizeof(name)); if (new_table(sid, name, kfile, dfile, first_log_write_lsn)) return LSN_ERROR; } @@ -2192,32 +2281,45 @@ static int new_page(File fileid, pgcache_page_no_t pageid, LSN rec_lsn, static int close_all_tables(void) { int error= 0; + uint count; LIST *list_element, *next_open; MARIA_HA *info; pthread_mutex_lock(&THR_LOCK_maria); if (maria_open_list == NULL) goto end; tprint(tracef, "Closing all tables\n"); - if (tracef != stdout && redo_phase_message_printed) + if (tracef != stdout) { - ulonglong old_now= now; - now= my_getsystime(); - float previous_phase_took= (now - old_now)/10000000.0; + if (recovery_message_printed == REC_MSG_NONE) + fprintf(stderr, preamble); + for (count= 0, list_element= maria_open_list ; + list_element ; count++, (list_element= list_element->next)) /** @todo RECOVERY BUG all prints to stderr should go to error log */ - fprintf(stderr, " (%.1f seconds); flushing tables", previous_phase_took); + fprintf(stderr, "tables to flush:"); + recovery_message_printed= REC_MSG_FLUSH; } - /* Since the end of end_of_redo_phase(), we may have written new records (if UNDO phase ran) and thus the state is newer than at end_of_redo_phase(), we need to bump is_of_horizon again. */ TRANSLOG_ADDRESS addr= translog_get_horizon(); - for (list_element= maria_open_list ; list_element ; list_element= next_open) + for (list_element= maria_open_list ; ; list_element= next_open) { + if (recovery_message_printed == REC_MSG_FLUSH) + fprintf(stderr, " %u", count--); + if (list_element == NULL) + break; next_open= list_element->next; info= (MARIA_HA*)list_element->data; pthread_mutex_unlock(&THR_LOCK_maria); /* ok, UNDO phase not online yet */ + /* + Tables which we see here are exactly those which were open at time of + crash. They might have open_count>0 as Checkpoint maybe flushed their + state while they were used. As Recovery corrected them, don't alarm the + user, don't ask for a table check: + */ + info->s->state.open_count= 0; prepare_table_for_close(info, addr); error|= maria_close(info); pthread_mutex_lock(&THR_LOCK_maria); @@ -2264,12 +2366,12 @@ static void print_redo_phase_progress(TRANSLOG_ADDRESS addr) static ulonglong initial_remainder= -1; if (tracef == stdout) return; - if (!redo_phase_message_printed) + if (recovery_message_printed == REC_MSG_NONE) { /** @todo RECOVERY BUG all prints to stderr should go to error log */ - fprintf(stderr, "Maria engine: starting recovery; recovered pages: 0%%"); - redo_phase_message_printed= TRUE; - now= my_getsystime(); + fprintf(stderr, preamble); + fprintf(stderr, "recovered pages: 0%%"); + recovery_message_printed= REC_MSG_REDO; } if (end_logno == FILENO_IMPOSSIBLE) { diff --git a/storage/maria/ma_recovery.h b/storage/maria/ma_recovery.h index da4e8895e2f..ea39fa3e98c 100644 --- a/storage/maria/ma_recovery.h +++ b/storage/maria/ma_recovery.h @@ -24,8 +24,11 @@ /* Performs recovery of the engine at start */ C_MODE_START +enum maria_apply_log_way +{ MARIA_LOG_APPLY, MARIA_LOG_DISPLAY_HEADER, MARIA_LOG_CHECK }; int maria_recover(void); -int maria_apply_log(LSN lsn, my_bool apply, FILE *trace_file, +int maria_apply_log(LSN lsn, enum maria_apply_log_way apply, + FILE *trace_file, my_bool execute_undo_phase, my_bool skip_DDLs, my_bool take_checkpoints); C_MODE_END diff --git a/storage/maria/ma_test1.c b/storage/maria/ma_test1.c index 3f08a302cfb..1f487e01cd3 100644 --- a/storage/maria/ma_test1.c +++ b/storage/maria/ma_test1.c @@ -20,6 +20,7 @@ #include <m_string.h> #include "ma_control_file.h" #include "ma_loghandler.h" +#include "ma_checkpoint.h" #include "trnman.h" extern PAGECACHE *maria_log_pagecache; @@ -29,7 +30,7 @@ extern const char *maria_data_root; static void usage(); -static int rec_pointer_size=0, flags[50], testflag; +static int rec_pointer_size=0, flags[50], testflag, checkpoint; static int key_field=FIELD_SKIP_PRESPACE,extra_field=FIELD_SKIP_ENDSPACE; static int key_type=HA_KEYTYPE_NUM; static int create_flag=0; @@ -83,7 +84,7 @@ int main(int argc,char *argv[]) translog_init(maria_data_root, TRANSLOG_FILE_SIZE, 0, 0, maria_log_pagecache, TRANSLOG_DEFAULT_FLAGS) || - (transactional && trnman_init(0))) + (transactional && (trnman_init(0) || ma_checkpoint_init(0)))) { fprintf(stderr, "Error in initialization"); exit(1); @@ -227,6 +228,9 @@ static int run_test(const char *filename) if (maria_commit(file) || maria_begin(file)) goto err; + if (checkpoint == 1 && ma_checkpoint_execute(CHECKPOINT_MEDIUM, FALSE)) + goto err; + if (testflag == 1) goto end; @@ -247,6 +251,9 @@ static int run_test(const char *filename) flags[0]=2; } + if (checkpoint == 2 && ma_checkpoint_execute(CHECKPOINT_MEDIUM, FALSE)) + goto err; + if (testflag == 2) { printf("Terminating after inserts\n"); @@ -308,6 +315,9 @@ static int run_test(const char *filename) maria_scan_end(file); } + if (checkpoint == 3 && ma_checkpoint_execute(CHECKPOINT_MEDIUM, FALSE)) + goto err; + if (testflag == 3) { printf("Terminating after updates\n"); @@ -371,6 +381,9 @@ static int run_test(const char *filename) } } + if (checkpoint == 4 && ma_checkpoint_execute(CHECKPOINT_MEDIUM, FALSE)) + goto err; + if (testflag == 4) { printf("Terminating after deletes\n"); @@ -673,6 +686,8 @@ static void update_record(uchar *record) static struct my_option my_long_options[] = { + {"checkpoint", 'H', "Checkpoint at specified stage", (uchar**) &checkpoint, + (uchar**) &checkpoint, 0, GET_INT, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"checksum", 'c', "Undocumented", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, #ifndef DBUG_OFF diff --git a/storage/maria/ma_test2.c b/storage/maria/ma_test2.c index 4afa978a897..6bc09fb1aee 100644 --- a/storage/maria/ma_test2.c +++ b/storage/maria/ma_test2.c @@ -28,7 +28,7 @@ #include "trnman.h" #include <m_ctype.h> #include <my_bit.h> - +#include "ma_checkpoint.h" #define STANDARD_LENGTH 37 #define MARIA_KEYS 6 @@ -50,7 +50,7 @@ static int verbose=0,testflag=0, opt_quick_mode=0, transactional= 0, skip_update= 0, die_in_middle_of_transaction= 0; static int pack_seg=HA_SPACE_PACK,pack_type=HA_PACK_KEY,remove_count=-1; -static int create_flag= 0, srand_arg= 0; +static int create_flag= 0, srand_arg= 0, checkpoint= 0; static ulong pagecache_size=IO_SIZE*16; static enum data_file_type record_type= DYNAMIC_RECORD; @@ -97,7 +97,7 @@ int main(int argc, char *argv[]) translog_init(maria_data_root, TRANSLOG_FILE_SIZE, 0, 0, maria_log_pagecache, TRANSLOG_DEFAULT_FLAGS) || - (transactional && trnman_init(0))) + (transactional && (trnman_init(0) || ma_checkpoint_init(0)))) { fprintf(stderr, "Error in initialization"); exit(1); @@ -239,6 +239,8 @@ int main(int argc, char *argv[]) maria_begin(file); if (testflag == 1) goto end; + if (checkpoint == 1 && ma_checkpoint_execute(CHECKPOINT_MEDIUM, FALSE)) + goto err; if (!silent) printf("- Writing key:s\n"); if (locking) @@ -299,6 +301,8 @@ int main(int argc, char *argv[]) } } } + if (checkpoint == 2 && ma_checkpoint_execute(CHECKPOINT_MEDIUM, FALSE)) + goto err; if (write_cacheing) { @@ -354,6 +358,8 @@ int main(int argc, char *argv[]) } if (testflag == 3) goto end; + if (checkpoint == 3 && ma_checkpoint_execute(CHECKPOINT_MEDIUM, FALSE)) + goto err; if (!silent) printf("- Update\n"); @@ -415,6 +421,8 @@ int main(int argc, char *argv[]) } if (testflag == 4) goto end; + if (checkpoint == 4 && ma_checkpoint_execute(CHECKPOINT_MEDIUM, FALSE)) + goto err; for (i=999, dupp_keys=j=0 ; i>0 ; i--) { @@ -825,6 +833,8 @@ int main(int argc, char *argv[]) if (testflag == 5) goto end; + if (checkpoint == 5 && ma_checkpoint_execute(CHECKPOINT_MEDIUM, FALSE)) + goto err; if (!silent) printf("- Removing keys\n"); @@ -1058,6 +1068,9 @@ static void get_options(int argc, char **argv) if ((first_key=atoi(++pos)) < 0 || first_key >= MARIA_KEYS) first_key=0; break; + case 'H': + checkpoint= atoi(++pos); + break; case 'k': if ((keys=(uint) atoi(++pos)) < 1 || keys > (uint) (MARIA_KEYS-first_key)) diff --git a/storage/maria/ma_test_recovery b/storage/maria/ma_test_recovery index 00b6e0e00b4..1219114ebc5 100755 --- a/storage/maria/ma_test_recovery +++ b/storage/maria/ma_test_recovery @@ -126,16 +126,22 @@ echo "Testing the REDO AND UNDO PHASE" # Then we run it again and let it exit at T2. Then we compare # and expect identity. +for take_checkpoint in "no" "yes" +do for blobs in "" "-b" # we test table without blobs and then table with blobs do for test_undo in 1 2 3 do # first iteration tests rollback of insert, second tests rollback of delete - set -- "ma_test1 $silent -M -T -c -N $blobs" "--testflag=1" "--testflag=2 --test-undo=" "ma_test1 $silent -M -T -c -N $blobs" "--testflag=3" "--testflag=4 --test-undo=" "ma_test1 $silent -M -T -c -N $blobs" "--testflag=2" "--testflag=3 --test-undo=" "ma_test2 $silent -L -K -W -P -M -T -c $blobs" "-t1" "-t2 -u" + set -- "ma_test1 $silent -M -T -c -N $blobs -H1" "--testflag=1" "--testflag=2 --test-undo=" "ma_test1 $silent -M -T -c -N $blobs -H2" "--testflag=3" "--testflag=4 --test-undo=" "ma_test1 $silent -M -T -c -N $blobs -H2 " "--testflag=2" "--testflag=3 --test-undo=" "ma_test2 $silent -L -K -W -P -M -T -c $blobs -H1" "-t1" "-t2 -u" # -N (create NULL fields) is needed because --test-undo adds it anyway while [ $# != 0 ] do prog=$1 + if [ "$take_checkpoint" == "no" ] + then + prog=`echo $prog | sed 's/ -H[0-9]//'` + fi commit_run_args=$2 abort_run_args=$3; rm -f maria_log.* maria_log_control @@ -192,6 +198,7 @@ do rm -f $table.* $tmp/$table* $tmp/maria_chk_*.txt $tmp/maria_read_log_$table.txt done done +done ) 2>&1 > $tmp/ma_test_recovery.output diff --git a/storage/maria/ma_test_recovery.expected b/storage/maria/ma_test_recovery.expected index 65c2f0d719c..a73d1f82f64 100644 --- a/storage/maria/ma_test_recovery.expected +++ b/storage/maria/ma_test_recovery.expected @@ -63,8 +63,8 @@ Differences in maria_chk -dvv, recovery not yet perfect ! > Datafile length: 2531328 Keyfile length: 8192 ========DIFF END======= Testing the REDO AND UNDO PHASE -TEST WITH ma_test1 -s -M -T -c -N --testflag=1 (commit at end) -TEST WITH ma_test1 -s -M -T -c -N --testflag=2 --test-undo=1 (additional aborted work) +TEST WITH ma_test1 -s -M -T -c -N --testflag=1 (commit at end) +TEST WITH ma_test1 -s -M -T -c -N --testflag=2 --test-undo=1 (additional aborted work) Terminating after inserts Dying on request without maria_commit()/maria_close() applying log @@ -105,9 +105,9 @@ Differences in maria_chk -dvv, recovery not yet perfect ! --- > 1 2 6 unique number NULL 0 8192 ========DIFF END======= -TEST WITH ma_test1 -s -M -T -c -N --testflag=3 (commit at end) +TEST WITH ma_test1 -s -M -T -c -N --testflag=3 (commit at end) Terminating after updates -TEST WITH ma_test1 -s -M -T -c -N --testflag=4 --test-undo=1 (additional aborted work) +TEST WITH ma_test1 -s -M -T -c -N --testflag=4 --test-undo=1 (additional aborted work) Terminating after deletes Dying on request without maria_commit()/maria_close() applying log @@ -126,9 +126,9 @@ Differences in maria_chk -dvv, recovery not yet perfect ! --- > 1 2 6 unique number NULL 0 8192 ========DIFF END======= -TEST WITH ma_test1 -s -M -T -c -N --testflag=2 (commit at end) +TEST WITH ma_test1 -s -M -T -c -N --testflag=2 (commit at end) Terminating after inserts -TEST WITH ma_test1 -s -M -T -c -N --testflag=3 --test-undo=1 (additional aborted work) +TEST WITH ma_test1 -s -M -T -c -N --testflag=3 --test-undo=1 (additional aborted work) Terminating after updates Dying on request without maria_commit()/maria_close() applying log @@ -169,8 +169,8 @@ Differences in maria_chk -dvv, recovery not yet perfect ! --- > 1 2 6 unique number NULL 0 8192 ========DIFF END======= -TEST WITH ma_test2 -s -L -K -W -P -M -T -c -t1 (commit at end) -TEST WITH ma_test2 -s -L -K -W -P -M -T -c -t2 -u1 (additional aborted work) +TEST WITH ma_test2 -s -L -K -W -P -M -T -c -t1 (commit at end) +TEST WITH ma_test2 -s -L -K -W -P -M -T -c -t2 -u1 (additional aborted work) Dying on request without maria_commit()/maria_close() applying log Differences in maria_chk -dvv, recovery not yet perfect ! @@ -210,8 +210,8 @@ Differences in maria_chk -dvv, recovery not yet perfect ! --- > Datafile length: 114688 Keyfile length: 8192 ========DIFF END======= -TEST WITH ma_test1 -s -M -T -c -N --testflag=1 (commit at end) -TEST WITH ma_test1 -s -M -T -c -N --testflag=2 --test-undo=2 (additional aborted work) +TEST WITH ma_test1 -s -M -T -c -N --testflag=1 (commit at end) +TEST WITH ma_test1 -s -M -T -c -N --testflag=2 --test-undo=2 (additional aborted work) Terminating after inserts Dying on request without maria_commit()/maria_close() applying log @@ -252,9 +252,9 @@ Differences in maria_chk -dvv, recovery not yet perfect ! --- > 1 2 6 unique number NULL 0 8192 ========DIFF END======= -TEST WITH ma_test1 -s -M -T -c -N --testflag=3 (commit at end) +TEST WITH ma_test1 -s -M -T -c -N --testflag=3 (commit at end) Terminating after updates -TEST WITH ma_test1 -s -M -T -c -N --testflag=4 --test-undo=2 (additional aborted work) +TEST WITH ma_test1 -s -M -T -c -N --testflag=4 --test-undo=2 (additional aborted work) Terminating after deletes Dying on request without maria_commit()/maria_close() applying log @@ -273,9 +273,9 @@ Differences in maria_chk -dvv, recovery not yet perfect ! --- > 1 2 6 unique number NULL 0 8192 ========DIFF END======= -TEST WITH ma_test1 -s -M -T -c -N --testflag=2 (commit at end) +TEST WITH ma_test1 -s -M -T -c -N --testflag=2 (commit at end) Terminating after inserts -TEST WITH ma_test1 -s -M -T -c -N --testflag=3 --test-undo=2 (additional aborted work) +TEST WITH ma_test1 -s -M -T -c -N --testflag=3 --test-undo=2 (additional aborted work) Terminating after updates Dying on request without maria_commit()/maria_close() applying log @@ -316,8 +316,8 @@ Differences in maria_chk -dvv, recovery not yet perfect ! --- > 1 2 6 unique number NULL 0 8192 ========DIFF END======= -TEST WITH ma_test2 -s -L -K -W -P -M -T -c -t1 (commit at end) -TEST WITH ma_test2 -s -L -K -W -P -M -T -c -t2 -u2 (additional aborted work) +TEST WITH ma_test2 -s -L -K -W -P -M -T -c -t1 (commit at end) +TEST WITH ma_test2 -s -L -K -W -P -M -T -c -t2 -u2 (additional aborted work) Dying on request without maria_commit()/maria_close() applying log Differences in maria_chk -dvv, recovery not yet perfect ! @@ -357,8 +357,8 @@ Differences in maria_chk -dvv, recovery not yet perfect ! --- > Datafile length: 114688 Keyfile length: 8192 ========DIFF END======= -TEST WITH ma_test1 -s -M -T -c -N --testflag=1 (commit at end) -TEST WITH ma_test1 -s -M -T -c -N --testflag=2 --test-undo=3 (additional aborted work) +TEST WITH ma_test1 -s -M -T -c -N --testflag=1 (commit at end) +TEST WITH ma_test1 -s -M -T -c -N --testflag=2 --test-undo=3 (additional aborted work) Terminating after inserts Dying on request without maria_commit()/maria_close() applying log @@ -399,9 +399,9 @@ Differences in maria_chk -dvv, recovery not yet perfect ! --- > 1 2 6 unique number NULL 0 8192 ========DIFF END======= -TEST WITH ma_test1 -s -M -T -c -N --testflag=3 (commit at end) +TEST WITH ma_test1 -s -M -T -c -N --testflag=3 (commit at end) Terminating after updates -TEST WITH ma_test1 -s -M -T -c -N --testflag=4 --test-undo=3 (additional aborted work) +TEST WITH ma_test1 -s -M -T -c -N --testflag=4 --test-undo=3 (additional aborted work) Terminating after deletes Dying on request without maria_commit()/maria_close() applying log @@ -420,9 +420,9 @@ Differences in maria_chk -dvv, recovery not yet perfect ! --- > 1 2 6 unique number NULL 0 8192 ========DIFF END======= -TEST WITH ma_test1 -s -M -T -c -N --testflag=2 (commit at end) +TEST WITH ma_test1 -s -M -T -c -N --testflag=2 (commit at end) Terminating after inserts -TEST WITH ma_test1 -s -M -T -c -N --testflag=3 --test-undo=3 (additional aborted work) +TEST WITH ma_test1 -s -M -T -c -N --testflag=3 --test-undo=3 (additional aborted work) Terminating after updates Dying on request without maria_commit()/maria_close() applying log @@ -463,8 +463,8 @@ Differences in maria_chk -dvv, recovery not yet perfect ! --- > 1 2 6 unique number NULL 0 8192 ========DIFF END======= -TEST WITH ma_test2 -s -L -K -W -P -M -T -c -t1 (commit at end) -TEST WITH ma_test2 -s -L -K -W -P -M -T -c -t2 -u3 (additional aborted work) +TEST WITH ma_test2 -s -L -K -W -P -M -T -c -t1 (commit at end) +TEST WITH ma_test2 -s -L -K -W -P -M -T -c -t2 -u3 (additional aborted work) Dying on request without maria_commit()/maria_close() applying log Differences in maria_chk -dvv, recovery not yet perfect ! @@ -917,6 +917,792 @@ Differences in maria_chk -dvv, recovery not yet perfect ! 11c11 < Datafile length: 8192 Keyfile length: 8192 --- +> Datafile length: 81920 Keyfile length: 212992 +========DIFF END======= +testing idempotency +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +6c6 +< Status: checked,analyzed,optimized keys,sorted index pages +--- +> Status: changed +11c11 +< Datafile length: 8192 Keyfile length: 8192 +--- +> Datafile length: 81920 Keyfile length: 212992 +========DIFF END======= +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +6c6 +< Status: checked,analyzed,optimized keys,sorted index pages +--- +> Status: changed +11c11 +< Datafile length: 8192 Keyfile length: 8192 +--- +> Datafile length: 81920 Keyfile length: 8192 +========DIFF END======= +TEST WITH ma_test1 -s -M -T -c -N -H1 --testflag=1 (commit at end) +TEST WITH ma_test1 -s -M -T -c -N -H1 --testflag=2 --test-undo=1 (additional aborted work) +Terminating after inserts +Dying on request without maria_commit()/maria_close() +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 16384 Keyfile length: 16384 +--- +> Datafile length: 16384 Keyfile length: 8192 +========DIFF END======= +testing idempotency +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 16384 Keyfile length: 16384 +--- +> Datafile length: 16384 Keyfile length: 8192 +========DIFF END======= +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 16384 Keyfile length: 16384 +--- +> Datafile length: 16384 Keyfile length: 8192 +18c18 +< 1 2 6 unique number NULL 0 8192 8192 +--- +> 1 2 6 unique number NULL 0 8192 +========DIFF END======= +TEST WITH ma_test1 -s -M -T -c -N -H2 --testflag=3 (commit at end) +Terminating after updates +TEST WITH ma_test1 -s -M -T -c -N -H2 --testflag=4 --test-undo=1 (additional aborted work) +Terminating after deletes +Dying on request without maria_commit()/maria_close() +applying log +testing idempotency +applying log +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 16384 Keyfile length: 16384 +--- +> Datafile length: 16384 Keyfile length: 8192 +18c18 +< 1 2 6 unique number NULL 0 8192 8192 +--- +> 1 2 6 unique number NULL 0 8192 +========DIFF END======= +TEST WITH ma_test1 -s -M -T -c -N -H2 --testflag=2 (commit at end) +Terminating after inserts +TEST WITH ma_test1 -s -M -T -c -N -H2 --testflag=3 --test-undo=1 (additional aborted work) +Terminating after updates +Dying on request without maria_commit()/maria_close() +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 16384 Keyfile length: 16384 +--- +> Datafile length: 16384 Keyfile length: 8192 +========DIFF END======= +testing idempotency +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 16384 Keyfile length: 16384 +--- +> Datafile length: 16384 Keyfile length: 8192 +========DIFF END======= +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 16384 Keyfile length: 16384 +--- +> Datafile length: 16384 Keyfile length: 8192 +18c18 +< 1 2 6 unique number NULL 0 8192 8192 +--- +> 1 2 6 unique number NULL 0 8192 +========DIFF END======= +TEST WITH ma_test2 -s -L -K -W -P -M -T -c -H1 -t1 (commit at end) +TEST WITH ma_test2 -s -L -K -W -P -M -T -c -H1 -t2 -u1 (additional aborted work) +Dying on request without maria_commit()/maria_close() +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +6c6 +< Status: checked,analyzed,optimized keys,sorted index pages +--- +> Status: changed +11c11 +< Datafile length: 8192 Keyfile length: 8192 +--- +> Datafile length: 90112 Keyfile length: 204800 +========DIFF END======= +testing idempotency +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +6c6 +< Status: checked,analyzed,optimized keys,sorted index pages +--- +> Status: changed +11c11 +< Datafile length: 8192 Keyfile length: 8192 +--- +> Datafile length: 90112 Keyfile length: 204800 +========DIFF END======= +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +6c6 +< Status: checked,analyzed,optimized keys,sorted index pages +--- +> Status: changed +11c11 +< Datafile length: 8192 Keyfile length: 8192 +--- +> Datafile length: 90112 Keyfile length: 8192 +========DIFF END======= +TEST WITH ma_test1 -s -M -T -c -N -H1 --testflag=1 (commit at end) +TEST WITH ma_test1 -s -M -T -c -N -H1 --testflag=2 --test-undo=2 (additional aborted work) +Terminating after inserts +Dying on request without maria_commit()/maria_close() +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 16384 Keyfile length: 16384 +--- +> Datafile length: 16384 Keyfile length: 8192 +========DIFF END======= +testing idempotency +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 16384 Keyfile length: 16384 +--- +> Datafile length: 16384 Keyfile length: 8192 +========DIFF END======= +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 16384 Keyfile length: 16384 +--- +> Datafile length: 16384 Keyfile length: 8192 +18c18 +< 1 2 6 unique number NULL 0 8192 8192 +--- +> 1 2 6 unique number NULL 0 8192 +========DIFF END======= +TEST WITH ma_test1 -s -M -T -c -N -H2 --testflag=3 (commit at end) +Terminating after updates +TEST WITH ma_test1 -s -M -T -c -N -H2 --testflag=4 --test-undo=2 (additional aborted work) +Terminating after deletes +Dying on request without maria_commit()/maria_close() +applying log +testing idempotency +applying log +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 16384 Keyfile length: 16384 +--- +> Datafile length: 16384 Keyfile length: 8192 +18c18 +< 1 2 6 unique number NULL 0 8192 8192 +--- +> 1 2 6 unique number NULL 0 8192 +========DIFF END======= +TEST WITH ma_test1 -s -M -T -c -N -H2 --testflag=2 (commit at end) +Terminating after inserts +TEST WITH ma_test1 -s -M -T -c -N -H2 --testflag=3 --test-undo=2 (additional aborted work) +Terminating after updates +Dying on request without maria_commit()/maria_close() +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 16384 Keyfile length: 16384 +--- +> Datafile length: 16384 Keyfile length: 8192 +========DIFF END======= +testing idempotency +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 16384 Keyfile length: 16384 +--- +> Datafile length: 16384 Keyfile length: 8192 +========DIFF END======= +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 16384 Keyfile length: 16384 +--- +> Datafile length: 16384 Keyfile length: 8192 +18c18 +< 1 2 6 unique number NULL 0 8192 8192 +--- +> 1 2 6 unique number NULL 0 8192 +========DIFF END======= +TEST WITH ma_test2 -s -L -K -W -P -M -T -c -H1 -t1 (commit at end) +TEST WITH ma_test2 -s -L -K -W -P -M -T -c -H1 -t2 -u2 (additional aborted work) +Dying on request without maria_commit()/maria_close() +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +6c6 +< Status: checked,analyzed,optimized keys,sorted index pages +--- +> Status: changed +11c11 +< Datafile length: 8192 Keyfile length: 8192 +--- +> Datafile length: 90112 Keyfile length: 204800 +========DIFF END======= +testing idempotency +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +6c6 +< Status: checked,analyzed,optimized keys,sorted index pages +--- +> Status: changed +11c11 +< Datafile length: 8192 Keyfile length: 8192 +--- +> Datafile length: 90112 Keyfile length: 204800 +========DIFF END======= +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +6c6 +< Status: checked,analyzed,optimized keys,sorted index pages +--- +> Status: changed +11c11 +< Datafile length: 8192 Keyfile length: 8192 +--- +> Datafile length: 90112 Keyfile length: 8192 +========DIFF END======= +TEST WITH ma_test1 -s -M -T -c -N -H1 --testflag=1 (commit at end) +TEST WITH ma_test1 -s -M -T -c -N -H1 --testflag=2 --test-undo=3 (additional aborted work) +Terminating after inserts +Dying on request without maria_commit()/maria_close() +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 16384 Keyfile length: 16384 +--- +> Datafile length: 16384 Keyfile length: 8192 +========DIFF END======= +testing idempotency +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 16384 Keyfile length: 16384 +--- +> Datafile length: 16384 Keyfile length: 8192 +========DIFF END======= +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 16384 Keyfile length: 16384 +--- +> Datafile length: 16384 Keyfile length: 8192 +18c18 +< 1 2 6 unique number NULL 0 8192 8192 +--- +> 1 2 6 unique number NULL 0 8192 +========DIFF END======= +TEST WITH ma_test1 -s -M -T -c -N -H2 --testflag=3 (commit at end) +Terminating after updates +TEST WITH ma_test1 -s -M -T -c -N -H2 --testflag=4 --test-undo=3 (additional aborted work) +Terminating after deletes +Dying on request without maria_commit()/maria_close() +applying log +testing idempotency +applying log +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 16384 Keyfile length: 16384 +--- +> Datafile length: 16384 Keyfile length: 8192 +18c18 +< 1 2 6 unique number NULL 0 8192 8192 +--- +> 1 2 6 unique number NULL 0 8192 +========DIFF END======= +TEST WITH ma_test1 -s -M -T -c -N -H2 --testflag=2 (commit at end) +Terminating after inserts +TEST WITH ma_test1 -s -M -T -c -N -H2 --testflag=3 --test-undo=3 (additional aborted work) +Terminating after updates +Dying on request without maria_commit()/maria_close() +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 16384 Keyfile length: 16384 +--- +> Datafile length: 16384 Keyfile length: 8192 +========DIFF END======= +testing idempotency +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 16384 Keyfile length: 16384 +--- +> Datafile length: 16384 Keyfile length: 8192 +========DIFF END======= +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 16384 Keyfile length: 16384 +--- +> Datafile length: 16384 Keyfile length: 8192 +18c18 +< 1 2 6 unique number NULL 0 8192 8192 +--- +> 1 2 6 unique number NULL 0 8192 +========DIFF END======= +TEST WITH ma_test2 -s -L -K -W -P -M -T -c -H1 -t1 (commit at end) +TEST WITH ma_test2 -s -L -K -W -P -M -T -c -H1 -t2 -u3 (additional aborted work) +Dying on request without maria_commit()/maria_close() +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +6c6 +< Status: checked,analyzed,optimized keys,sorted index pages +--- +> Status: changed +11c11 +< Datafile length: 8192 Keyfile length: 8192 +--- +> Datafile length: 90112 Keyfile length: 204800 +========DIFF END======= +testing idempotency +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +6c6 +< Status: checked,analyzed,optimized keys,sorted index pages +--- +> Status: changed +11c11 +< Datafile length: 8192 Keyfile length: 8192 +--- +> Datafile length: 90112 Keyfile length: 204800 +========DIFF END======= +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +6c6 +< Status: checked,analyzed,optimized keys,sorted index pages +--- +> Status: changed +11c11 +< Datafile length: 8192 Keyfile length: 8192 +--- +> Datafile length: 90112 Keyfile length: 8192 +========DIFF END======= +TEST WITH ma_test1 -s -M -T -c -N -b -H1 --testflag=1 (commit at end) +TEST WITH ma_test1 -s -M -T -c -N -b -H1 --testflag=2 --test-undo=1 (additional aborted work) +Terminating after inserts +Dying on request without maria_commit()/maria_close() +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 49152 Keyfile length: 16384 +--- +> Datafile length: 49152 Keyfile length: 8192 +========DIFF END======= +testing idempotency +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 49152 Keyfile length: 16384 +--- +> Datafile length: 49152 Keyfile length: 8192 +========DIFF END======= +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 49152 Keyfile length: 16384 +--- +> Datafile length: 49152 Keyfile length: 8192 +18c18 +< 1 2 6 unique varchar BLOB NULL 0 8192 8192 +--- +> 1 2 6 unique varchar BLOB NULL 0 8192 +========DIFF END======= +TEST WITH ma_test1 -s -M -T -c -N -b -H2 --testflag=3 (commit at end) +Terminating after updates +TEST WITH ma_test1 -s -M -T -c -N -b -H2 --testflag=4 --test-undo=1 (additional aborted work) +Terminating after deletes +Dying on request without maria_commit()/maria_close() +applying log +testing idempotency +applying log +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 49152 Keyfile length: 16384 +--- +> Datafile length: 49152 Keyfile length: 8192 +18c18 +< 1 2 6 unique varchar BLOB NULL 0 8192 8192 +--- +> 1 2 6 unique varchar BLOB NULL 0 8192 +========DIFF END======= +TEST WITH ma_test1 -s -M -T -c -N -b -H2 --testflag=2 (commit at end) +Terminating after inserts +TEST WITH ma_test1 -s -M -T -c -N -b -H2 --testflag=3 --test-undo=1 (additional aborted work) +Terminating after updates +Dying on request without maria_commit()/maria_close() +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 49152 Keyfile length: 16384 +--- +> Datafile length: 49152 Keyfile length: 8192 +========DIFF END======= +testing idempotency +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 49152 Keyfile length: 16384 +--- +> Datafile length: 49152 Keyfile length: 8192 +========DIFF END======= +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 49152 Keyfile length: 16384 +--- +> Datafile length: 49152 Keyfile length: 8192 +18c18 +< 1 2 6 unique varchar BLOB NULL 0 8192 8192 +--- +> 1 2 6 unique varchar BLOB NULL 0 8192 +========DIFF END======= +TEST WITH ma_test2 -s -L -K -W -P -M -T -c -b -H1 -t1 (commit at end) +TEST WITH ma_test2 -s -L -K -W -P -M -T -c -b -H1 -t2 -u1 (additional aborted work) +Dying on request without maria_commit()/maria_close() +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +6c6 +< Status: checked,analyzed,optimized keys,sorted index pages +--- +> Status: changed +11c11 +< Datafile length: 8192 Keyfile length: 8192 +--- +> Datafile length: 81920 Keyfile length: 212992 +========DIFF END======= +testing idempotency +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +6c6 +< Status: checked,analyzed,optimized keys,sorted index pages +--- +> Status: changed +11c11 +< Datafile length: 8192 Keyfile length: 8192 +--- +> Datafile length: 81920 Keyfile length: 212992 +========DIFF END======= +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +6c6 +< Status: checked,analyzed,optimized keys,sorted index pages +--- +> Status: changed +11c11 +< Datafile length: 8192 Keyfile length: 8192 +--- +> Datafile length: 81920 Keyfile length: 8192 +========DIFF END======= +TEST WITH ma_test1 -s -M -T -c -N -b -H1 --testflag=1 (commit at end) +TEST WITH ma_test1 -s -M -T -c -N -b -H1 --testflag=2 --test-undo=2 (additional aborted work) +Terminating after inserts +Dying on request without maria_commit()/maria_close() +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 49152 Keyfile length: 16384 +--- +> Datafile length: 49152 Keyfile length: 8192 +========DIFF END======= +testing idempotency +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 49152 Keyfile length: 16384 +--- +> Datafile length: 49152 Keyfile length: 8192 +========DIFF END======= +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 49152 Keyfile length: 16384 +--- +> Datafile length: 49152 Keyfile length: 8192 +18c18 +< 1 2 6 unique varchar BLOB NULL 0 8192 8192 +--- +> 1 2 6 unique varchar BLOB NULL 0 8192 +========DIFF END======= +TEST WITH ma_test1 -s -M -T -c -N -b -H2 --testflag=3 (commit at end) +Terminating after updates +TEST WITH ma_test1 -s -M -T -c -N -b -H2 --testflag=4 --test-undo=2 (additional aborted work) +Terminating after deletes +Dying on request without maria_commit()/maria_close() +applying log +testing idempotency +applying log +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 49152 Keyfile length: 16384 +--- +> Datafile length: 49152 Keyfile length: 8192 +18c18 +< 1 2 6 unique varchar BLOB NULL 0 8192 8192 +--- +> 1 2 6 unique varchar BLOB NULL 0 8192 +========DIFF END======= +TEST WITH ma_test1 -s -M -T -c -N -b -H2 --testflag=2 (commit at end) +Terminating after inserts +TEST WITH ma_test1 -s -M -T -c -N -b -H2 --testflag=3 --test-undo=2 (additional aborted work) +Terminating after updates +Dying on request without maria_commit()/maria_close() +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 49152 Keyfile length: 16384 +--- +> Datafile length: 49152 Keyfile length: 8192 +========DIFF END======= +testing idempotency +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 49152 Keyfile length: 16384 +--- +> Datafile length: 49152 Keyfile length: 8192 +========DIFF END======= +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 49152 Keyfile length: 16384 +--- +> Datafile length: 49152 Keyfile length: 8192 +18c18 +< 1 2 6 unique varchar BLOB NULL 0 8192 8192 +--- +> 1 2 6 unique varchar BLOB NULL 0 8192 +========DIFF END======= +TEST WITH ma_test2 -s -L -K -W -P -M -T -c -b -H1 -t1 (commit at end) +TEST WITH ma_test2 -s -L -K -W -P -M -T -c -b -H1 -t2 -u2 (additional aborted work) +Dying on request without maria_commit()/maria_close() +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +6c6 +< Status: checked,analyzed,optimized keys,sorted index pages +--- +> Status: changed +11c11 +< Datafile length: 8192 Keyfile length: 8192 +--- +> Datafile length: 81920 Keyfile length: 212992 +========DIFF END======= +testing idempotency +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +6c6 +< Status: checked,analyzed,optimized keys,sorted index pages +--- +> Status: changed +11c11 +< Datafile length: 8192 Keyfile length: 8192 +--- +> Datafile length: 81920 Keyfile length: 212992 +========DIFF END======= +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +6c6 +< Status: checked,analyzed,optimized keys,sorted index pages +--- +> Status: changed +11c11 +< Datafile length: 8192 Keyfile length: 8192 +--- +> Datafile length: 81920 Keyfile length: 8192 +========DIFF END======= +TEST WITH ma_test1 -s -M -T -c -N -b -H1 --testflag=1 (commit at end) +TEST WITH ma_test1 -s -M -T -c -N -b -H1 --testflag=2 --test-undo=3 (additional aborted work) +Terminating after inserts +Dying on request without maria_commit()/maria_close() +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 49152 Keyfile length: 16384 +--- +> Datafile length: 49152 Keyfile length: 8192 +========DIFF END======= +testing idempotency +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 49152 Keyfile length: 16384 +--- +> Datafile length: 49152 Keyfile length: 8192 +========DIFF END======= +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 49152 Keyfile length: 16384 +--- +> Datafile length: 49152 Keyfile length: 8192 +18c18 +< 1 2 6 unique varchar BLOB NULL 0 8192 8192 +--- +> 1 2 6 unique varchar BLOB NULL 0 8192 +========DIFF END======= +TEST WITH ma_test1 -s -M -T -c -N -b -H2 --testflag=3 (commit at end) +Terminating after updates +TEST WITH ma_test1 -s -M -T -c -N -b -H2 --testflag=4 --test-undo=3 (additional aborted work) +Terminating after deletes +Dying on request without maria_commit()/maria_close() +applying log +testing idempotency +applying log +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 49152 Keyfile length: 16384 +--- +> Datafile length: 49152 Keyfile length: 8192 +18c18 +< 1 2 6 unique varchar BLOB NULL 0 8192 8192 +--- +> 1 2 6 unique varchar BLOB NULL 0 8192 +========DIFF END======= +TEST WITH ma_test1 -s -M -T -c -N -b -H2 --testflag=2 (commit at end) +Terminating after inserts +TEST WITH ma_test1 -s -M -T -c -N -b -H2 --testflag=3 --test-undo=3 (additional aborted work) +Terminating after updates +Dying on request without maria_commit()/maria_close() +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 49152 Keyfile length: 16384 +--- +> Datafile length: 49152 Keyfile length: 8192 +========DIFF END======= +testing idempotency +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 49152 Keyfile length: 16384 +--- +> Datafile length: 49152 Keyfile length: 8192 +========DIFF END======= +testing applying of CLRs to recreate table +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +11c11 +< Datafile length: 49152 Keyfile length: 16384 +--- +> Datafile length: 49152 Keyfile length: 8192 +18c18 +< 1 2 6 unique varchar BLOB NULL 0 8192 8192 +--- +> 1 2 6 unique varchar BLOB NULL 0 8192 +========DIFF END======= +TEST WITH ma_test2 -s -L -K -W -P -M -T -c -b -H1 -t1 (commit at end) +TEST WITH ma_test2 -s -L -K -W -P -M -T -c -b -H1 -t2 -u3 (additional aborted work) +Dying on request without maria_commit()/maria_close() +applying log +Differences in maria_chk -dvv, recovery not yet perfect ! +========DIFF START======= +6c6 +< Status: checked,analyzed,optimized keys,sorted index pages +--- +> Status: changed +11c11 +< Datafile length: 8192 Keyfile length: 8192 +--- > Datafile length: 155648 Keyfile length: 212992 ========DIFF END======= testing idempotency diff --git a/storage/maria/maria_read_log.c b/storage/maria/maria_read_log.c index 6a565e8f20a..b8449c4d769 100644 --- a/storage/maria/maria_read_log.c +++ b/storage/maria/maria_read_log.c @@ -29,7 +29,8 @@ const char *default_dbug_option= "d:t:i:O,\\maria_read_log.trace"; const char *default_dbug_option= "d:t:i:o,/tmp/maria_read_log.trace"; #endif #endif /* DBUG_OFF */ -static my_bool opt_only_display, opt_apply, opt_apply_undo, opt_silent; +static my_bool opt_only_display, opt_apply, opt_apply_undo, opt_silent, + opt_check; static ulong opt_page_buffer_size; static const char *my_progname_short; @@ -102,7 +103,9 @@ int main(int argc, char **argv) LSN_IN_PARTS(lsn)); fprintf(stdout, "TRACE of the last maria_read_log\n"); - if (maria_apply_log(lsn, opt_apply, opt_silent ? NULL : stdout, + if (maria_apply_log(lsn, opt_apply ? MARIA_LOG_APPLY : + (opt_check ? MARIA_LOG_CHECK : + MARIA_LOG_DISPLAY_HEADER), opt_silent ? NULL : stdout, opt_apply_undo, FALSE, FALSE)) goto err; fprintf(stdout, "%s: SUCCESS\n", my_progname_short); @@ -130,6 +133,10 @@ static struct my_option my_long_options[] = "Apply log to tables. Will display a lot of information if not run with --silent", (uchar **) &opt_apply, (uchar **) &opt_apply, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, + {"check", 'c', + "if --only-display, check if record is fully readable (for debugging)", + (uchar **) &opt_check, (uchar **) &opt_check, 0, + GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, #ifndef DBUG_OFF {"debug", '#', "Output debug log. Often the argument is 'd:t:o,filename'.", 0, 0, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0}, diff --git a/strings/llstr.c b/strings/llstr.c index 12aea63e014..678f8b05f39 100644 --- a/strings/llstr.c +++ b/strings/llstr.c @@ -32,3 +32,10 @@ char *llstr(longlong value,char *buff) longlong10_to_str(value,buff,-10); return buff; } + +char *ullstr(longlong value,char *buff) +{ + longlong10_to_str(value,buff,10); + return buff; +} + |