diff options
author | Michael Widenius <monty@askmonty.org> | 2011-07-01 15:08:30 +0300 |
---|---|---|
committer | Michael Widenius <monty@askmonty.org> | 2011-07-01 15:08:30 +0300 |
commit | 3c78bfe7f1a1530daea83f5dae091bd654ea6d37 (patch) | |
tree | b54abedc9d4f6ac33b7aa2128454fcd37d739d78 /storage/maria | |
parent | 62e47b4402678abc85a689a820d201248dabbc36 (diff) | |
download | mariadb-git-3c78bfe7f1a1530daea83f5dae091bd654ea6d37.tar.gz |
Added progress reporting for alter table, LOAD DATA INFILE and for aria tables: check table, repair table, analyze table.
- The client gets a progress report message that triggers a callback function if requested with mysql_options(MYSQL_PROGRESS_CALLBACK, function)
- Added Progress field last to 'show processlist'
- Stage, Max_stage and Progress field added to information_schema.progresslist
- The 'mysql' client by defaults enables progress reports when the output is a tty.
- Added progress_report_time time variable to configure how often progress reports is sent to client
Added read only system variable 'in_transaction' which is 1 if we have executed a BEGIN statement.
client/client_priv.h:
Added OPT_REPORT_PROGRESS
client/mysql.cc:
Added option --progress-reports (on by default if not batch mode)
Progress reports is written to stdout for long running commands
include/Makefile.am:
Added mysql/service_progress_report.h
include/myisamchk.h:
Added variables to be able to do progress reporting in Aria and later in MyISAM
include/mysql.h:
Added new mysql_options() parameter: MYSQL_PROGRESS_CALLBACK
include/mysql.h.pp:
Added new mysql_options() parameter: MYSQL_PROGRESS_CALLBACK
include/mysql/plugin.h:
Added functions for reporting progress.
include/mysql/plugin_auth.h.pp:
Added functions for reporting progress.
include/mysql_com.h:
Added CLIENT_PROGRESS mysql_real_connect() flag.
include/sql_common.h:
Added callback function for reporting progress
mysql-test/r/old-mode.result:
Ensure that SHOW PROGRESSLIST doesn't have the Progress column in old mode.
mysql-test/suite/funcs_1/datadict/datadict_priv.inc:
Added new column
mysql-test/suite/funcs_1/datadict/processlist_priv.inc:
Test all new PROCESSLIST columns
mysql-test/suite/funcs_1/r/is_columns_is.result:
Updated results
mysql-test/suite/funcs_1/r/is_columns_is_embedded.result:
Updated results
mysql-test/suite/funcs_1/r/is_columns_mysql_embedded.result:
Updated results
mysql-test/suite/funcs_1/r/is_tables_is_embedded.result:
Updated results
mysql-test/suite/funcs_1/r/processlist_priv_no_prot.result:
Updated results
mysql-test/suite/funcs_1/r/processlist_priv_ps.result:
Updated results
mysql-test/suite/funcs_1/r/processlist_val_no_prot.result:
Updated results
mysql-test/suite/funcs_1/r/processlist_val_ps.result:
Updated results
mysql-test/suite/pbxt/r/pbxt_locking.result:
Updated results
mysql-test/suite/pbxt/r/skip_name_resolve.result:
Updated results
mysql-test/t/old-mode.test:
Ensure that SHOW PROGRESSLIST doesn't have the Progress column in old mode.
plugin/handler_socket/handlersocket/Makefile.am:
Added -lmysqlservices
scripts/mytop.sh:
Made 'State' field width dynamic.
Added 'Progress' to process list display.
sql-common/client.c:
Added handling of progress messages.
Removed check_license() function.
sql/mysql_priv.h:
Added opt_progress_report_time
sql/mysqld.cc:
Added progress_report_time time variable to configure how often progress reports is sent to client
sql/protocol.cc:
Added net_send_progress_packet()
sql/protocol.h:
New prototypes
sql/set_var.cc:
Added variables progress_report_time and in_transaction
sql/sql_acl.cc:
Safety fix: Made client_capabilities ulonglong
sql/sql_class.cc:
Added interface functions for progress reporting
sql/sql_class.h:
Added varibles in THD for progress reporting.
Added CF_REPORT_PROGRESS
sql/sql_load.cc:
Added progress reporting for LOAD DATA INFILE
sql/sql_parse.cc:
Added CF_REPORT_PROGRESS for top level commands for which it's safe to send progress reports to client
sql/sql_show.cc:
Added Progress field last to 'show processlist'
Stage, Max_stage and Progress field added to information_schema.progresslist
sql/sql_table.cc:
Added progress reporting for ALTER TABLE
Added THD as argument to copy_data_between_tables()
storage/maria/ha_maria.cc:
Added progress reporting for check table, repair table, analyze table
Fixed a bug in start_bulk_insert() that caused alter table to always run with all keys enabled.
storage/maria/ma_check.c:
Added progress reporting
Remember old state before starting repair. This removes some warnings from optimize_table if create-with-sort fails.
storage/maria/ma_check_standalone.h:
Added dummy reporting function for standalone Aria programs.
storage/maria/ma_sort.c:
Added progress reporting
storage/maria/maria_chk.c:
Updated version
storage/maria/maria_def.h:
Added new prototypes
tests/mysql_client_test.c:
Added test case for progress reporting
Diffstat (limited to 'storage/maria')
-rw-r--r-- | storage/maria/ha_maria.cc | 73 | ||||
-rw-r--r-- | storage/maria/ma_check.c | 46 | ||||
-rw-r--r-- | storage/maria/ma_check_standalone.h | 7 | ||||
-rw-r--r-- | storage/maria/ma_sort.c | 25 | ||||
-rw-r--r-- | storage/maria/maria_chk.c | 2 | ||||
-rw-r--r-- | storage/maria/maria_def.h | 2 |
6 files changed, 137 insertions, 18 deletions
diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index 57045ff2cd3..88995980f3d 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -711,6 +711,34 @@ int _ma_killed_ptr(HA_CHECK *param) } +/* + Report progress to mysqld + + This is a bit more complex than what a normal progress report + function normally is. + + The reason is that this is called by enable_index/repair which + is one stage in ALTER TABLE and we can't use the external + stage/max_stage for this. + + thd_progress_init/thd_progress_next_stage is to be called by + high level commands like CHECK TABLE or REPAIR TABLE, not + by sub commands like enable_index(). + + In ma_check.c it's easier to work with stages than with a total + progress, so we use internal stage/max_stage here to keep the + code simple. +*/ + +void _ma_report_progress(HA_CHECK *param, ulonglong progress, + ulonglong max_progress) +{ + thd_progress_report((THD*)param->thd, + progress + max_progress * param->stage, + max_progress * param->max_stage); +} + + void _ma_check_print_error(HA_CHECK *param, const char *fmt, ...) { va_list args; @@ -1104,7 +1132,7 @@ int ha_maria::check(THD * thd, HA_CHECK_OPT * check_opt) int error; HA_CHECK ¶m= *(HA_CHECK*) thd->alloc(sizeof(param)); MARIA_SHARE *share= file->s; - const char *old_proc_info= thd_proc_info(thd, "Checking table"); + const char *old_proc_info; TRN *old_trn= file->trn; if (!file || !¶m) return HA_ADMIN_INTERNAL_ERROR; @@ -1132,12 +1160,18 @@ int ha_maria::check(THD * thd, HA_CHECK_OPT * check_opt) return HA_ADMIN_ALREADY_DONE; maria_chk_init_for_check(¶m, file); + old_proc_info= thd_proc_info(thd, "Checking status"); + thd_progress_init(thd, 3); (void) maria_chk_status(¶m, file); // Not fatal error= maria_chk_size(¶m, file); if (!error) error|= maria_chk_del(¶m, file, param.testflag); + thd_proc_info(thd, "Checking keys"); + thd_progress_next_stage(thd); if (!error) error= maria_chk_key(¶m, file); + thd_proc_info(thd, "Checking data"); + thd_progress_next_stage(thd); if (!error) { if ((!(param.testflag & T_QUICK) && @@ -1188,6 +1222,7 @@ int ha_maria::check(THD * thd, HA_CHECK_OPT * check_opt) /* Reset trn, that may have been set by repair */ _ma_set_trn_for_table(file, old_trn); thd_proc_info(thd, old_proc_info); + thd_progress_end(thd); return error ? HA_ADMIN_CORRUPT : HA_ADMIN_OK; } @@ -1203,6 +1238,7 @@ int ha_maria::analyze(THD *thd, HA_CHECK_OPT * check_opt) int error= 0; HA_CHECK ¶m= *(HA_CHECK*) thd->alloc(sizeof(param)); MARIA_SHARE *share= file->s; + const char *old_proc_info; if (!¶m) return HA_ADMIN_INTERNAL_ERROR; @@ -1220,6 +1256,8 @@ int ha_maria::analyze(THD *thd, HA_CHECK_OPT * check_opt) if (!(share->state.changed & STATE_NOT_ANALYZED)) return HA_ADMIN_ALREADY_DONE; + old_proc_info= thd_proc_info(thd, "Scanning"); + thd_progress_init(thd, 1); error= maria_chk_key(¶m, file); if (!error) { @@ -1229,6 +1267,8 @@ int ha_maria::analyze(THD *thd, HA_CHECK_OPT * check_opt) } else if (!maria_is_crashed(file) && !thd->killed) maria_mark_crashed(file); + thd_proc_info(thd, old_proc_info); + thd_progress_end(thd); return error ? HA_ADMIN_CORRUPT : HA_ADMIN_OK; } @@ -1362,6 +1402,7 @@ int ha_maria::repair(THD * thd, HA_CHECK_OPT *check_opt) int error; HA_CHECK ¶m= *(HA_CHECK*) thd->alloc(sizeof(param)); ha_rows start_records; + const char *old_proc_info; if (!file || !¶m) return HA_ADMIN_INTERNAL_ERROR; @@ -1375,6 +1416,8 @@ int ha_maria::repair(THD * thd, HA_CHECK_OPT *check_opt) param.sort_buffer_length= THDVAR(thd, sort_buffer_size); param.backup_time= check_opt->start_time; start_records= file->state->records; + old_proc_info= thd_proc_info(thd, "Checking table"); + thd_progress_init(thd, 1); while ((error= repair(thd, ¶m, 0)) && param.retry_repair) { param.retry_repair= 0; @@ -1410,6 +1453,8 @@ int ha_maria::repair(THD * thd, HA_CHECK_OPT *check_opt) llstr(start_records, llbuff2), table->s->path.str); } + thd_proc_info(thd, old_proc_info); + thd_progress_end(thd); return error; } @@ -1457,14 +1502,15 @@ int ha_maria::optimize(THD * thd, HA_CHECK_OPT *check_opt) param.testflag= (check_opt->flags | T_SILENT | T_FORCE_CREATE | T_REP_BY_SORT | T_STATISTICS | T_SORT_INDEX); param.sort_buffer_length= THDVAR(thd, sort_buffer_size); + thd_progress_init(thd, 1); if ((error= repair(thd, ¶m, 1)) && param.retry_repair) { sql_print_warning("Warning: Optimize table got errno %d on %s.%s, retrying", my_errno, param.db_name, param.table_name); param.testflag &= ~T_REP_BY_SORT; - error= repair(thd, ¶m, 1); + error= repair(thd, ¶m, 0); } - + thd_progress_end(thd); return error; } @@ -1638,6 +1684,7 @@ int ha_maria::repair(THD *thd, HA_CHECK *param, bool do_optimize) } pthread_mutex_unlock(&share->intern_lock); thd_proc_info(thd, old_proc_info); + thd_progress_end(thd); // Mark done if (!thd->locked_tables) maria_lock_database(file, F_UNLCK); @@ -1999,19 +2046,21 @@ void ha_maria::start_bulk_insert(ha_rows rows) we don't want to update the key statistics based of only a few rows. Index file rebuild requires an exclusive lock, so if versioning is on don't do it (see how ha_maria::store_lock() tries to predict repair). - We can repair index only if we have an exclusive (TL_WRITE) lock. To - see if table is empty, we shouldn't rely on the old records' count from - our transaction's start (if that old count is 0 but now there are - records in the table, we would wrongly destroy them). - So we need to look at share->state.state.records. - As a safety net for now, we don't remove the test of - file->state->records, because there is uncertainty on what will happen - during repair if the two states disagree. + We can repair index only if we have an exclusive (TL_WRITE) lock or + if this is inside an ALTER TABLE, in which case lock_type == TL_UNLOCK. + + To see if table is empty, we shouldn't rely on the old record + count from our transaction's start (if that old count is 0 but + now there are records in the table, we would wrongly destroy + them). So we need to look at share->state.state.records. As a + safety net for now, we don't remove the test of + file->state->records, because there is uncertainty on what will + happen during repair if the two states disagree. */ if ((file->state->records == 0) && (share->state.state.records == 0) && can_enable_indexes && (!rows || rows >= MARIA_MIN_ROWS_TO_DISABLE_INDEXES) && - (file->lock.type == TL_WRITE)) + (file->lock.type == TL_WRITE || file->lock.type == TL_UNLOCK)) { /** @todo for a single-row INSERT SELECT, we will go into repair, which diff --git a/storage/maria/ma_check.c b/storage/maria/ma_check.c index d39e9f494ad..a50a588e8ac 100644 --- a/storage/maria/ma_check.c +++ b/storage/maria/ma_check.c @@ -125,6 +125,7 @@ void maria_chk_init(HA_CHECK *param) param->max_record_length= LONGLONG_MAX; param->pagecache_block_size= KEY_CACHE_BLOCK_SIZE; param->stats_method= MI_STATS_METHOD_NULLS_NOT_EQUAL; + param->max_stage= 1; } @@ -517,6 +518,7 @@ int maria_chk_key(HA_CHECK *param, register MARIA_HA *info) continue; } found_keys++; + _ma_report_progress(param, key, share->base.keys); param->record_checksum=init_checksum; @@ -1123,10 +1125,14 @@ static int check_keys_in_record(HA_CHECK *param, MARIA_HA *info, int extend, param->tmp_record_checksum+= (ha_checksum) start_recpos; param->records++; - if (param->testflag & T_WRITE_LOOP && param->records % WRITE_COUNT == 0) + if (param->records % WRITE_COUNT == 0) { - printf("%s\r", llstr(param->records, llbuff)); - VOID(fflush(stdout)); + if (param->testflag & T_WRITE_LOOP) + { + printf("%s\r", llstr(param->records, llbuff)); + VOID(fflush(stdout)); + } + _ma_report_progress(param, param->records, share->state.state.records); } /* Check if keys match the record */ @@ -2355,6 +2361,7 @@ static int initialize_variables_for_repair(HA_CHECK *param, /* calculate max_records */ sort_info->filelength= my_seek(info->dfile.file, 0L, MY_SEEK_END, MYF(0)); + param->max_progress= sort_info->filelength; if ((param->testflag & T_CREATE_MISSING_KEYS) || sort_info->org_data_file_type == COMPRESSED_RECORD) sort_info->max_records= share->state.state.records; @@ -2377,6 +2384,8 @@ static int initialize_variables_for_repair(HA_CHECK *param, maria_ignore_trids(info); /* Don't write transid's during repair */ maria_versioning(info, 0); + /* remember original number of rows */ + *info->state= info->s->state.state; return 0; } @@ -3609,7 +3618,7 @@ int maria_repair_by_sort(HA_CHECK *param, register MARIA_HA *info, const char * name, my_bool rep_quick) { int got_error; - uint i; + uint i, keys_to_repair; ha_rows start_records; my_off_t new_header_length, org_header_length, del; File new_file; @@ -3735,6 +3744,17 @@ int maria_repair_by_sort(HA_CHECK *param, register MARIA_HA *info, del=share->state.state.del; + /* Calculate number of keys to repair */ + keys_to_repair= 0; + for (sort_param.key=0 ; sort_param.key < share->base.keys ; + sort_param.key++) + { + if (maria_is_key_active(key_map, sort_param.key)) + keys_to_repair++; + } + /* For each key we scan and merge sort the keys */ + param->max_stage= keys_to_repair*2; + rec_per_key_part= param->new_rec_per_key_part; for (sort_param.key=0 ; sort_param.key < share->base.keys ; rec_per_key_part+=sort_param.keyinfo->keysegs, sort_param.key++) @@ -3855,6 +3875,9 @@ int maria_repair_by_sort(HA_CHECK *param, register MARIA_HA *info, /* Set for next loop */ sort_info.max_records= (ha_rows) sort_info.new_info->s->state.state.records; + param->stage++; /* Next stage */ + param->progress= 0; + if (param->testflag & T_STATISTICS) maria_update_key_parts(sort_param.keyinfo, rec_per_key_part, sort_param.unique, @@ -3935,6 +3958,10 @@ int maria_repair_by_sort(HA_CHECK *param, register MARIA_HA *info, sort_info.org_data_file_type= share->data_file_type; sort_info.filelength= share->state.state.data_file_length; sort_param.fix_datafile=0; + + /* Offsets are now in proportion to the new file length */ + param->max_progress= sort_info.filelength; + } else share->state.state.data_file_length=sort_param.max_pos; @@ -4735,6 +4762,11 @@ static int sort_get_next_record(MARIA_SORT_PARAM *sort_param) if (_ma_killed_ptr(param)) DBUG_RETURN(1); + if (param->progress_counter++ >= WRITE_COUNT) + { + param->progress_counter= 0; + _ma_report_progress(param, param->progress, param->max_progress); + } switch (sort_info->org_data_file_type) { case BLOCK_RECORD: @@ -4775,6 +4807,9 @@ static int sort_get_next_record(MARIA_SORT_PARAM *sort_param) flag= HA_ERR_ROW_NOT_VISIBLE; } } + param->progress= (ma_recordpos_to_page(info->cur_row.lastpos)* + share->block_size); + share->page_type= save_page_type; if (!flag) { @@ -4827,6 +4862,7 @@ static int sort_get_next_record(MARIA_SORT_PARAM *sort_param) DBUG_RETURN(-1); } sort_param->start_recpos=sort_param->pos; + param->progress= sort_param->pos; if (!sort_param->fix_datafile) { sort_param->current_filepos= sort_param->pos; @@ -4854,6 +4890,7 @@ static int sort_get_next_record(MARIA_SORT_PARAM *sort_param) LINT_INIT(to); pos=sort_param->pos; + param->progress= pos; searching=(sort_param->fix_datafile && (param->testflag & T_EXTEND)); parallel_flag= (sort_param->read_cache.file < 0) ? READING_NEXT : 0; for (;;) @@ -5163,6 +5200,7 @@ static int sort_get_next_record(MARIA_SORT_PARAM *sort_param) } } case COMPRESSED_RECORD: + param->progress= sort_param->pos; for (searching=0 ;; searching=1, sort_param->pos++) { if (_ma_read_cache(info, &sort_param->read_cache, block_info.header, diff --git a/storage/maria/ma_check_standalone.h b/storage/maria/ma_check_standalone.h index 8cda285bb99..f9242bbe123 100644 --- a/storage/maria/ma_check_standalone.h +++ b/storage/maria/ma_check_standalone.h @@ -35,6 +35,13 @@ int _ma_killed_ptr(HA_CHECK *param __attribute__((unused))) return 0; } + +void _ma_report_progress(HA_CHECK *param __attribute__((unused)), + ulonglong progress __attribute__((unused)), + ulonglong max_progress __attribute__((unused))) +{ +} + /* print warnings and errors */ /* VARARGS */ diff --git a/storage/maria/ma_sort.c b/storage/maria/ma_sort.c index a36bb980145..7bfb53ca0a1 100644 --- a/storage/maria/ma_sort.c +++ b/storage/maria/ma_sort.c @@ -191,6 +191,9 @@ int _ma_create_index_by_sort(MARIA_SORT_PARAM *info, my_bool no_messages, &tempfile,&tempfile_for_exceptions)) == HA_POS_ERROR) goto err; /* purecov: tested */ + + info->sort_info->param->stage++; /* Merge stage */ + if (maxbuffer == 0) { if (!no_messages) @@ -769,6 +772,8 @@ static int write_index(MARIA_SORT_PARAM *info, if ((*info->key_write)(info, *sort_keys++)) DBUG_RETURN(-1); /* purecov: inspected */ } + if (info->sort_info->param->max_stage != 1) /* If not parallel */ + _ma_report_progress(info->sort_info->param, 1, 1); DBUG_RETURN(0); } /* write_index */ @@ -779,7 +784,7 @@ static int merge_many_buff(MARIA_SORT_PARAM *info, uint keys, uchar **sort_keys, BUFFPEK *buffpek, int *maxbuffer, IO_CACHE *t_file) { - register int i; + int tmp, merges, max_merges; IO_CACHE t_file2, *from_file, *to_file, *temp; BUFFPEK *lastbuff; DBUG_ENTER("merge_many_buff"); @@ -791,9 +796,21 @@ static int merge_many_buff(MARIA_SORT_PARAM *info, uint keys, DISK_BUFFER_SIZE, info->sort_info->param->myf_rw)) DBUG_RETURN(1); /* purecov: inspected */ + /* Calculate how many merges are needed */ + max_merges= 1; /* Count merge_index */ + tmp= *maxbuffer; + while (tmp >= MERGEBUFF2) + { + merges= (tmp-MERGEBUFF*3/2 + 1) / MERGEBUFF + 1; + max_merges+= merges; + tmp= merges; + } + merges= 0; + from_file= t_file ; to_file= &t_file2; while (*maxbuffer >= MERGEBUFF2) { + int i; reinit_io_cache(from_file,READ_CACHE,0L,0,0); reinit_io_cache(to_file,WRITE_CACHE,0L,0,0); lastbuff=buffpek; @@ -802,6 +819,8 @@ static int merge_many_buff(MARIA_SORT_PARAM *info, uint keys, if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++, buffpek+i,buffpek+i+MERGEBUFF-1)) goto cleanup; + if (info->sort_info->param->max_stage != 1) /* If not parallel */ + _ma_report_progress(info->sort_info->param, merges++, max_merges); } if (merge_buffers(info,keys,from_file,to_file,sort_keys,lastbuff++, buffpek+i,buffpek+ *maxbuffer)) @@ -810,6 +829,8 @@ static int merge_many_buff(MARIA_SORT_PARAM *info, uint keys, break; /* purecov: inspected */ temp=from_file; from_file=to_file; to_file=temp; *maxbuffer= (int) (lastbuff-buffpek)-1; + if (info->sort_info->param->max_stage != 1) /* If not parallel */ + _ma_report_progress(info->sort_info->param, merges++, max_merges); } cleanup: close_cached_file(to_file); /* This holds old result */ @@ -1066,6 +1087,8 @@ merge_index(MARIA_SORT_PARAM *info, uint keys, uchar **sort_keys, if (merge_buffers(info,keys,tempfile,(IO_CACHE*) 0,sort_keys,buffpek,buffpek, buffpek+maxbuffer)) DBUG_RETURN(1); /* purecov: inspected */ + if (info->sort_info->param->max_stage != 1) /* If not parallel */ + _ma_report_progress(info->sort_info->param, 1, 1); DBUG_RETURN(0); } /* merge_index */ diff --git a/storage/maria/maria_chk.c b/storage/maria/maria_chk.c index 23692ca0382..24e747c584e 100644 --- a/storage/maria/maria_chk.c +++ b/storage/maria/maria_chk.c @@ -436,7 +436,7 @@ static struct my_option my_long_options[] = static void print_version(void) { - printf("%s Ver 1.0 for %s at %s\n", my_progname, SYSTEM_TYPE, + printf("%s Ver 1.1 for %s at %s\n", my_progname, SYSTEM_TYPE, MACHINE_TYPE); NETWARE_SET_SCREEN_MODE(1); } diff --git a/storage/maria/maria_def.h b/storage/maria/maria_def.h index eeb72c83287..eb74b5c5477 100644 --- a/storage/maria/maria_def.h +++ b/storage/maria/maria_def.h @@ -1220,6 +1220,8 @@ int _ma_flush_table_files(MARIA_HA *info, uint flush_data_or_index, See ma_check_standalone.h . */ int _ma_killed_ptr(HA_CHECK *param); +void _ma_report_progress(HA_CHECK *param, ulonglong progress, + ulonglong max_progress); void _ma_check_print_error _VARARGS((HA_CHECK *param, const char *fmt, ...)) ATTRIBUTE_FORMAT(printf, 2, 3); void _ma_check_print_warning _VARARGS((HA_CHECK *param, const char *fmt, ...)) |