diff options
Diffstat (limited to 'storage/innobase/row/row0merge.cc')
| -rw-r--r-- | storage/innobase/row/row0merge.cc | 478 |
1 files changed, 408 insertions, 70 deletions
diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index ec9b8f79e49..356eec8e7ee 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -24,6 +24,9 @@ New index creation routines using a merge sort Created 12/4/2005 Jan Lindstrom Completed by Sunny Bains and Marko Makela *******************************************************/ +#include <my_config.h> +#include <log.h> +#include <sql_class.h> #include "row0merge.h" #include "row0ext.h" @@ -39,6 +42,14 @@ Completed by Sunny Bains and Marko Makela #include "row0import.h" #include "handler0alter.h" #include "ha_prototypes.h" +#include "math.h" /* log() */ +#include "fil0crypt.h" + +float my_log2f(float n) +{ + /* log(n) / log(2) is log2. */ + return (float)(log((double)n) / log((double)2)); +} /* Ignore posix_fadvise() on those platforms where it does not exist */ #if defined __WIN__ @@ -186,7 +197,7 @@ row_merge_buf_create( ulint buf_size; mem_heap_t* heap; - max_tuples = srv_sort_buf_size + max_tuples = (srv_sort_buf_size) / ut_max(1, dict_index_get_min_size(index)); buf_size = (sizeof *buf); @@ -598,8 +609,8 @@ row_merge_buf_add( ut_ad(data_size < srv_sort_buf_size); - /* Reserve one byte for the end marker of row_merge_block_t. */ - if (buf->total_size + data_size >= srv_sort_buf_size - 1) { + /* Reserve bytes for the end marker of row_merge_block_t. */ + if (buf->total_size + data_size >= srv_sort_buf_size) { DBUG_RETURN(0); } @@ -720,7 +731,7 @@ UT_SORT_FUNCTION_BODY(). /**********************************************************************//** Merge sort the tuple buffer in main memory. */ -static MY_ATTRIBUTE((nonnull(4,5))) +static void row_merge_tuple_sort( /*=================*/ @@ -840,17 +851,19 @@ row_merge_heap_create( Read a merge block from the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_read( /*===========*/ int fd, /*!< in: file descriptor */ ulint offset, /*!< in: offset where to read in number of row_merge_block_t elements */ - row_merge_block_t* buf) /*!< out: data */ + row_merge_block_t* buf, /*!< out: data */ + row_merge_block_t* crypt_buf, /*!< in: crypt buf or NULL */ + ulint space) /*!< in: space id */ { os_offset_t ofs = ((os_offset_t) offset) * srv_sort_buf_size; - ibool success; + bool success; DBUG_EXECUTE_IF("row_merge_read_failure", return(FALSE);); @@ -864,6 +877,17 @@ row_merge_read( success = os_file_read_no_error_handling_int_fd(fd, buf, ofs, srv_sort_buf_size); + /* If encryption is enabled decrypt buffer */ + if (success && log_tmp_is_encrypted()) { + if (!log_tmp_block_decrypt(buf, srv_sort_buf_size, + crypt_buf, ofs, space)) { + return (FALSE); + } + + srv_stats.n_merge_blocks_decrypted.inc(); + memcpy(buf, crypt_buf, srv_sort_buf_size); + } + #ifdef POSIX_FADV_DONTNEED /* Each block is read exactly once. Free up the file cache. */ posix_fadvise(fd, ofs, srv_sort_buf_size, POSIX_FADV_DONTNEED); @@ -883,21 +907,35 @@ row_merge_read( Write a merge block to the file system. @return TRUE if request was successful, FALSE if fail */ UNIV_INTERN -ibool +bool row_merge_write( /*============*/ - int fd, /*!< in: file descriptor */ - ulint offset, /*!< in: offset where to write, - in number of row_merge_block_t elements */ - const void* buf) /*!< in: data */ + int fd, /*!< in: file descriptor */ + ulint offset, /*!< in: offset where to write, + in number of row_merge_block_t elements */ + const void* buf, /*!< in: data */ + void* crypt_buf, /*!< in: crypt buf or NULL */ + ulint space) /*!< in: space id */ { size_t buf_len = srv_sort_buf_size; os_offset_t ofs = buf_len * (os_offset_t) offset; - ibool ret; + bool ret; + void* out_buf = (void *)buf; DBUG_EXECUTE_IF("row_merge_write_failure", return(FALSE);); - ret = os_file_write_int_fd("(merge)", fd, buf, ofs, buf_len); + /* For encrypted tables, encrypt data before writing */ + if (log_tmp_is_encrypted()) { + if (!log_tmp_block_encrypt((const byte *)buf, buf_len, + (byte *)crypt_buf, ofs, space)) { + return (FALSE); + } + + srv_stats.n_merge_blocks_encrypted.inc(); + out_buf = crypt_buf; + } + + ret = os_file_write_int_fd("(merge)", fd, out_buf, ofs, buf_len); #ifdef UNIV_DEBUG if (row_merge_print_block_write) { @@ -931,7 +969,9 @@ row_merge_read_rec( const mrec_t** mrec, /*!< out: pointer to merge record, or NULL on end of list (non-NULL on I/O error) */ - ulint* offsets)/*!< out: offsets of mrec */ + ulint* offsets,/*!< out: offsets of mrec */ + row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ + ulint space) /*!< in: space id */ { ulint extra_size; ulint data_size; @@ -963,7 +1003,9 @@ row_merge_read_rec( /* Read another byte of extra_size. */ if (UNIV_UNLIKELY(b >= &block[srv_sort_buf_size])) { - if (!row_merge_read(fd, ++(*foffs), block)) { + if (!row_merge_read(fd, ++(*foffs), block, + crypt_block, + space)) { err_exit: /* Signal I/O error. */ *mrec = b; @@ -992,7 +1034,9 @@ err_exit: ut_ad(avail_size < sizeof *buf); memcpy(*buf, b, avail_size); - if (!row_merge_read(fd, ++(*foffs), block)) { + if (!row_merge_read(fd, ++(*foffs), block, + crypt_block, + space)) { goto err_exit; } @@ -1053,7 +1097,9 @@ err_exit: offsets[3] = (ulint) index; #endif /* UNIV_DEBUG */ - if (!row_merge_read(fd, ++(*foffs), block)) { + if (!row_merge_read(fd, ++(*foffs), block, + crypt_block, + space)) { goto err_exit; } @@ -1135,7 +1181,9 @@ row_merge_write_rec( int fd, /*!< in: file descriptor */ ulint* foffs, /*!< in/out: file offset */ const mrec_t* mrec, /*!< in: record to write */ - const ulint* offsets)/*!< in: offsets of mrec */ + const ulint* offsets,/*!< in: offsets of mrec */ + row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ + ulint space) /*!< in: space id */ { ulint extra_size; ulint size; @@ -1170,7 +1218,9 @@ row_merge_write_rec( record to the head of the new block. */ memcpy(b, buf[0], avail_size); - if (!row_merge_write(fd, (*foffs)++, block)) { + if (!row_merge_write(fd, (*foffs)++, block, + crypt_block, + space)) { return(NULL); } @@ -1196,10 +1246,12 @@ static byte* row_merge_write_eof( /*================*/ - row_merge_block_t* block, /*!< in/out: file buffer */ - byte* b, /*!< in: pointer to end of block */ - int fd, /*!< in: file descriptor */ - ulint* foffs) /*!< in/out: file offset */ + row_merge_block_t* block, /*!< in/out: file buffer */ + byte* b, /*!< in: pointer to end of block */ + int fd, /*!< in: file descriptor */ + ulint* foffs, /*!< in/out: file offset */ + row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ + ulint space) /*!< in: space id */ { ut_ad(block); ut_ad(b >= &block[0]); @@ -1215,17 +1267,21 @@ row_merge_write_eof( *b++ = 0; UNIV_MEM_ASSERT_RW(&block[0], b - &block[0]); UNIV_MEM_ASSERT_W(&block[0], srv_sort_buf_size); + #ifdef UNIV_DEBUG_VALGRIND /* The rest of the block is uninitialized. Initialize it to avoid bogus warnings. */ memset(b, 0xff, &block[srv_sort_buf_size] - b); #endif /* UNIV_DEBUG_VALGRIND */ - if (!row_merge_write(fd, (*foffs)++, block)) { + if (!row_merge_write(fd, (*foffs)++, block, + crypt_block, + space)) { return(NULL); } UNIV_MEM_INVALID(&block[0], srv_sort_buf_size); + return(&block[0]); } @@ -1294,11 +1350,13 @@ containing the index entries for the indexes to be built. NULL if old_table == new_table @param[in] add_autoinc number of added AUTO_INCREMENT columns, or ULINT_UNDEFINED if none is added -@param[in,out] sequence autoinc sequence -@param[in,out] block file buffer -@param[in,out] tmpfd temporary file handle -return DB_SUCCESS or error */ -static MY_ATTRIBUTE((nonnull(1,2,3,4,6,9,10,16), warn_unused_result)) +@param[in,out] sequence autoinc sequence +@param[in,out] block file buffer +@param[in,out] tmpfd temporary file handle +@param[in] pct_cost percent of task weight out of total alter job +@param[in,out] crypt_block crypted file buffer +@return DB_SUCCESS or error */ +static MY_ATTRIBUTE((warn_unused_result)) dberr_t row_merge_read_clustered_index( trx_t* trx, @@ -1317,7 +1375,9 @@ row_merge_read_clustered_index( ulint add_autoinc, ib_sequence_t& sequence, row_merge_block_t* block, - int* tmpfd) + int* tmpfd, + float pct_cost, + row_merge_block_t* crypt_block) { dict_index_t* clust_index; /* Clustered index */ mem_heap_t* row_heap; /* Heap memory to create @@ -1338,11 +1398,22 @@ row_merge_read_clustered_index( ibool fts_pll_sort = FALSE; ib_int64_t sig_count = 0; mem_heap_t* conv_heap = NULL; + + float curr_progress = 0.0; + ib_int64_t read_rows = 0; + ib_int64_t table_total_rows = 0; + DBUG_ENTER("row_merge_read_clustered_index"); ut_ad((old_table == new_table) == !col_map); ut_ad(!add_cols || col_map); + table_total_rows = dict_table_get_n_rows(old_table); + if(table_total_rows == 0) { + /* We don't know total row count */ + table_total_rows = 1; + } + trx->op_info = "reading clustered index"; #ifdef FTS_INTERNAL_DIAG_PRINT @@ -1451,6 +1522,14 @@ row_merge_read_clustered_index( mem_heap_empty(row_heap); + /* Do not continue if table pages are still encrypted */ + if (!old_table->is_readable() || + !new_table->is_readable()) { + err = DB_DECRYPTION_FAILED; + trx->error_key_num = 0; + goto func_exit; + } + page_cur_move_to_next(cur); if (page_cur_is_after_last(cur)) { @@ -1832,7 +1911,8 @@ write_buffers: row_merge_buf_write(buf, file, block); if (!row_merge_write(file->fd, file->offset++, - block)) { + block, crypt_block, + new_table->space)) { err = DB_TEMP_FILE_WRITE_FAILURE; trx->error_key_num = i; break; @@ -1840,6 +1920,7 @@ write_buffers: } UNIV_MEM_INVALID(&block[0], srv_sort_buf_size); + merge_buf[i] = row_merge_buf_empty(buf); if (UNIV_LIKELY(row != NULL)) { @@ -1874,6 +1955,17 @@ write_buffers: if (err != DB_SUCCESS) { goto func_exit; } + + /* Increment innodb_onlineddl_pct_progress status variable */ + read_rows++; + if(read_rows % 1000 == 0) { + /* Update progress for each 1000 rows */ + curr_progress = (read_rows >= table_total_rows) ? + pct_cost : + ((pct_cost * read_rows) / table_total_rows); + /* presenting 10.12% as 1012 integer */ + onlineddl_pct_progress = curr_progress * 100; + } } func_exit: @@ -1997,14 +2089,19 @@ wait_again: b2 = row_merge_write_rec(&block[2 * srv_sort_buf_size], \ &buf[2], b2, \ of->fd, &of->offset, \ - mrec##N, offsets##N); \ + mrec##N, offsets##N, \ + crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL , \ + space); \ if (UNIV_UNLIKELY(!b2 || ++of->n_rec > file->n_rec)) { \ goto corrupt; \ } \ b##N = row_merge_read_rec(&block[N * srv_sort_buf_size],\ &buf[N], b##N, INDEX, \ file->fd, foffs##N, \ - &mrec##N, offsets##N); \ + &mrec##N, offsets##N, \ + crypt_block ? &crypt_block[N * srv_sort_buf_size] : NULL, \ + space); \ + \ if (UNIV_UNLIKELY(!b##N)) { \ if (mrec##N) { \ goto corrupt; \ @@ -2016,7 +2113,7 @@ wait_again: /*************************************************************//** Merge two blocks of records on disk and write a bigger block. @return DB_SUCCESS or error code */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) +static __attribute__((warn_unused_result)) dberr_t row_merge_blocks( /*=============*/ @@ -2029,7 +2126,10 @@ row_merge_blocks( source list in the file */ ulint* foffs1, /*!< in/out: offset of second source list in the file */ - merge_file_t* of) /*!< in/out: output file */ + merge_file_t* of, /*!< in/out: output file */ + row_merge_block_t* crypt_block,/*!< in: in/out: crypted file + buffer */ + ulint space) /*!< in: space id */ { mem_heap_t* heap; /*!< memory heap for offsets0, offsets1 */ @@ -2061,8 +2161,12 @@ row_merge_blocks( /* Write a record and read the next record. Split the output file in two halves, which can be merged on the following pass. */ - if (!row_merge_read(file->fd, *foffs0, &block[0]) - || !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size])) { + if (!row_merge_read(file->fd, *foffs0, &block[0], + crypt_block ? &crypt_block[0] : NULL, + space) || + !row_merge_read(file->fd, *foffs1, &block[srv_sort_buf_size], + crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, + space)) { corrupt: mem_heap_free(heap); return(DB_CORRUPTION); @@ -2074,11 +2178,17 @@ corrupt: b0 = row_merge_read_rec( &block[0], &buf[0], b0, dup->index, - file->fd, foffs0, &mrec0, offsets0); + file->fd, foffs0, &mrec0, offsets0, + crypt_block ? &crypt_block[0] : NULL, + space); + b1 = row_merge_read_rec( &block[srv_sort_buf_size], &buf[srv_sort_buf_size], b1, dup->index, - file->fd, foffs1, &mrec1, offsets1); + file->fd, foffs1, &mrec1, offsets1, + crypt_block ? &crypt_block[srv_sort_buf_size] : NULL, + space); + if (UNIV_UNLIKELY(!b0 && mrec0) || UNIV_UNLIKELY(!b1 && mrec1)) { @@ -2120,23 +2230,29 @@ done0: done1: mem_heap_free(heap); + b2 = row_merge_write_eof(&block[2 * srv_sort_buf_size], - b2, of->fd, &of->offset); + b2, of->fd, &of->offset, + crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, + space); + return(b2 ? DB_SUCCESS : DB_CORRUPTION); } /*************************************************************//** Copy a block of index entries. @return TRUE on success, FALSE on failure */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) -ibool +static __attribute__((warn_unused_result)) +bool row_merge_blocks_copy( /*==================*/ const dict_index_t* index, /*!< in: index being created */ const merge_file_t* file, /*!< in: input file */ row_merge_block_t* block, /*!< in/out: 3 buffers */ ulint* foffs0, /*!< in/out: input file offset */ - merge_file_t* of) /*!< in/out: output file */ + merge_file_t* of, /*!< in/out: output file */ + row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ + ulint space) /*!< in: space id */ { mem_heap_t* heap; /*!< memory heap for offsets0, offsets1 */ @@ -2163,7 +2279,9 @@ row_merge_blocks_copy( /* Write a record and read the next record. Split the output file in two halves, which can be merged on the following pass. */ - if (!row_merge_read(file->fd, *foffs0, &block[0])) { + if (!row_merge_read(file->fd, *foffs0, &block[0], + crypt_block ? &crypt_block[0] : NULL, + space)) { corrupt: mem_heap_free(heap); return(FALSE); @@ -2174,7 +2292,10 @@ corrupt: b2 = &block[2 * srv_sort_buf_size]; b0 = row_merge_read_rec(&block[0], &buf[0], b0, index, - file->fd, foffs0, &mrec0, offsets0); + file->fd, foffs0, &mrec0, offsets0, + crypt_block ? &crypt_block[0] : NULL, + space); + if (UNIV_UNLIKELY(!b0 && mrec0)) { goto corrupt; @@ -2193,15 +2314,18 @@ done0: (*foffs0)++; mem_heap_free(heap); + return(row_merge_write_eof(&block[2 * srv_sort_buf_size], - b2, of->fd, &of->offset) + b2, of->fd, &of->offset, + crypt_block ? &crypt_block[2 * srv_sort_buf_size] : NULL, + space) != NULL); } /*************************************************************//** Merge disk files. @return DB_SUCCESS or error code */ -static MY_ATTRIBUTE((nonnull)) +static dberr_t row_merge( /*======*/ @@ -2214,9 +2338,11 @@ row_merge( int* tmpfd, /*!< in/out: temporary file handle */ ulint* num_run,/*!< in/out: Number of runs remain to be merged */ - ulint* run_offset) /*!< in/out: Array contains the + ulint* run_offset, /*!< in/out: Array contains the first offset number for each merge run */ + row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ + ulint space) /*!< in: space id */ { ulint foffs0; /*!< first input offset */ ulint foffs1; /*!< second input offset */ @@ -2229,6 +2355,10 @@ row_merge( UNIV_MEM_ASSERT_W(&block[0], 3 * srv_sort_buf_size); + if (crypt_block) { + UNIV_MEM_ASSERT_W(&crypt_block[0], 3 * srv_sort_buf_size); + } + ut_ad(ihalf < file->offset); of.fd = *tmpfd; @@ -2259,7 +2389,9 @@ row_merge( run_offset[n_run++] = of.offset; error = row_merge_blocks(dup, file, block, - &foffs0, &foffs1, &of); + &foffs0, &foffs1, &of, + crypt_block, + space); if (error != DB_SUCCESS) { return(error); @@ -2279,7 +2411,9 @@ row_merge( run_offset[n_run++] = of.offset; if (!row_merge_blocks_copy(dup->index, file, block, - &foffs0, &of)) { + &foffs0, &of, + crypt_block, + space)) { return(DB_CORRUPTION); } } @@ -2296,7 +2430,9 @@ row_merge( run_offset[n_run++] = of.offset; if (!row_merge_blocks_copy(dup->index, file, block, - &foffs1, &of)) { + &foffs1, &of, + crypt_block, + space)) { return(DB_CORRUPTION); } } @@ -2343,17 +2479,38 @@ row_merge_sort( merge_file_t* file, /*!< in/out: file containing index entries */ row_merge_block_t* block, /*!< in/out: 3 buffers */ - int* tmpfd) /*!< in/out: temporary file handle */ + int* tmpfd, /*!< in/out: temporary file handle + */ + const bool update_progress, + /*!< in: update progress + status variable or not */ + const float pct_progress, + /*!< in: total progress percent + until now */ + const float pct_cost, /*!< in: current progress percent */ + row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ + ulint space) /*!< in: space id */ { const ulint half = file->offset / 2; ulint num_runs; ulint* run_offset; dberr_t error = DB_SUCCESS; + ulint merge_count = 0; + ulint total_merge_sort_count; + float curr_progress = 0; + DBUG_ENTER("row_merge_sort"); /* Record the number of merge runs we need to perform */ num_runs = file->offset; + /* Find the number N which 2^N is greater or equal than num_runs */ + /* N is merge sort running count */ + total_merge_sort_count = ceil(my_log2f(num_runs)); + if(total_merge_sort_count <= 0) { + total_merge_sort_count=1; + } + /* If num_runs are less than 1, nothing to merge */ if (num_runs <= 1) { DBUG_RETURN(error); @@ -2381,20 +2538,37 @@ row_merge_sort( } #endif /* UNIV_SOLARIS */ + if (global_system_variables.log_warnings > 2) { + sql_print_information("InnoDB: Online DDL : merge-sorting" + " has estimated " ULINTPF " runs", + num_runs); + } + /* Merge the runs until we have one big run */ do { - error = row_merge(trx, dup, file, block, tmpfd, - &num_runs, run_offset); - /* Report progress of merge sort to MySQL for - show processlist progress field only for - "normal" indexes. */ + show processlist progress field */ + /* Progress report only for "normal" indexes. */ #ifndef UNIV_SOLARIS if (!(dup->index->type & DICT_FTS)) { thd_progress_report(trx->mysql_thd, file->offset - num_runs, file->offset); } #endif /* UNIV_SOLARIS */ + error = row_merge(trx, dup, file, block, tmpfd, + &num_runs, run_offset, + crypt_block, + space); + + if(update_progress) { + merge_count++; + curr_progress = (merge_count >= total_merge_sort_count) ? + pct_cost : + ((pct_cost * merge_count) / total_merge_sort_count); + /* presenting 10.12% as 1012 integer */; + onlineddl_pct_progress = (pct_progress + curr_progress) * 100; + } + if (error != DB_SUCCESS) { break; } @@ -2404,6 +2578,7 @@ row_merge_sort( mem_free(run_offset); + /* Progress report only for "normal" indexes. */ #ifndef UNIV_SOLARIS if (!(dup->index->type & DICT_FTS)) { thd_progress_end(trx->mysql_thd); @@ -2460,7 +2635,7 @@ row_merge_copy_blobs( Read sorted file containing index data tuples and insert these data tuples to the index @return DB_SUCCESS or error number */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) +static __attribute__((warn_unused_result)) dberr_t row_merge_insert_index_tuples( /*==========================*/ @@ -2468,7 +2643,13 @@ row_merge_insert_index_tuples( dict_index_t* index, /*!< in: index */ const dict_table_t* old_table,/*!< in: old table */ int fd, /*!< in: file descriptor */ - row_merge_block_t* block) /*!< in/out: file buffer */ + row_merge_block_t* block, /*!< in/out: file buffer */ + const ib_int64_t table_total_rows, /*!< in: total rows of old table */ + const float pct_progress, /*!< in: total progress percent until now */ + const float pct_cost, /*!< in: current progress percent + */ + row_merge_block_t* crypt_block, /*!< in: crypt buf or NULL */ + ulint space) /*!< in: space id */ { const byte* b; mem_heap_t* heap; @@ -2478,6 +2659,8 @@ row_merge_insert_index_tuples( ulint foffs = 0; ulint* offsets; mrec_buf_t* buf; + ib_int64_t inserted_rows = 0; + float curr_progress; DBUG_ENTER("row_merge_insert_index_tuples"); ut_ad(!srv_read_only_mode); @@ -2497,9 +2680,11 @@ row_merge_insert_index_tuples( offsets[1] = dict_index_get_n_fields(index); } - b = block; + b = &block[0]; - if (!row_merge_read(fd, foffs, block)) { + if (!row_merge_read(fd, foffs, block, + crypt_block, + space)) { error = DB_CORRUPTION; } else { buf = static_cast<mrec_buf_t*>( @@ -2515,7 +2700,10 @@ row_merge_insert_index_tuples( mtr_t mtr; b = row_merge_read_rec(block, buf, b, index, - fd, &foffs, &mrec, offsets); + fd, &foffs, &mrec, offsets, + crypt_block, + space); + if (UNIV_UNLIKELY(!b)) { /* End of list, or I/O error */ if (mrec) { @@ -2654,6 +2842,19 @@ row_merge_insert_index_tuples( mem_heap_empty(tuple_heap); mem_heap_empty(ins_heap); + + /* Increment innodb_onlineddl_pct_progress status variable */ + inserted_rows++; + if(inserted_rows % 1000 == 0) { + /* Update progress for each 1000 rows */ + curr_progress = (inserted_rows >= table_total_rows || + table_total_rows <= 0) ? + pct_cost : + ((pct_cost * inserted_rows) / table_total_rows); + + /* presenting 10.12% as 1012 integer */; + onlineddl_pct_progress = (pct_progress + curr_progress) * 100; + } } } @@ -3190,7 +3391,7 @@ row_merge_file_create( if (merge_file->fd >= 0) { if (srv_disable_sort_file_cache) { - os_file_set_nocache(merge_file->fd, + os_file_set_nocache((os_file_t)merge_file->fd, "row0merge.cc", "sort"); } } @@ -3429,7 +3630,7 @@ row_merge_rename_tables_dict( table is in a non-system tablespace where space > 0. */ if (err == DB_SUCCESS && old_table->space != TRX_SYS_SPACE - && !old_table->ibd_file_missing) { + && fil_space_get(old_table->space) != NULL) { /* Make pathname to update SYS_DATAFILES. */ char* tmp_path = row_make_new_pathname(old_table, tmp_name); @@ -3664,6 +3865,7 @@ row_merge_build_indexes( { merge_file_t* merge_files; row_merge_block_t* block; + row_merge_block_t* crypt_block = NULL; ulint block_size; ulint i; ulint j; @@ -3674,6 +3876,13 @@ row_merge_build_indexes( fts_psort_t* merge_info = NULL; ib_int64_t sig_count = 0; bool fts_psort_initiated = false; + + float total_static_cost = 0; + float total_dynamic_cost = 0; + uint total_index_blocks = 0; + float pct_cost=0; + float pct_progress=0; + DBUG_ENTER("row_merge_build_indexes"); ut_ad(!srv_read_only_mode); @@ -3691,6 +3900,17 @@ row_merge_build_indexes( DBUG_RETURN(DB_OUT_OF_MEMORY); } + /* If temporal log file is encrypted allocate memory for + encryption/decryption. */ + if (log_tmp_is_encrypted()) { + crypt_block = static_cast<row_merge_block_t*>( + os_mem_alloc_large(&block_size)); + + if (crypt_block == NULL) { + DBUG_RETURN(DB_OUT_OF_MEMORY); + } + } + trx_start_if_not_started_xa(trx); merge_files = static_cast<merge_file_t*>( @@ -3702,8 +3922,12 @@ row_merge_build_indexes( for (i = 0; i < n_indexes; i++) { merge_files[i].fd = -1; + merge_files[i].offset = 0; } + total_static_cost = COST_BUILD_INDEX_STATIC * n_indexes + COST_READ_CLUSTERED_INDEX; + total_dynamic_cost = COST_BUILD_INDEX_DYNAMIC * n_indexes; + for (i = 0; i < n_indexes; i++) { if (indexes[i]->type & DICT_FTS) { @@ -3737,6 +3961,27 @@ row_merge_build_indexes( duplicate keys. */ innobase_rec_reset(table); + if (global_system_variables.log_warnings > 2) { + sql_print_information("InnoDB: Online DDL : Start reading" + " clustered index of the table" + " and create temporary files"); + } + + pct_cost = COST_READ_CLUSTERED_INDEX * 100 / (total_static_cost + total_dynamic_cost); + + /* Do not continue if we can't encrypt table pages */ + if (!old_table->is_readable() || + !new_table->is_readable()) { + error = DB_DECRYPTION_FAILED; + ib_push_warning(trx->mysql_thd, DB_DECRYPTION_FAILED, + "Table %s is encrypted but encryption service or" + " used key_id is not available. " + " Can't continue reading table.", + !old_table->is_readable() ? old_table->name : + new_table->name); + goto func_exit; + } + /* Read clustered index of the table and create files for secondary index entries for merge sort */ @@ -3744,10 +3989,22 @@ row_merge_build_indexes( trx, table, old_table, new_table, online, indexes, fts_sort_idx, psort_info, merge_files, key_numbers, n_indexes, add_cols, col_map, - add_autoinc, sequence, block, &tmpfd); + add_autoinc, sequence, block, &tmpfd, + pct_cost, crypt_block); - if (error != DB_SUCCESS) { + pct_progress += pct_cost; + if (global_system_variables.log_warnings > 2) { + sql_print_information("InnoDB: Online DDL : End of reading " + "clustered index of the table" + " and create temporary files"); + } + + for (i = 0; i < n_indexes; i++) { + total_index_blocks += merge_files[i].offset; + } + + if (error != DB_SUCCESS) { goto func_exit; } @@ -3833,17 +4090,89 @@ wait_again: DEBUG_FTS_SORT_PRINT("FTS_SORT: Complete Insert\n"); #endif } else if (merge_files[i].fd != -1) { + char buf[NAME_LEN + 1]; row_merge_dup_t dup = { sort_idx, table, col_map, 0}; + pct_cost = (COST_BUILD_INDEX_STATIC + + (total_dynamic_cost * merge_files[i].offset / + total_index_blocks)) / + (total_static_cost + total_dynamic_cost) + * PCT_COST_MERGESORT_INDEX * 100; + char* bufend = innobase_convert_name( + buf, sizeof buf, + indexes[i]->name, + strlen(indexes[i]->name), + trx->mysql_thd, + FALSE); + buf[bufend - buf]='\0'; + + if (global_system_variables.log_warnings > 2) { + sql_print_information("InnoDB: Online DDL :" + " Start merge-sorting" + " index %s" + " (" ULINTPF + " / " ULINTPF ")," + " estimated cost :" + " %2.4f", + buf, i + 1, n_indexes, + pct_cost); + } + error = row_merge_sort( - trx, &dup, &merge_files[i], - block, &tmpfd); + trx, &dup, &merge_files[i], + block, &tmpfd, true, + pct_progress, pct_cost, + crypt_block, + new_table->space); + + pct_progress += pct_cost; + + if (global_system_variables.log_warnings > 2) { + sql_print_information("InnoDB: Online DDL :" + " End of " + " merge-sorting index %s" + " (" ULINTPF + " / " ULINTPF ")", + buf, i + 1, n_indexes); + } + + DBUG_EXECUTE_IF( + "ib_merge_wait_after_sort", + os_thread_sleep(20000000);); /* 20 sec */ if (error == DB_SUCCESS) { + pct_cost = (COST_BUILD_INDEX_STATIC + + (total_dynamic_cost * merge_files[i].offset / + total_index_blocks)) / + (total_static_cost + total_dynamic_cost) * + PCT_COST_INSERT_INDEX * 100; + + if (global_system_variables.log_warnings > 2) { + sql_print_information( + "InnoDB: Online DDL : Start " + "building index %s" + " (" ULINTPF + " / " ULINTPF "), estimated " + "cost : %2.4f", buf, i + 1, + n_indexes, pct_cost); + } + error = row_merge_insert_index_tuples( trx->id, sort_idx, old_table, - merge_files[i].fd, block); + merge_files[i].fd, block, + merge_files[i].n_rec, pct_progress, pct_cost, + crypt_block, new_table->space); + + pct_progress += pct_cost; + + if (global_system_variables.log_warnings > 2) { + sql_print_information( + "InnoDB: Online DDL : " + "End of building index %s" + " (" ULINTPF " / " ULINTPF ")", + buf, i + 1, n_indexes); + } } } @@ -3860,6 +4189,11 @@ wait_again: ut_ad(sort_idx->online_status == ONLINE_INDEX_COMPLETE); } else { + if (global_system_variables.log_warnings > 2) { + sql_print_information( + "InnoDB: Online DDL : Applying" + " log to index"); + } DEBUG_SYNC_C("row_log_apply_before"); error = row_log_apply(trx, sort_idx, table); DEBUG_SYNC_C("row_log_apply_after"); @@ -3908,6 +4242,10 @@ func_exit: mem_free(merge_files); os_mem_free_large(block, block_size); + if (crypt_block) { + os_mem_free_large(crypt_block, block_size); + } + DICT_TF2_FLAG_UNSET(new_table, DICT_TF2_FTS_ADD_DOC_ID); if (online && old_table == new_table && error != DB_SUCCESS) { |