diff options
author | Marko Mäkelä <marko.makela@mariadb.com> | 2017-06-05 22:47:20 +0300 |
---|---|---|
committer | Marko Mäkelä <marko.makela@mariadb.com> | 2017-06-29 23:15:05 +0300 |
commit | 84e4e4506ffee95d71f1cac916176e9b64ba1bd8 (patch) | |
tree | 3612230402a916a68cca89a089d00a5f8966d28b | |
parent | e903d458bb5267df4c37a6862587c403aa87dc35 (diff) | |
download | mariadb-git-84e4e4506ffee95d71f1cac916176e9b64ba1bd8.tar.gz |
Reduce the granularity of innodb_log_file_size
In Mariabackup, we would want the backed-up redo log file size to be
a multiple of 512 bytes, or OS_FILE_LOG_BLOCK_SIZE. However, at startup,
InnoDB would be picky, requiring the file size to be a multiple of
innodb_page_size.
Furthermore, InnoDB would require the parameter to be a multiple of
one megabyte, while the minimum granularity is 512 bytes. Because
the data-file-oriented fil_io() API is being used for writing the
InnoDB redo log, writes will for now require innodb_log_file_size to
be a multiple of the maximum innodb_page_size (65536 bytes).
To complicate matters, InnoDB startup divided srv_log_file_size by
UNIV_PAGE_SIZE, so that initially, the unit was bytes, and later it
was innodb_page_size. We will simplify this and keep srv_log_file_size
in bytes at all times.
innobase_log_file_size: Remove. Remove some obsolete checks against
overflow on 32-bit systems. srv_log_file_size is always 64 bits, and
the maximum size 512GiB in multiples of innodb_page_size always fits
in ulint (which is 32 or 64 bits). 512GiB would be 8,388,608*64KiB or
134,217,728*4KiB.
log_init(): Remove the parameter file_size that was always passed as
srv_log_file_size.
log_set_capacity(): Add a parameter for passing the requested file size.
srv_log_file_size_requested: Declare static in srv0start.cc.
create_log_file(), create_log_files(),
innobase_start_or_create_for_mysql(): Invoke fil_node_create()
with srv_log_file_size expressed in multiples of innodb_page_size.
innobase_start_or_create_for_mysql(): Require the redo log file sizes
to be multiples of 512 bytes.
-rw-r--r-- | mysql-test/suite/innodb/r/log_file.result | 2 | ||||
-rw-r--r-- | mysql-test/suite/innodb/r/log_file_size.result | 8 | ||||
-rw-r--r-- | mysql-test/suite/innodb/t/log_file.test | 2 | ||||
-rw-r--r-- | mysql-test/suite/innodb/t/log_file_size.test | 8 | ||||
-rw-r--r-- | storage/innobase/handler/ha_innodb.cc | 10 | ||||
-rw-r--r-- | storage/innobase/include/log0log.h | 8 | ||||
-rw-r--r-- | storage/innobase/include/srv0srv.h | 14 | ||||
-rw-r--r-- | storage/innobase/log/log0log.cc | 14 | ||||
-rw-r--r-- | storage/innobase/srv/srv0srv.cc | 16 | ||||
-rw-r--r-- | storage/innobase/srv/srv0start.cc | 74 |
10 files changed, 59 insertions, 97 deletions
diff --git a/mysql-test/suite/innodb/r/log_file.result b/mysql-test/suite/innodb/r/log_file.result index f0f8007cb09..b93bfc0d02b 100644 --- a/mysql-test/suite/innodb/r/log_file.result +++ b/mysql-test/suite/innodb/r/log_file.result @@ -340,7 +340,7 @@ WHERE engine='innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); 1 1 -FOUND 1 /Resizing redo log from 1\*\d+ to 3\*\d+ pages; LSN=\d+/ in mysqld.1.err +FOUND 1 /Resizing redo log from 1\*\d+ to 3\*\d+ bytes; LSN=\d+/ in mysqld.1.err # Cleanup bak_ib_logfile0 bak_ib_logfile1 diff --git a/mysql-test/suite/innodb/r/log_file_size.result b/mysql-test/suite/innodb/r/log_file_size.result index 9e0cb4dbbf7..b576061e74b 100644 --- a/mysql-test/suite/innodb/r/log_file_size.result +++ b/mysql-test/suite/innodb/r/log_file_size.result @@ -33,16 +33,16 @@ ERROR 42000: Unknown storage engine 'InnoDB' FOUND 1 /InnoDB: innodb_read_only prevents crash recovery/ in mysqld.1.err SELECT * FROM t1; ERROR 42000: Unknown storage engine 'InnoDB' -FOUND 2 /redo log from 3\*[0-9]+ to 2\*[0-9]+ pages/ in mysqld.1.err +FOUND 2 /redo log from 3\*[0-9]+ to 2\*[0-9]+ bytes/ in mysqld.1.err SELECT * FROM t1; ERROR 42000: Unknown storage engine 'InnoDB' -FOUND 3 /redo log from 3\*[0-9]+ to 2\*[0-9]+ pages/ in mysqld.1.err +FOUND 3 /redo log from 3\*[0-9]+ to 2\*[0-9]+ bytes/ in mysqld.1.err SELECT * FROM t1; ERROR 42000: Unknown storage engine 'InnoDB' FOUND 2 /InnoDB: innodb_read_only prevents crash recovery/ in mysqld.1.err SELECT * FROM t1; ERROR 42000: Unknown storage engine 'InnoDB' -FOUND 4 /redo log from 3\*[0-9]+ to 2\*[0-9]+ pages/ in mysqld.1.err +FOUND 4 /redo log from 3\*[0-9]+ to 2\*[0-9]+ bytes/ in mysqld.1.err SELECT * FROM t1; ERROR 42000: Unknown storage engine 'InnoDB' SELECT * FROM t1; @@ -56,7 +56,7 @@ ERROR 42000: Unknown storage engine 'InnoDB' FOUND 1 /InnoDB: Setting log file .*ib_logfile[0-9]+ size to/ in mysqld.1.err SELECT * FROM t1; ERROR 42000: Unknown storage engine 'InnoDB' -FOUND 1 /InnoDB: Log file .*ib_logfile0 size 7 is not a multiple of innodb_page_size/ in mysqld.1.err +FOUND 1 /InnoDB: Log file .*ib_logfile0 size 7 is not a multiple of 512 bytes/ in mysqld.1.err SELECT * FROM t1; ERROR 42000: Unknown storage engine 'InnoDB' FOUND 1 /InnoDB: Log file .*ib_logfile1 is of different size 1048576 bytes than other log files/ in mysqld.1.err diff --git a/mysql-test/suite/innodb/t/log_file.test b/mysql-test/suite/innodb/t/log_file.test index f5d46b1db10..8a82ab7f29f 100644 --- a/mysql-test/suite/innodb/t/log_file.test +++ b/mysql-test/suite/innodb/t/log_file.test @@ -218,7 +218,7 @@ eval $check_no_innodb; --source include/start_mysqld.inc eval $check_yes_innodb; --source include/shutdown_mysqld.inc ---let SEARCH_PATTERN=Resizing redo log from 1\*\d+ to 3\*\d+ pages; LSN=\d+ +--let SEARCH_PATTERN=Resizing redo log from 1\*\d+ to 3\*\d+ bytes; LSN=\d+ --source include/search_pattern_in_file.inc --let $restart_parameters= diff --git a/mysql-test/suite/innodb/t/log_file_size.test b/mysql-test/suite/innodb/t/log_file_size.test index 14cc889e34b..2a7ca9e0e45 100644 --- a/mysql-test/suite/innodb/t/log_file_size.test +++ b/mysql-test/suite/innodb/t/log_file_size.test @@ -100,14 +100,14 @@ let SEARCH_PATTERN= InnoDB: innodb_read_only prevents crash recovery; --source include/restart_mysqld.inc --error ER_UNKNOWN_STORAGE_ENGINE SELECT * FROM t1; -let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ pages; +let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ bytes; --source include/search_pattern_in_file.inc --let $restart_parameters= --debug=d,innodb_log_abort_5 --source include/restart_mysqld.inc --error ER_UNKNOWN_STORAGE_ENGINE SELECT * FROM t1; -let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ pages; +let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ bytes; --source include/search_pattern_in_file.inc --let $restart_parameters= --innodb-read-only @@ -122,7 +122,7 @@ let SEARCH_PATTERN= InnoDB: innodb_read_only prevents crash recovery; --error ER_UNKNOWN_STORAGE_ENGINE SELECT * FROM t1; -let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ pages; +let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ bytes; --source include/search_pattern_in_file.inc --let $restart_parameters= --debug=d,innodb_log_abort_7 @@ -170,7 +170,7 @@ EOF --source include/start_mysqld.inc --error ER_UNKNOWN_STORAGE_ENGINE SELECT * FROM t1; -let SEARCH_PATTERN= InnoDB: Log file .*ib_logfile0 size 7 is not a multiple of innodb_page_size; +let SEARCH_PATTERN= InnoDB: Log file .*ib_logfile0 size 7 is not a multiple of 512 bytes; --source include/search_pattern_in_file.inc --remove_file $MYSQLD_DATADIR/ib_logfile0 --move_file $MYSQLD_DATADIR/ib_logfile101 $MYSQLD_DATADIR/ib_logfile0 diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index d8a4d8d2742..56465ae1257 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -212,7 +212,7 @@ static ulong innobase_commit_concurrency = 0; static ulong innobase_read_io_threads; static ulong innobase_write_io_threads; -static long long innobase_buffer_pool_size, innobase_log_file_size; +static long long innobase_buffer_pool_size; /** Percentage of the buffer pool to reserve for 'old' blocks. Connected to buf_LRU_old_ratio. */ @@ -4240,8 +4240,6 @@ innobase_change_buffering_inited_ok: srv_file_flush_method_str = innobase_file_flush_method; - srv_log_file_size = (ib_uint64_t) innobase_log_file_size; - if (UNIV_PAGE_SIZE_DEF != srv_page_size) { ib::info() << "innodb_page_size=" << srv_page_size; @@ -21184,10 +21182,12 @@ static MYSQL_SYSVAR_LONG(log_buffer_size, innobase_log_buffer_size, "The size of the buffer which InnoDB uses to write log to the log files on disk.", NULL, NULL, 16*1024*1024L, 256*1024L, LONG_MAX, 1024); -static MYSQL_SYSVAR_LONGLONG(log_file_size, innobase_log_file_size, +static MYSQL_SYSVAR_ULONGLONG(log_file_size, srv_log_file_size, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "Size of each log file in a log group.", - NULL, NULL, 48 << 20, 1 << 20, 512ULL << 30, 1 << 20); + NULL, NULL, 48 << 20, 1 << 20, 512ULL << 30, UNIV_PAGE_SIZE_MAX); +/* OS_FILE_LOG_BLOCK_SIZE would be more appropriate than UNIV_PAGE_SIZE_MAX, +but fil_space_t is being used for the redo log, and it uses data pages. */ static MYSQL_SYSVAR_ULONG(log_files_in_group, srv_n_log_files, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 32668d7b929..1062a785fdb 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -156,17 +156,17 @@ void log_sys_init(); /** Initialize the redo log. -@param[in] n_files number of files -@param[in] file_size file size in bytes */ +@param[in] n_files number of files */ void -log_init(ulint n_files, lsn_t file_size); +log_init(ulint n_files); /** Calculate the recommended highest values for lsn - last_checkpoint_lsn and lsn - buf_get_oldest_modification(). +@param[in] file_size requested innodb_log_file_size @retval true on success @retval false if the smallest log group is too small to accommodate the number of OS threads in the database server */ bool -log_set_capacity() +log_set_capacity(ulonglong file_size) MY_ATTRIBUTE((warn_unused_result)); /******************************************************//** diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 4704625e1fe..4ee8962a0c7 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -360,17 +360,9 @@ extern char* srv_log_group_home_dir; /** Maximum number of srv_n_log_files, or innodb_log_files_in_group */ #define SRV_N_LOG_FILES_MAX 100 extern ulong srv_n_log_files; -/** At startup, this is the current redo log file size. -During startup, if this is different from srv_log_file_size_requested -(innodb_log_file_size), the redo log will be rebuilt and this size -will be initialized to srv_log_file_size_requested. -When upgrading from a previous redo log format, this will be set to 0, -and writing to the redo log is not allowed. - -During startup, this is in bytes, and later converted to pages. */ -extern ib_uint64_t srv_log_file_size; -/** The value of the startup parameter innodb_log_file_size */ -extern ib_uint64_t srv_log_file_size_requested; +/** The InnoDB redo log file size, or 0 when changing the redo log format +at startup (while disallowing writes to the redo log). */ +extern ulonglong srv_log_file_size; extern ulint srv_log_buffer_size; extern ulong srv_flush_log_at_trx_commit; extern uint srv_flush_log_at_timeout; diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 71c9b23c439..182df269015 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -669,18 +669,17 @@ log_group_set_fields( /** Calculate the recommended highest values for lsn - last_checkpoint_lsn and lsn - buf_get_oldest_modification(). +@param[in] file_size requested innodb_log_file_size @retval true on success @retval false if the smallest log group is too small to accommodate the number of OS threads in the database server */ bool -log_set_capacity() +log_set_capacity(ulonglong file_size) { lsn_t margin; ulint free; - lsn_t smallest_capacity = ((srv_log_file_size_requested - << srv_page_size_shift) - - LOG_FILE_HDR_SIZE) + lsn_t smallest_capacity = (file_size - LOG_FILE_HDR_SIZE) * srv_n_log_files; /* Add extra safety */ smallest_capacity -= smallest_capacity / 10; @@ -798,10 +797,9 @@ log_sys_init() } /** Initialize the redo log. -@param[in] n_files number of files -@param[in] file_size file size in bytes */ +@param[in] n_files number of files */ void -log_init(ulint n_files, lsn_t file_size) +log_init(ulint n_files) { ulint i; log_group_t* group = &log_sys->log; @@ -810,7 +808,7 @@ log_init(ulint n_files, lsn_t file_size) group->format = srv_encrypt_log ? LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED : LOG_HEADER_FORMAT_CURRENT; - group->file_size = file_size; + group->file_size = srv_log_file_size; group->state = LOG_GROUP_OK; group->lsn = LOG_START_LSN; group->lsn_offset = LOG_FILE_HDR_SIZE; diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index d7b7711e402..53009ca7621 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -201,17 +201,9 @@ static os_event_t srv_master_thread_disabled_event; char* srv_log_group_home_dir; ulong srv_n_log_files; -/** At startup, this is the current redo log file size. -During startup, if this is different from srv_log_file_size_requested -(innodb_log_file_size), the redo log will be rebuilt and this size -will be initialized to srv_log_file_size_requested. -When upgrading from a previous redo log format, this will be set to 0, -and writing to the redo log is not allowed. - -During startup, this is in bytes, and later converted to pages. */ -ib_uint64_t srv_log_file_size; -/** The value of the startup parameter innodb_log_file_size */ -ib_uint64_t srv_log_file_size_requested; +/** The InnoDB redo log file size, or 0 when changing the redo log format +at startup (while disallowing writes to the redo log). */ +ulonglong srv_log_file_size; /** copy of innodb_log_buffer_size, but in database pages */ ulint srv_log_buffer_size; /** innodb_flush_log_at_trx_commit */ @@ -1161,8 +1153,6 @@ srv_normalize_init_values(void) srv_tmp_space.normalize(); - srv_log_file_size /= UNIV_PAGE_SIZE; - srv_log_buffer_size /= UNIV_PAGE_SIZE; srv_lock_table_size = 5 * (srv_buf_pool_size / UNIV_PAGE_SIZE); diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index ccc0e89e67f..399d02e7395 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -138,6 +138,8 @@ bool srv_is_being_started; bool srv_sys_tablespaces_open; /** TRUE if the server was successfully started */ bool srv_was_started; +/** The original value of srv_log_file_size (innodb_log_file_size) */ +static ulonglong srv_log_file_size_requested; /** TRUE if innobase_start_or_create_for_mysql() has been called */ static bool srv_start_has_been_called; @@ -374,17 +376,13 @@ create_log_file( } ib::info() << "Setting log file " << name << " size to " - << (srv_log_file_size >> (20 - UNIV_PAGE_SIZE_SHIFT)) - << " MB"; + << srv_log_file_size << " bytes"; - ret = os_file_set_size(name, *file, - (os_offset_t) srv_log_file_size - << UNIV_PAGE_SIZE_SHIFT, + ret = os_file_set_size(name, *file, srv_log_file_size, srv_read_only_mode); if (!ret) { - ib::error() << "Cannot set log file " << name << " to size " - << (srv_log_file_size >> (20 - UNIV_PAGE_SIZE_SHIFT)) - << " MB"; + ib::error() << "Cannot set log file " << name << " size to " + << srv_log_file_size << " bytes"; return(DB_ERROR); } @@ -467,17 +465,17 @@ create_log_files( ut_a(fil_validate()); ut_a(log_space != NULL); + const ulint size = ulint(srv_log_file_size >> srv_page_size_shift); + logfile0 = fil_node_create( - logfilename, (ulint) srv_log_file_size, - log_space, false, false); + logfilename, size, log_space, false, false); ut_a(logfile0); for (unsigned i = 1; i < srv_n_log_files; i++) { sprintf(logfilename + dirnamelen, "ib_logfile%u", i); - if (!fil_node_create(logfilename, - (ulint) srv_log_file_size, + if (!fil_node_create(logfilename, size, log_space, false, false)) { ib::error() @@ -488,8 +486,8 @@ create_log_files( } } - log_init(srv_n_log_files, srv_log_file_size * UNIV_PAGE_SIZE); - if (!log_set_capacity()) { + log_init(srv_n_log_files); + if (!log_set_capacity(srv_log_file_size_requested)) { return(DB_ERROR); } @@ -1410,7 +1408,7 @@ srv_prepare_to_delete_redo_log_files( info << srv_n_log_files << "*" << srv_log_file_size_requested - << " pages; LSN=" << flushed_lsn; + << " bytes; LSN=" << flushed_lsn; } /* Flush the old log files. */ @@ -1863,8 +1861,7 @@ innobase_start_or_create_for_mysql() srv_start_state_set(SRV_START_STATE_IO); } - if (srv_n_log_files * srv_log_file_size * UNIV_PAGE_SIZE - >= 512ULL * 1024ULL * 1024ULL * 1024ULL) { + if (srv_n_log_files * srv_log_file_size >= 512ULL << 30) { /* log_block_convert_lsn_to_no() limits the returned block number to 1G and given that OS_FILE_LOG_BLOCK_SIZE is 512 bytes, then we have a limit of 512 GB. If that limit is to @@ -1875,18 +1872,8 @@ innobase_start_or_create_for_mysql() return(srv_init_abort(DB_ERROR)); } - if (srv_n_log_files * srv_log_file_size >= ULINT_MAX) { - /* fil_io() takes ulint as an argument and we are passing - (next_offset / UNIV_PAGE_SIZE) to it in log_group_write_buf(). - So (next_offset / UNIV_PAGE_SIZE) must be less than ULINT_MAX. - So next_offset must be < ULINT_MAX * UNIV_PAGE_SIZE. This - means that we are limited to ULINT_MAX * UNIV_PAGE_SIZE which - is 64 TB on 32 bit systems. */ - ib::error() << "Combined size of log files must be < " - << ULINT_MAX / 1073741824 * UNIV_PAGE_SIZE << " GB"; - - return(srv_init_abort(DB_ERROR)); - } + compile_time_assert(ulonglong(ULINT_MAX) * UNIV_PAGE_SIZE_MIN + >= 512ULL << 30); os_normalize_path(srv_data_home); @@ -2025,27 +2012,22 @@ innobase_start_or_create_for_mysql() ut_a(size != (os_offset_t) -1); - if (size & ((1 << UNIV_PAGE_SIZE_SHIFT) - 1)) { + if (size & (OS_FILE_LOG_BLOCK_SIZE - 1)) { ib::error() << "Log file " << logfilename << " size " << size << " is not a" - " multiple of innodb_page_size"; + " multiple of 512 bytes"; return(srv_init_abort(DB_ERROR)); } - size >>= UNIV_PAGE_SIZE_SHIFT; - if (i == 0) { srv_log_file_size = size; } else if (size != srv_log_file_size) { ib::error() << "Log file " << logfilename - << " is of different size " - << (size << UNIV_PAGE_SIZE_SHIFT) + << " is of different size " << size << " bytes than other log files " - << (srv_log_file_size - << UNIV_PAGE_SIZE_SHIFT) - << " bytes!"; + << srv_log_file_size << " bytes!"; return(srv_init_abort(DB_ERROR)); } } @@ -2066,23 +2048,23 @@ innobase_start_or_create_for_mysql() ut_a(fil_validate()); ut_a(log_space); - /* srv_log_file_size is measured in pages; if page size is 16KB, - then we have a limit of 64TB on 32 bit systems */ - ut_a(srv_log_file_size <= ULINT_MAX); + ut_a(srv_log_file_size <= 512ULL << 30); + + const ulint size = 1 + ulint((srv_log_file_size - 1) + >> srv_page_size_shift); - for (unsigned j = 0; j < i; j++) { + for (unsigned j = 0; j < srv_n_log_files_found; j++) { sprintf(logfilename + dirnamelen, "ib_logfile%u", j); - if (!fil_node_create(logfilename, - (ulint) srv_log_file_size, + if (!fil_node_create(logfilename, size, log_space, false, false)) { return(srv_init_abort(DB_ERROR)); } } - log_init(i, srv_log_file_size * UNIV_PAGE_SIZE); + log_init(srv_n_log_files_found); - if (!log_set_capacity()) { + if (!log_set_capacity(srv_log_file_size_requested)) { return(srv_init_abort(DB_ERROR)); } } |