From d09426f9e60fd93296464ec9eb5f9d85566437d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 10 Sep 2021 19:15:41 +0300 Subject: MDEV-26537 InnoDB corrupts files due to incorrect st_blksize calculation The st_blksize returned by fstat(2) is not documented to be a power of 2, like we assumed in commit 58252fff15acfe7c7b0452a87e202e3f8e454e19 (MDEV-26040). While on Linux, the st_blksize appears to report the file system block size (which hopefully is not smaller than the sector size of the underlying block device), on FreeBSD we observed st_blksize values that might have been something similar to st_size. Also IBM AIX was affected by this. A simple test case would lead to a crash when using the minimum innodb_buffer_pool_size=5m on both FreeBSD and AIX: seq -f 'create table t%g engine=innodb select * from seq_1_to_200000;' \ 1 100|mysql test& seq -f 'create table u%g engine=innodb select * from seq_1_to_200000;' \ 1 100|mysql test& We will fix this by not trusting st_blksize at all, and assuming that the smallest allowed write size (for O_DIRECT) is 4096 bytes. We hope that no storage systems with larger block size exist. Anything larger than 4096 bytes should be unlikely, given that it is the minimum virtual memory page size of many contemporary processors. MariaDB Server on Microsoft Windows was not affected by this. While the 512-byte sector size of the venerable Seagate ST-225 is still in widespread use, the minimum innodb_page_size is 4096 bytes, and innodb_log_file_size can be set in integer multiples of 65536 bytes. The only occasion where InnoDB uses smaller data file block sizes than 4096 bytes is with ROW_FORMAT=COMPRESSED tables with KEY_BLOCK_SIZE=1 or KEY_BLOCK_SIZE=2 (or innodb_page_size=4096). For such tables, we will from now on preallocate space in integer multiples of 4096 bytes and let regular writes extend the file by 1024, 2048, or 3072 bytes. The view INFORMATION_SCHEMA.INNODB_SYS_TABLESPACES.FS_BLOCK_SIZE should report the raw st_blksize. For page_compressed tables, the function fil_space_get_block_size() will map to 512 any st_blksize value that is larger than 4096. os_file_set_size(): Assume that the file system block size is 4096 bytes, and only support extending files to integer multiples of 4096 bytes. fil_space_extend_must_retry(): Round down the preallocation size to an integer multiple of 4096 bytes. --- storage/innobase/fil/fil0fil.cc | 13 +++++++++---- storage/innobase/os/os0file.cc | 7 ++++--- 2 files changed, 13 insertions(+), 7 deletions(-) (limited to 'storage/innobase') diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index a0542825891..8075defac4c 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -980,11 +980,16 @@ fil_space_extend_must_retry( const page_size_t pageSize(space->flags); const ulint page_size = pageSize.physical(); - /* fil_read_first_page() expects UNIV_PAGE_SIZE bytes. - fil_node_open_file() expects at least 4 * UNIV_PAGE_SIZE bytes.*/ + /* fil_read_first_page() expects innodb_page_size bytes. + fil_node_open_file() expects at least 4 * innodb_page_size bytes. + os_file_set_size() expects multiples of 4096 bytes. + For ROW_FORMAT=COMPRESSED tables using 1024-byte or 2048-byte + pages, we will preallocate up to an integer multiple of 4096 bytes, + and let normal writes append 1024, 2048, or 3072 bytes to the file. */ os_offset_t new_size = std::max( - os_offset_t(size - file_start_page_no) * page_size, - os_offset_t(FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE)); + (os_offset_t(size - file_start_page_no) * page_size) + & ~os_offset_t(4095), + os_offset_t(FIL_IBD_FILE_INITIAL_SIZE << srv_page_size_shift)); *success = os_file_set_size(node->name, node->handle, new_size, FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags)); diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index a793f9b30cb..48be5256ab1 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -5405,6 +5405,8 @@ os_file_set_size( os_offset_t size, bool is_sparse) { + ut_ad(!(size & 4095)); + #ifdef _WIN32 /* On Windows, changing file size works well and as expected for both sparse and normal files. @@ -5446,7 +5448,7 @@ fallback: if (current_size >= size) { return true; } - current_size &= ~os_offset_t(statbuf.st_blksize - 1); + current_size &= ~4095ULL; err = posix_fallocate(file, current_size, size - current_size); } @@ -5486,8 +5488,7 @@ fallback: if (fstat(file, &statbuf)) { return false; } - os_offset_t current_size = statbuf.st_size - & ~os_offset_t(statbuf.st_blksize - 1); + os_offset_t current_size = statbuf.st_size & ~4095ULL; #endif if (current_size >= size) { return true; -- cgit v1.2.1 From 098106b4327d549bca226ccdeec107dbc3237e97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 11 Sep 2021 11:21:25 +0300 Subject: MDEV-25951 followup: Add #ifdef around debug code --- storage/innobase/dict/dict0load.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'storage/innobase') diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index 2830aea167f..ab3aaa61970 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, 2020, MariaDB Corporation. +Copyright (c) 2016, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -2577,6 +2577,7 @@ corrupted: goto func_exit; } +#ifdef UNIV_DEBUG // The following assertion doesn't hold for FTS indexes // as it may have prefix_len=1 with any charset if (index->type != DICT_FTS) { @@ -2587,6 +2588,7 @@ corrupted: % f.col->mbmaxlen == 0); } } +#endif /* UNIV_DEBUG */ } next_rec: btr_pcur_move_to_next_user_rec(&pcur, &mtr); -- cgit v1.2.1