diff options
39 files changed, 1483 insertions, 1497 deletions
diff --git a/mysql-test/suite/innodb/include/ibd_convert.pl b/mysql-test/suite/innodb/include/ibd_convert.pl new file mode 100644 index 00000000000..aba742ac649 --- /dev/null +++ b/mysql-test/suite/innodb/include/ibd_convert.pl @@ -0,0 +1,25 @@ +#!/usr/bin/perl +# Convert tablespace flags to the format understood by MariaDB 10.1.0..10.1.20, +# with the assumption that the flags were correct. + +sub convert_to_mariadb_101 +{ + my ($file, $page_size) = @_; + open(FILE, "+<", $file) or die "Unable to open $file\n"; + sysread(FILE, $_, $page_size)==$page_size||die "Unable to read $file\n"; + sysseek(FILE, 0, 0)||die "Unable to seek $file\n"; + + # FIL_PAGE_DATA + FSP_SPACE_FLAGS = 38 + 16 = 54 bytes from the start + my($flags) = unpack "x[54]N"; + my $badflags = ($flags & 0x3f); + my $compression_level=6; + $badflags |= 1<<6|$compression_level<<7 if ($flags & 1 << 16); + $badflags |= ($flags & 15 << 6) << 7; # PAGE_SSIZE + + substr ($_, 54, 4) = pack("N", $badflags); + # Replace the innodb_checksum_algorithm=none checksum + substr ($_, 0, 4) = pack("N", 0xdeadbeef); + substr ($_, $page_size - 8, 4) = pack("N", 0xdeadbeef); + syswrite(FILE, $_, $page_size)==$page_size||die "Unable to write $file\n"; + close(FILE); +} diff --git a/mysql-test/suite/innodb/r/101_compatibility.result b/mysql-test/suite/innodb/r/101_compatibility.result new file mode 100644 index 00000000000..f9cc8c288ca --- /dev/null +++ b/mysql-test/suite/innodb/r/101_compatibility.result @@ -0,0 +1,48 @@ +# +# MDEV-11623 MariaDB 10.1 fails to start datadir created with +# MariaDB 10.0/MySQL 5.6 using innodb-page-size!=16K +# +call mtr.add_suppression("InnoDB: adjusting FSP_SPACE_FLAGS of tablespace"); +SET GLOBAL innodb_file_per_table=1; +SET GLOBAL innodb_file_format=Barracuda; +CREATE TABLE tr(a INT)ENGINE=InnoDB ROW_FORMAT=REDUNDANT; +CREATE TABLE tc(a INT)ENGINE=InnoDB ROW_FORMAT=COMPACT; +CREATE TABLE td(a INT)ENGINE=InnoDB ROW_FORMAT=DYNAMIC; +CREATE TABLE tz(a INT)ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=1; +CREATE TABLE tdd(a INT) ENGINE=InnoDB, DATA DIRECTORY='MYSQL_TMP_DIR'; +CREATE TABLE tp(a INT) ENGINE=InnoDB page_compressed=1; +CREATE TABLE ti(a INT) ENGINE=InnoDB; +FLUSH TABLES ti FOR EXPORT; +backup: ti +UNLOCK TABLES; +ALTER TABLE ti DISCARD TABLESPACE; +restore: ti .ibd and .cfg files +ALTER TABLE ti IMPORT TABLESPACE; +BEGIN; +INSERT INTO tr VALUES(1); +INSERT INTO tc VALUES(1); +INSERT INTO td VALUES(1); +INSERT INTO tz VALUES(1); +INSERT INTO tdd VALUES(1); +INSERT INTO tp VALUES(1); +INSERT INTO ti VALUES(1); +# Kill the server +CHECK TABLE tr,tc,td,tz,tdd,tp,ti; +Table Op Msg_type Msg_text +test.tr check status OK +test.tc check status OK +test.td check status OK +test.tz check status OK +test.tdd check status OK +test.tp check status OK +test.ti check status OK +CHECK TABLE tr,tc,td,tz,tdd,tp,ti; +Table Op Msg_type Msg_text +test.tr check status OK +test.tc check status OK +test.td check status OK +test.tz check status OK +test.tdd check status OK +test.tp check status OK +test.ti check status OK +DROP TABLE tr,tc,td,tz,tdd,tp,ti; diff --git a/mysql-test/suite/innodb/r/doublewrite.result b/mysql-test/suite/innodb/r/doublewrite.result index aa96a5f2d93..6b913f49972 100644 --- a/mysql-test/suite/innodb/r/doublewrite.result +++ b/mysql-test/suite/innodb/r/doublewrite.result @@ -39,6 +39,8 @@ set global innodb_buf_flush_list_now = 1; # Kill the server # Make the first page (page_no=0) of the user tablespace # full of zeroes. +# +# MDEV-11623: Use old FSP_SPACE_FLAGS in the doublewrite buffer. check table t1; Table Op Msg_type Msg_text test.t1 check status OK diff --git a/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result b/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result index 6231a875804..58721a8732a 100644 --- a/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result +++ b/mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result @@ -1,4 +1,5 @@ -call mtr.add_suppression("InnoDB: Page for tablespace .* "); +call mtr.add_suppression("InnoDB: Page for tablespace "); +call mtr.add_suppression("InnoDB: Invalid FSP_SPACE_FLAGS=0x"); FLUSH TABLES; SET GLOBAL innodb_file_per_table = 1; SELECT @@innodb_file_per_table; @@ -565,7 +566,7 @@ ERROR HY000: Tablespace has been discarded for table 't1' restore: t1 .ibd and .cfg files SET SESSION debug_dbug="+d,fsp_flags_is_valid_failure"; ALTER TABLE test_wl5522.t1 IMPORT TABLESPACE; -ERROR HY000: Internal error: Cannot reset LSNs in table '"test_wl5522"."t1"' : Unsupported +ERROR HY000: Internal error: Cannot reset LSNs in table '"test_wl5522"."t1"' : Data structure corruption SET SESSION debug_dbug="-d,fsp_flags_is_valid_failure"; DROP TABLE test_wl5522.t1; unlink: t1.ibd diff --git a/mysql-test/suite/innodb/t/101_compatibility.test b/mysql-test/suite/innodb/t/101_compatibility.test new file mode 100644 index 00000000000..125a559c1cb --- /dev/null +++ b/mysql-test/suite/innodb/t/101_compatibility.test @@ -0,0 +1,101 @@ +--source include/have_innodb.inc +--source include/not_embedded.inc + +-- echo # +-- echo # MDEV-11623 MariaDB 10.1 fails to start datadir created with +-- echo # MariaDB 10.0/MySQL 5.6 using innodb-page-size!=16K +-- echo # + +# This is actually testing the opposite: starting the fixed 10.1 with +# buggy 10.1 files (by manually converting the flags in the files). + +call mtr.add_suppression("InnoDB: adjusting FSP_SPACE_FLAGS of tablespace"); +SET GLOBAL innodb_file_per_table=1; +SET GLOBAL innodb_file_format=Barracuda; +let INNODB_PAGE_SIZE=`select @@innodb_page_size`; +let MYSQLD_DATADIR=`select @@datadir`; + +CREATE TABLE tr(a INT)ENGINE=InnoDB ROW_FORMAT=REDUNDANT; +CREATE TABLE tc(a INT)ENGINE=InnoDB ROW_FORMAT=COMPACT; +CREATE TABLE td(a INT)ENGINE=InnoDB ROW_FORMAT=DYNAMIC; +--disable_warnings +CREATE TABLE tz(a INT)ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=1; +--enable_warnings + +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +EVAL CREATE TABLE tdd(a INT) ENGINE=InnoDB, DATA DIRECTORY='$MYSQL_TMP_DIR'; + +CREATE TABLE tp(a INT) ENGINE=InnoDB page_compressed=1; +CREATE TABLE ti(a INT) ENGINE=InnoDB; +FLUSH TABLES ti FOR EXPORT; +perl; +do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl"; +ib_backup_tablespaces("test", "ti"); +EOF +UNLOCK TABLES; +ALTER TABLE ti DISCARD TABLESPACE; +perl; +do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl"; +ib_discard_tablespaces("test", "ti"); +ib_restore_tablespaces("test", "ti"); +do "$ENV{MTR_SUITE_DIR}/include/ibd_convert.pl"; +my $ps = $ENV{INNODB_PAGE_SIZE}; +my $dd = $ENV{MYSQLD_DATADIR}; +convert_to_mariadb_101("$dd/test/ti.ibd", $ps); +EOF + +ALTER TABLE ti IMPORT TABLESPACE; + +BEGIN; +INSERT INTO tr VALUES(1); +INSERT INTO tc VALUES(1); +INSERT INTO td VALUES(1); +INSERT INTO tz VALUES(1); +INSERT INTO tdd VALUES(1); +INSERT INTO tp VALUES(1); +INSERT INTO ti VALUES(1); + +--source include/kill_mysqld.inc + +perl; +do "$ENV{MTR_SUITE_DIR}/include/ibd_convert.pl"; +my $ps = $ENV{INNODB_PAGE_SIZE}; +my $dd = $ENV{MYSQLD_DATADIR}; + +convert_to_mariadb_101("$dd/ibdata1", $ps); +convert_to_mariadb_101("$dd/test/tr.ibd", $ps); +convert_to_mariadb_101("$dd/test/tc.ibd", $ps); +convert_to_mariadb_101("$dd/test/td.ibd", $ps); +convert_to_mariadb_101("$dd/test/tz.ibd", 1024) if $ps<32768; +convert_to_mariadb_101("$dd/test/tp.ibd", $ps); +convert_to_mariadb_101("$dd/test/ti.ibd", $ps); +convert_to_mariadb_101("$ENV{MYSQL_TMP_DIR}/test/tdd.ibd", $ps); +EOF + +--source include/start_mysqld.inc +CHECK TABLE tr,tc,td,tz,tdd,tp,ti; +--source include/shutdown_mysqld.inc + +perl; +do "$ENV{MTR_SUITE_DIR}/include/ibd_convert.pl"; +my $ps = $ENV{INNODB_PAGE_SIZE}; +my $dd = $ENV{MYSQLD_DATADIR}; + +convert_to_mariadb_101("$dd/ibdata1", $ps); +convert_to_mariadb_101("$dd/test/tr.ibd", $ps); +convert_to_mariadb_101("$dd/test/tc.ibd", $ps); +convert_to_mariadb_101("$dd/test/td.ibd", $ps); +convert_to_mariadb_101("$dd/test/tz.ibd", 1024) if $ps<32768; +convert_to_mariadb_101("$dd/test/tp.ibd", $ps); +convert_to_mariadb_101("$dd/test/ti.ibd", $ps); +convert_to_mariadb_101("$ENV{MYSQL_TMP_DIR}/test/tdd.ibd", $ps); +EOF + +--let $restart_parameters=--innodb-read-only +--source include/start_mysqld.inc +CHECK TABLE tr,tc,td,tz,tdd,tp,ti; +--source include/shutdown_mysqld.inc + +--let $restart_parameters= +--source include/start_mysqld.inc +DROP TABLE tr,tc,td,tz,tdd,tp,ti; diff --git a/mysql-test/suite/innodb/t/doublewrite.test b/mysql-test/suite/innodb/t/doublewrite.test index 56cc88f5590..245b4e1a676 100644 --- a/mysql-test/suite/innodb/t/doublewrite.test +++ b/mysql-test/suite/innodb/t/doublewrite.test @@ -17,7 +17,7 @@ call mtr.add_suppression("space header page consists of zero bytes.*test.t1"); call mtr.add_suppression("checksum mismatch in tablespace.*test.t1"); call mtr.add_suppression("Current page size .* != page size on page"); call mtr.add_suppression("innodb-page-size mismatch in tablespace.*test.t1"); -call mtr.add_suppression("Database page corruption"); +call mtr.add_suppression("Trying to recover page.*from the doublewrite buffer"); --enable_query_log let INNODB_PAGE_SIZE=`select @@innodb_page_size`; @@ -65,14 +65,48 @@ set global innodb_buf_flush_list_now = 1; --echo # Make the first page (page_no=0) of the user tablespace --echo # full of zeroes. +--echo # +--echo # MDEV-11623: Use old FSP_SPACE_FLAGS in the doublewrite buffer. + perl; use IO::Handle; my $fname= "$ENV{'MYSQLD_DATADIR'}test/t1.ibd"; +my $page_size = $ENV{INNODB_PAGE_SIZE}; +my $page; open(FILE, "+<", $fname) or die; -FILE->autoflush(1); -binmode FILE; -print FILE chr(0) x ($ENV{'INNODB_PAGE_SIZE'}); +sysread(FILE, $page, $page_size)==$page_size||die "Unable to read $file\n"; +sysseek(FILE, 0, 0)||die "Unable to seek $file\n"; +die unless syswrite(FILE, chr(0) x $page_size, $page_size) == $page_size; close FILE; + +open(FILE, "+<", "$ENV{MYSQLD_DATADIR}ibdata1")||die "cannot open ibdata1\n"; +sysseek(FILE, 6 * $page_size - 190, 0)||die "Unable to seek ibdata1\n"; +sysread(FILE, $_, 12) == 12||die "Unable to read TRX_SYS\n"; +my($magic,$d1,$d2)=unpack "NNN"; +die "magic=$magic, $d1, $d2\n" unless $magic == 536853855 && $d2 >= $d1 + 64; +sysseek(FILE, $d1 * $page_size, 0)||die "Unable to seek ibdata1\n"; +# Find the page in the doublewrite buffer +for (my $d = $d1; $d < $d2 + 64; $d++) +{ + sysread(FILE, $_, $page_size)==$page_size||die "Cannot read doublewrite\n"; + next unless $_ eq $page; + sysseek(FILE, $d * $page_size, 0)||die "Unable to seek ibdata1\n"; + # Write buggy MariaDB 10.1.x FSP_SPACE_FLAGS to the doublewrite buffer + my($flags) = unpack "x[54]N"; + my $badflags = ($flags & 0x3f); + my $compression_level=6; + $badflags |= 1<<6|$compression_level<<7 if ($flags & 1 << 16); + $badflags |= ($flags & 15 << 6) << 7; # PAGE_SSIZE + + substr ($_, 54, 4) = pack("N", $badflags); + # Replace the innodb_checksum_algorithm=none checksum + substr ($_, 0, 4) = pack("N", 0xdeadbeef); + substr ($_, $page_size - 8, 4) = pack("N", 0xdeadbeef); + syswrite(FILE, $_, $page_size)==$page_size||die "Unable to write $file\n"; + close(FILE); + exit 0; +} +die "Did not find the page in the doublewrite buffer ($d1,$d2)\n"; EOF --source include/start_mysqld.inc diff --git a/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test b/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test index bc3edbb2643..0bfe67c06bc 100644 --- a/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test +++ b/mysql-test/suite/innodb/t/innodb-wl5522-debug-zip.test @@ -17,7 +17,8 @@ # allow test to run only when innodb-page-size=16 --source include/have_innodb_16k.inc -call mtr.add_suppression("InnoDB: Page for tablespace .* "); +call mtr.add_suppression("InnoDB: Page for tablespace "); +call mtr.add_suppression("InnoDB: Invalid FSP_SPACE_FLAGS=0x"); FLUSH TABLES; let MYSQLD_DATADIR =`SELECT @@datadir`; diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 0b42e38a655..e49185f6a1f 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -589,6 +589,21 @@ buf_dblwr_process() continue; } + if (page_no == 0) { + /* Check the FSP_SPACE_FLAGS. */ + ulint flags = fsp_header_get_flags(page); + if (!fsp_flags_is_valid(flags) + && fsp_flags_convert_from_101(flags) + == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_WARN, + "Ignoring a doublewrite copy of page " + ULINTPF ":0 due to invalid flags 0x%x", + space_id, int(flags)); + continue; + } + /* The flags on the page should be converted later. */ + } + /* Write the good page from the doublewrite buffer to the intended position. */ diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index 0ab25ee1eac..0f9f4f153f5 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -1052,8 +1052,6 @@ loop: btr_pcur_store_position(&pcur, &mtr); - mtr_commit(&mtr); - /* For tables created with old versions of InnoDB, SYS_TABLES.MIX_LEN may contain garbage. Such tables would always be in ROW_FORMAT=REDUNDANT. Pretend that @@ -1087,16 +1085,19 @@ loop: if (space_id == 0) { /* The system tablespace always exists. */ ut_ad(!discarded); - goto next_tablespace; + mem_free(name); + goto loop; } + mtr_commit(&mtr); + switch (dict_check) { case DICT_CHECK_ALL_LOADED: /* All tablespaces should have been found in fil_load_single_table_tablespaces(). */ if (fil_space_for_table_exists_in_mem( - space_id, name, TRUE, !(is_temp || discarded), - false, NULL, 0) + space_id, name, !(is_temp || discarded), + false, NULL, 0, flags) && !(is_temp || discarded)) { /* If user changes the path of .ibd files in *.isl files before doing crash recovery , @@ -1128,8 +1129,8 @@ loop: /* Some tablespaces may have been opened in trx_resurrect_table_locks(). */ if (fil_space_for_table_exists_in_mem( - space_id, name, FALSE, FALSE, - false, NULL, 0)) { + space_id, name, false, + false, NULL, 0, flags)) { break; } /* fall through */ @@ -1191,7 +1192,6 @@ loop: max_space_id = space_id; } -next_tablespace: mem_free(name); mtr_start(&mtr); @@ -2382,8 +2382,8 @@ err_exit: table->ibd_file_missing = TRUE; } else if (!fil_space_for_table_exists_in_mem( - table->space, name, FALSE, FALSE, true, heap, - table->id)) { + table->space, name, false, true, heap, + table->id, table->flags)) { if (DICT_TF2_FLAG_IS_SET(table, DICT_TF2_TEMPORARY)) { /* Do not bother to retry opening temporary tables. */ diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 1e8b2be6f01..fe6a2922b35 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -596,10 +596,7 @@ fil_node_open_file( ibool success; byte* buf2; byte* page; - ulint space_id; - ulint flags=0; ulint page_size; - ulint atomic_writes=0; ut_ad(mutex_own(&(system->mutex))); ut_a(node->n_pending == 0); @@ -621,8 +618,6 @@ fil_node_open_file( /* The following call prints an error message */ os_file_get_last_error(true); - ut_print_timestamp(stderr); - ib_logf(IB_LOG_LEVEL_WARN, "InnoDB: Error: cannot " "open %s\n. InnoDB: Have you deleted .ibd " "files under a running mysqld server?\n", @@ -648,17 +643,13 @@ fil_node_open_file( ut_a(fil_is_user_tablespace_id(space->id)); if (size_bytes < FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE) { - fprintf(stderr, - "InnoDB: Error: the size of single-table" - " tablespace file %s\n" - "InnoDB: is only " UINT64PF "," - " should be at least %lu!\n", - node->name, - size_bytes, - (ulong) (FIL_IBD_FILE_INITIAL_SIZE - * UNIV_PAGE_SIZE)); - - ut_a(0); + ib_logf(IB_LOG_LEVEL_ERROR, + "The size of the file %s is only " UINT64PF + " bytes, should be at least " ULINTPF, + node->name, size_bytes, + FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE); + os_file_close(node->handle); + return(false); } /* Read the first page of the tablespace */ @@ -671,78 +662,34 @@ fil_node_open_file( success = os_file_read(node->handle, page, 0, UNIV_PAGE_SIZE); srv_stats.page0_read.add(1); - space_id = fsp_header_get_space_id(page); - flags = fsp_header_get_flags(page); - - page_size = fsp_flags_get_page_size(flags); - atomic_writes = fsp_flags_get_atomic_writes(flags); - + const ulint space_id = fsp_header_get_space_id(page); + ulint flags = fsp_header_get_flags(page); ut_free(buf2); - - /* Close the file now that we have read the space id from it */ - os_file_close(node->handle); - if (UNIV_UNLIKELY(space_id != space->id)) { - fprintf(stderr, - "InnoDB: Error: tablespace id is %lu" - " in the data dictionary\n" - "InnoDB: but in file %s it is %lu!\n", - space->id, node->name, space_id); - - ut_error; - } - - if (UNIV_UNLIKELY(space_id == ULINT_UNDEFINED - || space_id == 0)) { - fprintf(stderr, - "InnoDB: Error: tablespace id %lu" - " in file %s is not sensible\n", - (ulong) space_id, node->name); - - ut_error; - } - - if (UNIV_UNLIKELY(fsp_flags_get_page_size(space->flags) - != page_size)) { - fprintf(stderr, - "InnoDB: Error: tablespace file %s" - " has page size 0x%lx\n" - "InnoDB: but the data dictionary" - " expects page size 0x%lx!\n", - node->name, flags, - fsp_flags_get_page_size(space->flags)); + if (!fsp_flags_is_valid(flags)) { + ulint cflags = fsp_flags_convert_from_101(flags); + if (cflags == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Expected tablespace flags 0x%x" + " but found 0x%x in the file %s", + int(space->flags), int(flags), + node->name); + return(false); + } - ut_error; + flags = cflags; } - if (UNIV_UNLIKELY(space->flags != flags)) { - ulint sflags = (space->flags & ~FSP_FLAGS_MASK_DATA_DIR); - ulint fflags = (flags & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - - /* DATA_DIR option is on different place on MariaDB - compared to MySQL. If this is the difference. Fix - it. */ - - if (sflags == fflags) { - fprintf(stderr, - "InnoDB: Warning: Table flags 0x%lx" - " in the data dictionary but in file %s are 0x%lx!\n" - " Temporally corrected because DATA_DIR option to 0x%lx.\n", - space->flags, node->name, flags, space->flags); - - flags = space->flags; - } + page_size = fsp_flags_get_page_size(flags); - if (!dict_tf_verify_flags(space->flags, flags)) { - fprintf(stderr, - "InnoDB: Error: table flags are 0x%lx" - " in the data dictionary\n" - "InnoDB: but the flags in file %s are 0x%lx!\n", - space->flags, node->name, flags); - ut_error; - } + if (UNIV_UNLIKELY(space_id != space->id)) { + ib_logf(IB_LOG_LEVEL_ERROR, + "tablespace id is " ULINTPF " in the data dictionary" + " but in file %s it is " ULINTPF "!\n", + space->id, node->name, space_id); + return(false); } if (size_bytes >= (1024*1024)) { @@ -764,7 +711,7 @@ add_size: space->size += node->size; } - atomic_writes = fsp_flags_get_atomic_writes(space->flags); + ulint atomic_writes = fsp_flags_get_atomic_writes(space->flags); /* printf("Opening file %s\n", node->name); */ @@ -1534,7 +1481,6 @@ fil_space_create( fil_system->tablespace_version++; space->tablespace_version = fil_system->tablespace_version; - space->mark = FALSE; if (purpose == FIL_TABLESPACE && !recv_recovery_on && id > fil_system->max_assigned_id) { @@ -2273,27 +2219,21 @@ fil_write_flushed_lsn_to_data_files( return(DB_SUCCESS); } -/*******************************************************************//** -Checks the consistency of the first data page of a tablespace +/** Check the consistency of the first data page of a tablespace at database startup. +@param[in] page page frame +@param[in] space_id tablespace identifier +@param[in] flags tablespace flags @retval NULL on success, or if innodb_force_recovery is set @return pointer to an error message string */ static MY_ATTRIBUTE((warn_unused_result)) const char* -fil_check_first_page( -/*=================*/ - const page_t* page) /*!< in: data page */ +fil_check_first_page(const page_t* page, ulint space_id, ulint flags) { - ulint space_id; - ulint flags; - if (srv_force_recovery >= SRV_FORCE_IGNORE_CORRUPT) { return(NULL); } - space_id = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_ID + page); - flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); - if (UNIV_PAGE_SIZE != fsp_flags_get_page_size(flags)) { fprintf(stderr, "InnoDB: Error: Current page size %lu != " @@ -2342,7 +2282,7 @@ fil_read_first_page( ibool one_read_already, /*!< in: TRUE if min and max parameters below already contain sensible data */ - ulint* flags, /*!< out: tablespace flags */ + ulint* flags, /*!< out: FSP_SPACE_FLAGS */ ulint* space_id, /*!< out: tablespace ID */ #ifdef UNIV_LOG_ARCHIVE ulint* min_arch_log_no, /*!< out: min of archived @@ -2378,12 +2318,22 @@ fil_read_first_page( *flags and *space_id as they were read from the first file and do not validate the first page. */ if (!one_read_already) { - *flags = fsp_header_get_flags(page); *space_id = fsp_header_get_space_id(page); - } + *flags = fsp_header_get_flags(page); - if (!one_read_already) { - check_msg = fil_check_first_page(page); + if (!fsp_flags_is_valid(*flags)) { + ulint cflags = fsp_flags_convert_from_101(*flags); + if (cflags == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Invalid flags 0x%x in tablespace %u", + unsigned(*flags), unsigned(*space_id)); + return "invalid tablespace flags"; + } else { + *flags = cflags; + } + } + + check_msg = fil_check_first_page(page, *space_id, *flags); } flushed_lsn = mach_read_from_8(page + @@ -2577,6 +2527,7 @@ fil_op_write_log( ulint len; log_ptr = mlog_open(mtr, 11 + 2 + 1); + ut_ad(fsp_flags_is_valid(flags)); if (!log_ptr) { /* Logging in mtr is switched off during crash recovery: @@ -3783,7 +3734,7 @@ fil_create_new_single_table_tablespace( ibool success; /* TRUE if a table is created with CREATE TEMPORARY TABLE */ bool is_temp = !!(flags2 & DICT_TF2_TEMPORARY); - bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags); + bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags) != 0; ulint atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(flags); fil_space_crypt_t *crypt_data = NULL; @@ -3791,7 +3742,7 @@ fil_create_new_single_table_tablespace( ut_ad(!srv_read_only_mode); ut_a(space_id < SRV_LOG_SPACE_FIRST_ID); ut_a(size >= FIL_IBD_FILE_INITIAL_SIZE); - ut_a(fsp_flags_is_valid(flags)); + ut_a(fsp_flags_is_valid(flags & ~FSP_FLAGS_MEM_MASK)); if (is_temp) { /* Temporary table filepath */ @@ -3884,12 +3835,9 @@ fil_create_new_single_table_tablespace( memset(page, '\0', UNIV_PAGE_SIZE); - /* Add the UNIV_PAGE_SIZE to the table flags and write them to the - tablespace header. */ - flags = fsp_flags_set_page_size(flags, UNIV_PAGE_SIZE); + flags |= FSP_FLAGS_PAGE_SSIZE(); fsp_header_init_fields(page, space_id, flags); mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id); - ut_ad(fsp_flags_is_valid(flags)); if (!(fsp_flags_is_compressed(flags))) { buf_flush_init_for_writing(page, NULL, 0); @@ -3968,7 +3916,8 @@ fil_create_new_single_table_tablespace( fil_op_write_log(flags ? MLOG_FILE_CREATE2 : MLOG_FILE_CREATE, - space_id, mlog_file_flag, flags, + space_id, mlog_file_flag, + flags & ~FSP_FLAGS_MEM_MASK, tablename, NULL, &mtr); mtr_commit(&mtr); @@ -4032,6 +3981,39 @@ fil_report_bad_tablespace( (ulong) expected_id, (ulong) expected_flags); } +/** Try to adjust FSP_SPACE_FLAGS if they differ from the expectations. +(Typically when upgrading from MariaDB 10.1.0..10.1.20.) +@param[in] space_id tablespace ID +@param[in] flags desired tablespace flags */ +UNIV_INTERN +void +fsp_flags_try_adjust(ulint space_id, ulint flags) +{ + ut_ad(!srv_read_only_mode); + ut_ad(fsp_flags_is_valid(flags)); + + mtr_t mtr; + mtr_start(&mtr); + if (buf_block_t* b = buf_page_get( + space_id, fsp_flags_get_zip_size(flags), 0, RW_X_LATCH, + &mtr)) { + ulint f = fsp_header_get_flags(b->frame); + /* Suppress the message if only the DATA_DIR flag to differs. */ + if ((f ^ flags) & ~(1U << FSP_FLAGS_POS_RESERVED)) { + ib_logf(IB_LOG_LEVEL_WARN, + "adjusting FSP_SPACE_FLAGS of tablespace " + ULINTPF " from 0x%x to 0x%x", + space_id, int(f), int(flags)); + } + if (f != flags) { + mlog_write_ulint(FSP_HEADER_OFFSET + + FSP_SPACE_FLAGS + b->frame, + flags, MLOG_4BYTES, &mtr); + } + } + mtr_commit(&mtr); +} + /********************************************************************//** Tries to open a single-table tablespace and optionally checks that the space id in it is correct. If this does not succeed, print an error message @@ -4061,7 +4043,7 @@ fil_open_single_table_tablespace( bool validate, /*!< in: Do we validate tablespace? */ bool fix_dict, /*!< in: Can we fix the dictionary? */ ulint id, /*!< in: space id */ - ulint flags, /*!< in: tablespace flags */ + ulint flags, /*!< in: expected FSP_SPACE_FLAGS */ const char* tablename, /*!< in: table name in the databasename/tablename format */ const char* path_in, /*!< in: tablespace filepath */ @@ -4086,20 +4068,13 @@ fil_open_single_table_tablespace( /* Table flags can be ULINT_UNDEFINED if dict_tf_to_fsp_flags_failure is set. */ - if (flags != ULINT_UNDEFINED) { - if (!fsp_flags_is_valid(flags)) { - return(DB_CORRUPTION); - } - } else { + if (flags == ULINT_UNDEFINED) { return(DB_CORRUPTION); } + ut_ad(fsp_flags_is_valid(flags & ~FSP_FLAGS_MEM_MASK)); atomic_writes = fsp_flags_get_atomic_writes(flags); - /* If the tablespace was relocated, we do not - compare the DATA_DIR flag */ - ulint mod_flags = flags & ~FSP_FLAGS_MASK_DATA_DIR; - memset(&def, 0, sizeof(def)); memset(&dict, 0, sizeof(dict)); memset(&remote, 0, sizeof(remote)); @@ -4180,31 +4155,17 @@ fil_open_single_table_tablespace( &space_arch_log_no, &space_arch_log_no, #endif /* UNIV_LOG_ARCHIVE */ &def.lsn, &def.lsn, &def.crypt_data); - def.valid = !def.check_msg; if (table) { table->crypt_data = def.crypt_data; table->page_0_read = true; } - /* Validate this single-table-tablespace with SYS_TABLES, - but do not compare the DATA_DIR flag, in case the - tablespace was relocated. */ - - ulint newf = def.flags; - if (newf != mod_flags) { - if (FSP_FLAGS_HAS_DATA_DIR(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR); - } else if(FSP_FLAGS_HAS_DATA_DIR_ORACLE(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - } - } - - if (def.valid && def.id == id - && newf == mod_flags) { + def.valid = !def.check_msg && def.id == id + && fsp_flags_match(flags, def.flags); + if (def.valid) { valid_tablespaces_found++; } else { - def.valid = false; /* Do not use this tablespace. */ fil_report_bad_tablespace( def.filepath, def.check_msg, def.id, @@ -4220,30 +4181,18 @@ fil_open_single_table_tablespace( &remote.arch_log_no, &remote.arch_log_no, #endif /* UNIV_LOG_ARCHIVE */ &remote.lsn, &remote.lsn, &remote.crypt_data); - remote.valid = !remote.check_msg; if (table) { table->crypt_data = remote.crypt_data; table->page_0_read = true; } - /* Validate this single-table-tablespace with SYS_TABLES, - but do not compare the DATA_DIR flag, in case the - tablespace was relocated. */ - ulint newf = remote.flags; - if (newf != mod_flags) { - if (FSP_FLAGS_HAS_DATA_DIR(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR); - } else if(FSP_FLAGS_HAS_DATA_DIR_ORACLE(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - } - } - - if (remote.valid && remote.id == id - && newf == mod_flags) { + /* Validate this single-table-tablespace with SYS_TABLES. */ + remote.valid = !remote.check_msg && remote.id == id + && fsp_flags_match(flags, remote.flags); + if (remote.valid) { valid_tablespaces_found++; } else { - remote.valid = false; /* Do not use this linked tablespace. */ fil_report_bad_tablespace( remote.filepath, remote.check_msg, remote.id, @@ -4260,30 +4209,19 @@ fil_open_single_table_tablespace( &dict.arch_log_no, &dict.arch_log_no, #endif /* UNIV_LOG_ARCHIVE */ &dict.lsn, &dict.lsn, &dict.crypt_data); - dict.valid = !dict.check_msg; if (table) { table->crypt_data = dict.crypt_data; table->page_0_read = true; } - /* Validate this single-table-tablespace with SYS_TABLES, - but do not compare the DATA_DIR flag, in case the - tablespace was relocated. */ - ulint newf = dict.flags; - if (newf != mod_flags) { - if (FSP_FLAGS_HAS_DATA_DIR(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR); - } else if(FSP_FLAGS_HAS_DATA_DIR_ORACLE(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - } - } + /* Validate this single-table-tablespace with SYS_TABLES. */ + dict.valid = !dict.check_msg && dict.id == id + && fsp_flags_match(flags, dict.flags); - if (dict.valid && dict.id == id - && newf == mod_flags) { + if (dict.valid) { valid_tablespaces_found++; } else { - dict.valid = false; /* Do not use this tablespace. */ fil_report_bad_tablespace( dict.filepath, dict.check_msg, dict.id, @@ -4481,6 +4419,10 @@ cleanup_and_exit: mem_free(def.filepath); + if (err == DB_SUCCESS && !srv_read_only_mode) { + fsp_flags_try_adjust(id, flags & ~FSP_FLAGS_MEM_MASK); + } + return(err); } #endif /* !UNIV_HOTBACKUP */ @@ -4662,7 +4604,23 @@ fil_user_tablespace_restore_page( goto out; } - flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); + flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); + + if (!fsp_flags_is_valid(flags)) { + ulint cflags = fsp_flags_convert_from_101(flags); + if (cflags == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_WARN, + "Ignoring a doublewrite copy of page " + ULINTPF ":" ULINTPF + " due to invalid flags 0x%x", + fsp->id, page_no, int(flags)); + err = false; + goto out; + } + flags = cflags; + /* The flags on the page should be converted later. */ + } + zip_size = fsp_flags_get_zip_size(flags); page_size = fsp_flags_get_page_size(flags); @@ -5051,6 +5009,16 @@ will_not_choose: } mutex_exit(&fil_system->mutex); #endif /* UNIV_HOTBACKUP */ + /* Adjust the memory-based flags that would normally be set by + dict_tf_to_fsp_flags(). In recovery, we have no data dictionary. */ + if (FSP_FLAGS_HAS_PAGE_COMPRESSION(fsp->flags)) { + fsp->flags |= page_zip_level + << FSP_FLAGS_MEM_COMPRESSION_LEVEL; + } + remote.flags |= 1U << FSP_FLAGS_MEM_DATA_DIR; + /* We will leave atomic_writes at ATOMIC_WRITES_DEFAULT. + That will be adjusted in fil_space_for_table_exists_in_mem(). */ + ibool file_space_create_success = fil_space_create( tablename, fsp->id, fsp->flags, FIL_TABLESPACE, fsp->crypt_data, false); @@ -5342,13 +5310,12 @@ fil_report_missing_tablespace( name, space_id); } -/*******************************************************************//** -Returns TRUE if a matching tablespace exists in the InnoDB tablespace memory +/** Check if a matching tablespace exists in the InnoDB tablespace memory cache. Note that if we have not done a crash recovery at the database startup, there may be many tablespaces which are not yet in the memory cache. -@return TRUE if a matching tablespace exists in the memory cache */ +@return whether a matching tablespace exists in the memory cache */ UNIV_INTERN -ibool +bool fil_space_for_table_exists_in_mem( /*==============================*/ ulint id, /*!< in: space id */ @@ -5356,13 +5323,7 @@ fil_space_for_table_exists_in_mem( fil_space_create(). Either the standard 'dbname/tablename' format or table->dir_path_of_temp_table */ - ibool mark_space, /*!< in: in crash recovery, at database - startup we mark all spaces which have - an associated table in the InnoDB - data dictionary, so that - we can print a warning about orphaned - tablespaces */ - ibool print_error_if_does_not_exist, + bool print_error_if_does_not_exist, /*!< in: print detailed error information to the .err log if a matching tablespace is not found from @@ -5370,12 +5331,13 @@ fil_space_for_table_exists_in_mem( bool adjust_space, /*!< in: whether to adjust space id when find table space mismatch */ mem_heap_t* heap, /*!< in: heap memory */ - table_id_t table_id) /*!< in: table id */ + table_id_t table_id, /*!< in: table id */ + ulint table_flags) /*!< in: table flags */ { fil_space_t* fnamespace; fil_space_t* space; - ut_ad(fil_system); + const ulint expected_flags = dict_tf_to_fsp_flags(table_flags); mutex_enter(&fil_system->mutex); @@ -5387,42 +5349,31 @@ fil_space_for_table_exists_in_mem( directory path from the datadir to the file */ fnamespace = fil_space_get_by_name(name); - if (space && space == fnamespace) { - /* Found */ - - if (mark_space) { - space->mark = TRUE; - } - - mutex_exit(&fil_system->mutex); - - return(TRUE); - } - - /* Info from "fnamespace" comes from the ibd file itself, it can - be different from data obtained from System tables since it is - not transactional. If adjust_space is set, and the mismatching - space are between a user table and its temp table, we shall - adjust the ibd file name according to system table info */ - if (adjust_space - && space != NULL - && row_is_mysql_tmp_table_name(space->name) - && !row_is_mysql_tmp_table_name(name)) { + bool valid = space && !((space->flags ^ expected_flags) + & ~FSP_FLAGS_MEM_MASK); + if (!space) { + } else if (!valid || space == fnamespace) { + /* Found with the same file name, or got a flag mismatch. */ + goto func_exit; + } else if (adjust_space + && row_is_mysql_tmp_table_name(space->name) + && !row_is_mysql_tmp_table_name(name)) { + /* Info from fnamespace comes from the ibd file + itself, it can be different from data obtained from + System tables since renaming files is not + transactional. We shall adjust the ibd file name + according to system table info. */ mutex_exit(&fil_system->mutex); DBUG_EXECUTE_IF("ib_crash_before_adjust_fil_space", DBUG_SUICIDE();); - if (fnamespace) { - char* tmp_name; + char* tmp_name = dict_mem_create_temporary_tablename( + heap, name, table_id); - tmp_name = dict_mem_create_temporary_tablename( - heap, name, table_id); - - fil_rename_tablespace(fnamespace->name, fnamespace->id, - tmp_name, NULL); - } + fil_rename_tablespace(fnamespace->name, fnamespace->id, + tmp_name, NULL); DBUG_EXECUTE_IF("ib_crash_after_adjust_one_fil_space", DBUG_SUICIDE();); @@ -5435,16 +5386,12 @@ fil_space_for_table_exists_in_mem( mutex_enter(&fil_system->mutex); fnamespace = fil_space_get_by_name(name); ut_ad(space == fnamespace); - mutex_exit(&fil_system->mutex); - - return(TRUE); + goto func_exit; } if (!print_error_if_does_not_exist) { - - mutex_exit(&fil_system->mutex); - - return(FALSE); + valid = false; + goto func_exit; } if (space == NULL) { @@ -5471,10 +5418,8 @@ error_exit: fputs("InnoDB: Please refer to\n" "InnoDB: " REFMAN "innodb-troubleshooting-datadict.html\n" "InnoDB: for how to resolve the issue.\n", stderr); - - mutex_exit(&fil_system->mutex); - - return(FALSE); + valid = false; + goto func_exit; } if (0 != strcmp(space->name, name)) { @@ -5501,9 +5446,19 @@ error_exit: goto error_exit; } +func_exit: + if (valid) { + /* Adjust the flags that are in FSP_FLAGS_MEM_MASK. + FSP_SPACE_FLAGS will not be written back here. */ + space->flags = expected_flags; + } mutex_exit(&fil_system->mutex); - return(FALSE); + if (valid && !srv_read_only_mode) { + fsp_flags_try_adjust(id, expected_flags & ~FSP_FLAGS_MEM_MASK); + } + + return(valid); } /*******************************************************************//** @@ -7331,9 +7286,8 @@ fil_space_get_crypt_data( byte *page = static_cast<byte*>(ut_align(buf, UNIV_PAGE_SIZE)); fil_read(true, space_id, 0, 0, 0, UNIV_PAGE_SIZE, page, NULL, NULL); - ulint flags = fsp_header_get_flags(page); ulint offset = fsp_header_get_crypt_offset( - fsp_flags_get_zip_size(flags), NULL); + fsp_header_get_zip_size(page), NULL); space->crypt_data = fil_space_read_crypt_data(space_id, page, offset); ut_free(buf); diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index 87aa5f7db5c..a71d9f95ac2 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -660,6 +660,7 @@ fsp_header_init_fields( ulint space_id, /*!< in: space id */ ulint flags) /*!< in: tablespace flags (FSP_SPACE_FLAGS) */ { + flags &= ~FSP_FLAGS_MEM_MASK; ut_a(fsp_flags_is_valid(flags)); mach_write_to_4(FSP_HEADER_OFFSET + FSP_SPACE_ID + page, @@ -710,7 +711,7 @@ fsp_header_init( mlog_write_ulint(header + FSP_SIZE, size, MLOG_4BYTES, mtr); mlog_write_ulint(header + FSP_FREE_LIMIT, 0, MLOG_4BYTES, mtr); - mlog_write_ulint(header + FSP_SPACE_FLAGS, flags, + mlog_write_ulint(header + FSP_SPACE_FLAGS, flags & ~FSP_FLAGS_MEM_MASK, MLOG_4BYTES, mtr); mlog_write_ulint(header + FSP_FRAG_N_USED, 0, MLOG_4BYTES, mtr); diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic index 3d2f0dff0da..800065ddaa1 100644 --- a/storage/innobase/include/dict0dict.ic +++ b/storage/innobase/include/dict0dict.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates -Copyright (c) 2013, 2016, MariaDB Corporation +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -914,10 +914,13 @@ dict_tf_to_fsp_flags( ulint table_flags) /*!< in: dict_table_t::flags */ { ulint fsp_flags; - ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); - ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); + ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL( + table_flags); ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); + ut_ad((DICT_TF_GET_PAGE_COMPRESSION(table_flags) == 0) + == (page_compression_level == 0)); + DBUG_EXECUTE_IF("dict_tf_to_fsp_flags_failure", return(ULINT_UNDEFINED);); @@ -925,30 +928,23 @@ dict_tf_to_fsp_flags( fsp_flags = DICT_TF_HAS_ATOMIC_BLOBS(table_flags) ? 1 : 0; /* ZIP_SSIZE and ATOMIC_BLOBS are at the same position. */ - fsp_flags |= table_flags & DICT_TF_MASK_ZIP_SSIZE; - fsp_flags |= table_flags & DICT_TF_MASK_ATOMIC_BLOBS; - - /* In addition, tablespace flags also contain the page size. */ - fsp_flags |= fsp_flags_set_page_size(fsp_flags, UNIV_PAGE_SIZE); + fsp_flags |= table_flags + & (DICT_TF_MASK_ZIP_SSIZE | DICT_TF_MASK_ATOMIC_BLOBS); - /* The DATA_DIR flag is in a different position in fsp_flag */ - fsp_flags |= DICT_TF_HAS_DATA_DIR(table_flags) - ? FSP_FLAGS_MASK_DATA_DIR : 0; + fsp_flags |= FSP_FLAGS_PAGE_SSIZE(); - /* In addition, tablespace flags also contain if the page - compression is used for this table. */ - fsp_flags |= FSP_FLAGS_SET_PAGE_COMPRESSION(fsp_flags, page_compression); + if (page_compression_level) { + fsp_flags |= FSP_FLAGS_MASK_PAGE_COMPRESSION; + } - /* In addition, tablespace flags also contain page compression level - if page compression is used for this table. */ - fsp_flags |= FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(fsp_flags, page_compression_level); + ut_a(fsp_flags_is_valid(fsp_flags)); - /* In addition, tablespace flags also contain flag if atomic writes - is used for this table */ - fsp_flags |= FSP_FLAGS_SET_ATOMIC_WRITES(fsp_flags, atomic_writes); + if (DICT_TF_HAS_DATA_DIR(table_flags)) { + fsp_flags |= 1U << FSP_FLAGS_MEM_DATA_DIR; + } - ut_a(fsp_flags_is_valid(fsp_flags)); - ut_a(dict_tf_verify_flags(table_flags, fsp_flags)); + fsp_flags |= atomic_writes << FSP_FLAGS_MEM_ATOMIC_WRITES; + fsp_flags |= page_compression_level << FSP_FLAGS_MEM_COMPRESSION_LEVEL; return(fsp_flags); } diff --git a/storage/innobase/include/dict0pagecompress.h b/storage/innobase/include/dict0pagecompress.h index 19a2a6c52f3..6503c86ffa2 100644 --- a/storage/innobase/include/dict0pagecompress.h +++ b/storage/innobase/include/dict0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013 SkySQL Ab. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -57,17 +57,6 @@ dict_table_page_compression_level( __attribute__((const)); /********************************************************************//** -Verify that dictionary flags match tablespace flags -@return true if flags match, false if not */ -UNIV_INLINE -ibool -dict_tf_verify_flags( -/*=================*/ - ulint table_flags, /*!< in: dict_table_t::flags */ - ulint fsp_flags) /*!< in: fil_space_t::flags */ - __attribute__((const)); - -/********************************************************************//** Extract the atomic writes flag from table flags. @return true if atomic writes are used, false if not used */ UNIV_INLINE diff --git a/storage/innobase/include/dict0pagecompress.ic b/storage/innobase/include/dict0pagecompress.ic index 811976434a8..13c2b46c51c 100644 --- a/storage/innobase/include/dict0pagecompress.ic +++ b/storage/innobase/include/dict0pagecompress.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013 SkySQL Ab. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -25,92 +25,6 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com ***********************************************************************/ /********************************************************************//** -Verify that dictionary flags match tablespace flags -@return true if flags match, false if not */ -UNIV_INLINE -ibool -dict_tf_verify_flags( -/*=================*/ - ulint table_flags, /*!< in: dict_table_t::flags */ - ulint fsp_flags) /*!< in: fil_space_t::flags */ -{ - ulint table_unused = DICT_TF_GET_UNUSED(table_flags); - ulint compact = DICT_TF_GET_COMPACT(table_flags); - ulint ssize = DICT_TF_GET_ZIP_SSIZE(table_flags); - ulint atomic_blobs = DICT_TF_HAS_ATOMIC_BLOBS(table_flags); - ulint data_dir = DICT_TF_HAS_DATA_DIR(table_flags); - ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); - ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); - ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); - ulint post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(fsp_flags); - ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(fsp_flags); - ulint fsp_atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(fsp_flags); - ulint page_ssize = FSP_FLAGS_GET_PAGE_SSIZE(fsp_flags); - ulint fsp_unused = FSP_FLAGS_GET_UNUSED(fsp_flags); - ulint fsp_page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(fsp_flags); - ulint fsp_page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(fsp_flags); - ulint fsp_atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(fsp_flags); - - DBUG_EXECUTE_IF("dict_tf_verify_flags_failure", - return(ULINT_UNDEFINED);); - - ut_a(!table_unused); - ut_a(!fsp_unused); - ut_a(page_ssize == 0 || page_ssize != 0); /* silence compiler */ - ut_a(compact == 0 || compact == 1); /* silence compiler */ - ut_a(data_dir == 0 || data_dir == 1); /* silence compiler */ - ut_a(post_antelope == 0 || post_antelope == 1); /* silence compiler */ - - if (ssize != zip_ssize) { - fprintf(stderr, - "InnoDB: Error: table flags has zip_ssize %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has zip_ssize %ld\n", - ssize, zip_ssize); - return (FALSE); - } - if (atomic_blobs != fsp_atomic_blobs) { - fprintf(stderr, - "InnoDB: Error: table flags has atomic_blobs %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has atomic_blobs %ld\n", - atomic_blobs, fsp_atomic_blobs); - - return (FALSE); - } - if (page_compression != fsp_page_compression) { - fprintf(stderr, - "InnoDB: Error: table flags has page_compression %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file ahas page_compression %ld\n", - page_compression, fsp_page_compression); - - return (FALSE); - } - if (page_compression_level != fsp_page_compression_level) { - fprintf(stderr, - "InnoDB: Error: table flags has page_compression_level %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has page_compression_level %ld\n", - page_compression_level, fsp_page_compression_level); - - return (FALSE); - } - - if (atomic_writes != fsp_atomic_writes) { - fprintf(stderr, - "InnoDB: Error: table flags has atomic writes %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has atomic_writes %ld\n", - atomic_writes, fsp_atomic_writes); - - return (FALSE); - } - - return(TRUE); -} - -/********************************************************************//** Extract the page compression level from dict_table_t::flags. These flags are in memory, so assert that they are valid. @return page compression level, or 0 if not compressed */ diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index cc67d918d5f..525a68bf5e6 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -44,7 +44,6 @@ Created 10/25/1995 Heikki Tuuri // Forward declaration struct trx_t; -struct fil_space_t; typedef std::list<const char*> space_name_list_t; @@ -271,10 +270,6 @@ struct fil_space_t { an insert buffer merge request for a page because it actually was for the previous incarnation of the space */ - ibool mark; /*!< this is set to TRUE at database startup if - the space corresponds to a table in the InnoDB - data dictionary; so we can print a warning of - orphaned tablespaces */ ibool stop_ios;/*!< TRUE if we want to rename the .ibd file of tablespace and want to stop temporarily posting of new i/o @@ -303,7 +298,8 @@ struct fil_space_t { /*!< recovered tablespace size in pages; 0 if no size change was read from the redo log, or if the size change was implemented */ - ulint flags; /*!< tablespace flags; see + ulint flags; /*!< FSP_SPACE_FLAGS and FSP_FLAGS_MEM_ flags; + see fsp0fsp.h, fsp_flags_is_valid(), fsp_flags_get_zip_size() */ ulint n_reserved_extents; @@ -455,6 +451,7 @@ fil_node_create( ibool is_raw) /*!< in: TRUE if a raw device or a raw disk partition */ MY_ATTRIBUTE((nonnull, warn_unused_result)); + #ifdef UNIV_LOG_ARCHIVE /****************************************************************//** Drops files from the start of a file space, so that its size is cut by @@ -618,7 +615,7 @@ fil_read_first_page( ibool one_read_already, /*!< in: TRUE if min and max parameters below already contain sensible data */ - ulint* flags, /*!< out: tablespace flags */ + ulint* flags, /*!< out: FSP_SPACE_FLAGS */ ulint* space_id, /*!< out: tablespace ID */ #ifdef UNIV_LOG_ARCHIVE ulint* min_arch_log_no, /*!< out: min of archived @@ -836,6 +833,14 @@ fil_create_new_single_table_tablespace( ulint key_id) /*!< in: encryption key_id */ __attribute__((nonnull, warn_unused_result)); #ifndef UNIV_HOTBACKUP +/** Try to adjust FSP_SPACE_FLAGS if they differ from the expectations. +(Typically when upgrading from MariaDB 10.1.0..10.1.20.) +@param[in] space_id tablespace ID +@param[in] flags desired tablespace flags */ +UNIV_INTERN +void +fsp_flags_try_adjust(ulint space_id, ulint flags); + /********************************************************************//** Tries to open a single-table tablespace and optionally checks the space id is right in it. If does not succeed, prints an error message to the .err log. This @@ -864,7 +869,7 @@ fil_open_single_table_tablespace( bool validate, /*!< in: Do we validate tablespace? */ bool fix_dict, /*!< in: Can we fix the dictionary? */ ulint id, /*!< in: space id */ - ulint flags, /*!< in: tablespace flags */ + ulint flags, /*!< in: expected FSP_SPACE_FLAGS */ const char* tablename, /*!< in: table name in the databasename/tablename format */ const char* filepath, /*!< in: tablespace filepath */ @@ -905,25 +910,18 @@ fil_tablespace_exists_in_mem( /*=========================*/ ulint id); /*!< in: space id */ #ifndef UNIV_HOTBACKUP -/*******************************************************************//** -Returns TRUE if a matching tablespace exists in the InnoDB tablespace memory +/** Check if a matching tablespace exists in the InnoDB tablespace memory cache. Note that if we have not done a crash recovery at the database startup, there may be many tablespaces which are not yet in the memory cache. -@return TRUE if a matching tablespace exists in the memory cache */ +@return whether a matching tablespace exists in the memory cache */ UNIV_INTERN -ibool +bool fil_space_for_table_exists_in_mem( /*==============================*/ ulint id, /*!< in: space id */ const char* name, /*!< in: table name in the standard 'databasename/tablename' format */ - ibool mark_space, /*!< in: in crash recovery, at database - startup we mark all spaces which have - an associated table in the InnoDB - data dictionary, so that - we can print a warning about orphaned - tablespaces */ - ibool print_error_if_does_not_exist, + bool print_error_if_does_not_exist, /*!< in: print detailed error information to the .err log if a matching tablespace is not found from @@ -931,7 +929,8 @@ fil_space_for_table_exists_in_mem( bool adjust_space, /*!< in: whether to adjust space id when find table space mismatch */ mem_heap_t* heap, /*!< in: heap memory */ - table_id_t table_id); /*!< in: table id */ + table_id_t table_id, /*!< in: table id */ + ulint table_flags); /*!< in: table flags */ #else /* !UNIV_HOTBACKUP */ /********************************************************************//** Extends all tablespaces to the size stored in the space header. During the diff --git a/storage/innobase/include/fil0pagecompress.h b/storage/innobase/include/fil0pagecompress.h index 10db59fb218..1fe5cb66bf6 100644 --- a/storage/innobase/include/fil0pagecompress.h +++ b/storage/innobase/include/fil0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2016 MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017 MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -44,20 +44,11 @@ Returns the page compression flag of the space, or false if the space is not compressed. The tablespace must be cached in the memory cache. @return true if page compressed, false if not or space not found */ UNIV_INLINE -ibool +bool fil_space_is_page_compressed( /*=========================*/ ulint id); /*!< in: space id */ /*******************************************************************//** -Returns the page compression flag of the space, or false if the space -is not compressed. The tablespace must be cached in the memory cache. -@return true if page compressed, false if not or space not found */ -UNIV_INTERN -ibool -fil_space_get_page_compressed( -/*=========================*/ - fil_space_t* space); /*!< in: space id */ -/*******************************************************************//** Returns the atomic writes flag of the space, or false if the space is not using atomic writes. The tablespace must be cached in the memory cache. @return atomic write table option value */ diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index abcd5721a47..24fe382b344 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -52,28 +52,67 @@ to the two Barracuda row formats COMPRESSED and DYNAMIC. */ #define FSP_FLAGS_WIDTH_ATOMIC_BLOBS 1 /** Number of flag bits used to indicate the tablespace page size */ #define FSP_FLAGS_WIDTH_PAGE_SSIZE 4 -/** Width of the DATA_DIR flag. This flag indicates that the tablespace -is found in a remote location, not the default data directory. */ -#define FSP_FLAGS_WIDTH_DATA_DIR 1 -/** Number of flag bits used to indicate the page compression and compression level */ -#define FSP_FLAGS_WIDTH_PAGE_COMPRESSION 1 -#define FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL 4 +/** Number of reserved bits */ +#define FSP_FLAGS_WIDTH_RESERVED 6 +/** Number of flag bits used to indicate the page compression */ +#define FSP_FLAGS_WIDTH_PAGE_COMPRESSION 1 -/** Number of flag bits used to indicate atomic writes for this tablespace */ -#define FSP_FLAGS_WIDTH_ATOMIC_WRITES 2 - -/** Width of all the currently known tablespace flags */ +/** Width of all the currently known persistent tablespace flags */ #define FSP_FLAGS_WIDTH (FSP_FLAGS_WIDTH_POST_ANTELOPE \ + FSP_FLAGS_WIDTH_ZIP_SSIZE \ + FSP_FLAGS_WIDTH_ATOMIC_BLOBS \ + FSP_FLAGS_WIDTH_PAGE_SSIZE \ - + FSP_FLAGS_WIDTH_DATA_DIR \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES ) - -/** A mask of all the known/used bits in tablespace flags */ -#define FSP_FLAGS_MASK (~(~0 << FSP_FLAGS_WIDTH)) + + FSP_FLAGS_WIDTH_RESERVED \ + + FSP_FLAGS_WIDTH_PAGE_COMPRESSION) + +/** A mask of all the known/used bits in FSP_SPACE_FLAGS */ +#define FSP_FLAGS_MASK (~(~0U << FSP_FLAGS_WIDTH)) + +/* FSP_SPACE_FLAGS position and name in MySQL 5.6/MariaDB 10.0 or older +and MariaDB 10.1.20 or older MariaDB 10.1 and in MariaDB 10.1.21 +or newer. +MySQL 5.6 MariaDB 10.1.x MariaDB 10.1.21 +==================================================================== +Below flags in same offset +==================================================================== +0: POST_ANTELOPE 0:POST_ANTELOPE 0: POST_ANTELOPE +1..4: ZIP_SSIZE(0..5) 1..4:ZIP_SSIZE(0..5) 1..4: ZIP_SSIZE(0..5) +(NOTE: bit 4 is always 0) +5: ATOMIC_BLOBS 5:ATOMIC_BLOBS 5: ATOMIC_BLOBS +===================================================================== +Below note the order difference: +===================================================================== +6..9: PAGE_SSIZE(3..7) 6: COMPRESSION 6..9: PAGE_SSIZE(3..7) +10: DATA_DIR 7..10: COMP_LEVEL(0..9) 10: RESERVED (5.6 DATA_DIR) +===================================================================== +The flags below were in incorrect position in MariaDB 10.1, +or have been introduced in MySQL 5.7 or 8.0: +===================================================================== +11: UNUSED 11..12:ATOMIC_WRITES 11: RESERVED (5.7 SHARED) + 12: RESERVED (5.7 TEMPORARY) + 13..15:PAGE_SSIZE(3..7) 13: RESERVED (5.7 ENCRYPTION) + 14: RESERVED (8.0 SDI) + 15: RESERVED + 16: PAGE_SSIZE_msb(0) 16: COMPRESSION + 17: DATA_DIR 17: UNUSED + 18: UNUSED +===================================================================== +The flags below only exist in fil_space_t::flags, not in FSP_SPACE_FLAGS: +===================================================================== + 25: DATA_DIR + 26..27: ATOMIC_WRITES + 28..31: COMPRESSION_LEVEL +*/ + +/** A mask of the memory-only flags in fil_space_t::flags */ +#define FSP_FLAGS_MEM_MASK (~0U << FSP_FLAGS_MEM_DATA_DIR) + +/** Zero relative shift position of the DATA_DIR flag */ +#define FSP_FLAGS_MEM_DATA_DIR 25 +/** Zero relative shift position of the ATOMIC_WRITES field */ +#define FSP_FLAGS_MEM_ATOMIC_WRITES 26 +/** Zero relative shift position of the COMPRESSION_LEVEL field */ +#define FSP_FLAGS_MEM_COMPRESSION_LEVEL 28 /** Zero relative shift position of the POST_ANTELOPE field */ #define FSP_FLAGS_POS_POST_ANTELOPE 0 @@ -83,29 +122,16 @@ is found in a remote location, not the default data directory. */ /** Zero relative shift position of the ATOMIC_BLOBS field */ #define FSP_FLAGS_POS_ATOMIC_BLOBS (FSP_FLAGS_POS_ZIP_SSIZE \ + FSP_FLAGS_WIDTH_ZIP_SSIZE) -/** Note that these need to be before the page size to be compatible with -dictionary */ -/** Zero relative shift position of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_POS_PAGE_COMPRESSION (FSP_FLAGS_POS_ATOMIC_BLOBS \ - + FSP_FLAGS_WIDTH_ATOMIC_BLOBS) -/** Zero relative shift position of the PAGE_COMPRESSION_LEVEL field */ -#define FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL (FSP_FLAGS_POS_PAGE_COMPRESSION \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION) -/** Zero relative shift position of the ATOMIC_WRITES field */ -#define FSP_FLAGS_POS_ATOMIC_WRITES (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL) -/** Zero relative shift position of the PAGE_SSIZE field */ -#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_ATOMIC_WRITES \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES) -/** Zero relative shift position of the start of the UNUSED bits */ -#define FSP_FLAGS_POS_DATA_DIR (FSP_FLAGS_POS_PAGE_SSIZE \ - + FSP_FLAGS_WIDTH_PAGE_SSIZE) -#define FSP_FLAGS_POS_DATA_DIR_ORACLE (FSP_FLAGS_POS_ATOMIC_BLOBS \ - + FSP_FLAGS_WIDTH_ATOMIC_BLOBS \ +/** Zero relative shift position of the start of the PAGE_SSIZE bits */ +#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_ATOMIC_BLOBS \ + + FSP_FLAGS_WIDTH_ATOMIC_BLOBS) +/** Zero relative shift position of the start of the RESERVED bits +these are only used in MySQL 5.7 and used for compatibility. */ +#define FSP_FLAGS_POS_RESERVED (FSP_FLAGS_POS_PAGE_SSIZE \ + FSP_FLAGS_WIDTH_PAGE_SSIZE) -/** Zero relative shift position of the start of the UNUSED bits */ -#define FSP_FLAGS_POS_UNUSED (FSP_FLAGS_POS_DATA_DIR \ - + FSP_FLAGS_WIDTH_DATA_DIR) +/** Zero relative shift position of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_POS_PAGE_COMPRESSION (FSP_FLAGS_POS_RESERVED \ + + FSP_FLAGS_WIDTH_RESERVED) /** Bit mask of the POST_ANTELOPE field */ #define FSP_FLAGS_MASK_POST_ANTELOPE \ @@ -123,26 +149,23 @@ dictionary */ #define FSP_FLAGS_MASK_PAGE_SSIZE \ ((~(~0U << FSP_FLAGS_WIDTH_PAGE_SSIZE)) \ << FSP_FLAGS_POS_PAGE_SSIZE) -/** Bit mask of the DATA_DIR field */ -#define FSP_FLAGS_MASK_DATA_DIR \ - ((~(~0U << FSP_FLAGS_WIDTH_DATA_DIR)) \ - << FSP_FLAGS_POS_DATA_DIR) -/** Bit mask of the DATA_DIR field */ -#define FSP_FLAGS_MASK_DATA_DIR_ORACLE \ - ((~(~0U << FSP_FLAGS_WIDTH_DATA_DIR)) \ - << FSP_FLAGS_POS_DATA_DIR_ORACLE) +/** Bit mask of the RESERVED1 field */ +#define FSP_FLAGS_MASK_RESERVED \ + ((~(~0U << FSP_FLAGS_WIDTH_RESERVED)) \ + << FSP_FLAGS_POS_RESERVED) /** Bit mask of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_MASK_PAGE_COMPRESSION \ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION \ ((~(~0U << FSP_FLAGS_WIDTH_PAGE_COMPRESSION)) \ << FSP_FLAGS_POS_PAGE_COMPRESSION) -/** Bit mask of the PAGE_COMPRESSION_LEVEL field */ -#define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL \ - ((~(~0U << FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL)) \ - << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL) -/** Bit mask of the ATOMIC_WRITES field */ -#define FSP_FLAGS_MASK_ATOMIC_WRITES \ - ((~(~0U << FSP_FLAGS_WIDTH_ATOMIC_WRITES)) \ - << FSP_FLAGS_POS_ATOMIC_WRITES) + +/** Bit mask of the in-memory ATOMIC_WRITES field */ +#define FSP_FLAGS_MASK_MEM_ATOMIC_WRITES \ + (3U << FSP_FLAGS_MEM_ATOMIC_WRITES) + +/** Bit mask of the in-memory COMPRESSION_LEVEL field */ +#define FSP_FLAGS_MASK_MEM_COMPRESSION_LEVEL \ + (15U << FSP_FLAGS_MEM_COMPRESSION_LEVEL) + /** Return the value of the POST_ANTELOPE field */ #define FSP_FLAGS_GET_POST_ANTELOPE(flags) \ ((flags & FSP_FLAGS_MASK_POST_ANTELOPE) \ @@ -159,49 +182,78 @@ dictionary */ #define FSP_FLAGS_GET_PAGE_SSIZE(flags) \ ((flags & FSP_FLAGS_MASK_PAGE_SSIZE) \ >> FSP_FLAGS_POS_PAGE_SSIZE) -/** Return the value of the DATA_DIR field */ -#define FSP_FLAGS_HAS_DATA_DIR(flags) \ - ((flags & FSP_FLAGS_MASK_DATA_DIR) \ - >> FSP_FLAGS_POS_DATA_DIR) -#define FSP_FLAGS_HAS_DATA_DIR_ORACLE(flags) \ - ((flags & FSP_FLAGS_MASK_DATA_DIR_ORACLE) \ - >> FSP_FLAGS_POS_DATA_DIR_ORACLE) +/** @return the RESERVED flags */ +#define FSP_FLAGS_GET_RESERVED(flags) \ + ((flags & FSP_FLAGS_MASK_RESERVED) \ + >> FSP_FLAGS_POS_RESERVED) +/** @return the PAGE_COMPRESSION flag */ +#define FSP_FLAGS_HAS_PAGE_COMPRESSION(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION) \ + >> FSP_FLAGS_POS_PAGE_COMPRESSION) + /** Return the contents of the UNUSED bits */ #define FSP_FLAGS_GET_UNUSED(flags) \ (flags >> FSP_FLAGS_POS_UNUSED) -/** Return the value of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_GET_PAGE_COMPRESSION(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION) \ - >> FSP_FLAGS_POS_PAGE_COMPRESSION) -/** Return the value of the PAGE_COMPRESSION_LEVEL field */ +/** @return the PAGE_SSIZE flags for the current innodb_page_size */ +#define FSP_FLAGS_PAGE_SSIZE() \ + ((UNIV_PAGE_SIZE == UNIV_PAGE_SIZE_ORIG) ? \ + 0 : (UNIV_PAGE_SIZE_SHIFT - UNIV_ZIP_SIZE_SHIFT_MIN + 1) \ + << FSP_FLAGS_POS_PAGE_SSIZE) + +/** @return the value of the DATA_DIR field */ +#define FSP_FLAGS_HAS_DATA_DIR(flags) \ + (flags & 1U << FSP_FLAGS_MEM_DATA_DIR) +/** @return the COMPRESSION_LEVEL field */ #define FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL) \ - >> FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL) -/** Return the value of the ATOMIC_WRITES field */ + ((flags & FSP_FLAGS_MASK_MEM_COMPRESSION_LEVEL) \ + >> FSP_FLAGS_MEM_COMPRESSION_LEVEL) +/** @return the ATOMIC_WRITES field */ #define FSP_FLAGS_GET_ATOMIC_WRITES(flags) \ - ((flags & FSP_FLAGS_MASK_ATOMIC_WRITES) \ - >> FSP_FLAGS_POS_ATOMIC_WRITES) - -/** Set a PAGE_SSIZE into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_SSIZE(flags, ssize) \ - (flags | (ssize << FSP_FLAGS_POS_PAGE_SSIZE)) + ((flags & FSP_FLAGS_MASK_MEM_ATOMIC_WRITES) \ + >> FSP_FLAGS_MEM_ATOMIC_WRITES) -/** Set a PAGE_COMPRESSION into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_COMPRESSION(flags, compression) \ - (flags | (compression << FSP_FLAGS_POS_PAGE_COMPRESSION)) +/* Compatibility macros for MariaDB 10.1.20 or older 10.1 see +table above. */ +/** Zero relative shift position of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101 \ + (FSP_FLAGS_POS_ATOMIC_BLOBS \ + + FSP_FLAGS_WIDTH_ATOMIC_BLOBS) +/** Zero relative shift position of the PAGE_COMPRESSION_LEVEL field */ +#define FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101 \ + (FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101 + 1) +/** Zero relative shift position of the ATOMIC_WRITES field */ +#define FSP_FLAGS_POS_ATOMIC_WRITES_MARIADB101 \ + (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101 + 4) +/** Zero relative shift position of the PAGE_SSIZE field */ +#define FSP_FLAGS_POS_PAGE_SSIZE_MARIADB101 \ + (FSP_FLAGS_POS_ATOMIC_WRITES_MARIADB101 + 2) -/** Set a PAGE_COMPRESSION_LEVEL into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(flags, level) \ - (flags | (level << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL)) +/** Bit mask of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION_MARIADB101 \ + (1U << FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101) +/** Bit mask of the PAGE_COMPRESSION_LEVEL field */ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL_MARIADB101 \ + (15U << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101) +/** Bit mask of the ATOMIC_WRITES field */ +#define FSP_FLAGS_MASK_ATOMIC_WRITES_MARIADB101 \ + (3U << FSP_FLAGS_POS_ATOMIC_WRITES_MARIADB101) +/** Bit mask of the PAGE_SSIZE field */ +#define FSP_FLAGS_MASK_PAGE_SSIZE_MARIADB101 \ + (15U << FSP_FLAGS_POS_PAGE_SSIZE_MARIADB101) -/** Set a ATOMIC_WRITES into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_ATOMIC_WRITES(flags, atomics) \ - (flags | (atomics << FSP_FLAGS_POS_ATOMIC_WRITES)) +/** Return the value of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION_MARIADB101) \ + >> FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101) +/** Return the value of the PAGE_COMPRESSION_LEVEL field */ +#define FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL_MARIADB101(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL_MARIADB101) \ + >> FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101) +/** Return the value of the PAGE_SSIZE field */ +#define FSP_FLAGS_GET_PAGE_SSIZE_MARIADB101(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_SSIZE_MARIADB101) \ + >> FSP_FLAGS_POS_PAGE_SSIZE_MARIADB101) /* @} */ @@ -735,19 +787,193 @@ fseg_print( mtr_t* mtr); /*!< in/out: mini-transaction */ #endif /* UNIV_BTR_PRINT */ -/********************************************************************//** -Validate and return the tablespace flags, which are stored in the -tablespace header at offset FSP_SPACE_FLAGS. They should be 0 for -ROW_FORMAT=COMPACT and ROW_FORMAT=REDUNDANT. The newer row formats, -COMPRESSED and DYNAMIC, use a file format > Antelope so they should -have a file format number plus the DICT_TF_COMPACT bit set. -@return true if check ok */ +/** Validate the tablespace flags, which are stored in the +tablespace header at offset FSP_SPACE_FLAGS. +@param[in] flags the contents of FSP_SPACE_FLAGS +@return whether the flags are correct (not in the buggy 10.1) format */ +MY_ATTRIBUTE((warn_unused_result, const)) UNIV_INLINE bool -fsp_flags_is_valid( -/*===============*/ - ulint flags) /*!< in: tablespace flags */ - MY_ATTRIBUTE((warn_unused_result, const)); +fsp_flags_is_valid(ulint flags) +{ + DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", + return(false);); + if (flags == 0) { + return(true); + } + if (flags & ~FSP_FLAGS_MASK) { + return(false); + } + if ((flags & (FSP_FLAGS_MASK_POST_ANTELOPE | FSP_FLAGS_MASK_ATOMIC_BLOBS)) + == FSP_FLAGS_MASK_ATOMIC_BLOBS) { + /* If the "atomic blobs" flag (indicating + ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED) flag + is set, then the "post Antelope" (ROW_FORMAT!=REDUNDANT) flag + must also be set. */ + return(false); + } + /* Bits 10..14 should be 0b0000d where d is the DATA_DIR flag + of MySQL 5.6 and MariaDB 10.0, which we ignore. + In the buggy FSP_SPACE_FLAGS written by MariaDB 10.1.0 to 10.1.20, + bits 10..14 would be nonzero 0bsssaa where sss is + nonzero PAGE_SSIZE (3, 4, 6, or 7) + and aa is ATOMIC_WRITES (not 0b11). */ + if (FSP_FLAGS_GET_RESERVED(flags) & ~1) { + return(false); + } + + const ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); + if (ssize == 1 || ssize == 2 || ssize == 5 || ssize & 8) { + /* the page_size is not between 4k and 64k; + 16k should be encoded as 0, not 5 */ + return(false); + } + const ulint zssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); + if (zssize == 0) { + /* not ROW_FORMAT=COMPRESSED */ + } else if (zssize > (ssize ? ssize : 5)) { + /* invalid KEY_BLOCK_SIZE */ + return(false); + } else if (~flags & (FSP_FLAGS_MASK_POST_ANTELOPE + | FSP_FLAGS_MASK_ATOMIC_BLOBS)) { + /* both these flags should be set for + ROW_FORMAT=COMPRESSED */ + return(false); + } + + return(true); +} + +/** Convert FSP_SPACE_FLAGS from the buggy MariaDB 10.1.0..10.1.20 format. +@param[in] flags the contents of FSP_SPACE_FLAGS +@return the flags corrected from the buggy MariaDB 10.1 format +@retval ULINT_UNDEFINED if the flags are not in the buggy 10.1 format */ +MY_ATTRIBUTE((warn_unused_result, const)) +UNIV_INLINE +ulint +fsp_flags_convert_from_101(ulint flags) +{ + DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", + return(ULINT_UNDEFINED);); + if (flags == 0) { + return(flags); + } + + if (flags >> 18) { + /* The most significant FSP_SPACE_FLAGS bit that was ever set + by MariaDB 10.1.0 to 10.1.20 was bit 17 (misplaced DATA_DIR flag). + The flags must be less than 1<<18 in order to be valid. */ + return(ULINT_UNDEFINED); + } + + if ((flags & (FSP_FLAGS_MASK_POST_ANTELOPE | FSP_FLAGS_MASK_ATOMIC_BLOBS)) + == FSP_FLAGS_MASK_ATOMIC_BLOBS) { + /* If the "atomic blobs" flag (indicating + ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED) flag + is set, then the "post Antelope" (ROW_FORMAT!=REDUNDANT) flag + must also be set. */ + return(ULINT_UNDEFINED); + } + + /* Bits 6..10 denote compression in MariaDB 10.1.0 to 10.1.20. + They must be either 0b00000 or 0b00011 through 0b10011. + In correct versions, these bits would be + 0bd0sss where d is the DATA_DIR flag (garbage bit) and + sss is the PAGE_SSIZE (3, 4, 6, or 7). + + NOTE: MariaDB 10.1.0 to 10.1.20 can misinterpret + uncompressed data files with innodb_page_size=4k or 64k as + compressed innodb_page_size=16k files. Below is an exhaustive + state space analysis. + + -0by1zzz: impossible (the bit 4 must be clean; see above) + -0b101xx: DATA_DIR, innodb_page_size>4k: invalid (COMPRESSION_LEVEL>9) + +0bx0011: innodb_page_size=4k: + !!! Misinterpreted as COMPRESSION_LEVEL=9 or 1, COMPRESSION=1. + -0bx0010: impossible, because sss must be 0b011 or 0b1xx + -0bx0001: impossible, because sss must be 0b011 or 0b1xx + -0b10000: DATA_DIR, innodb_page_size=16: + invalid (COMPRESSION_LEVEL=8 but COMPRESSION=0) + +0b00111: no DATA_DIR, innodb_page_size=64k: + !!! Misinterpreted as COMPRESSION_LEVEL=3, COMPRESSION=1. + -0b00101: impossible, because sss must be 0 for 16k, not 0b101 + -0b001x0: no DATA_DIR, innodb_page_size=32k or 8k: + invalid (COMPRESSION_LEVEL=3 but COMPRESSION=0) + +0b00000: innodb_page_size=16k (looks like COMPRESSION=0) + ??? Could actually be compressed; see PAGE_SSIZE below */ + const ulint level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL_MARIADB101( + flags); + if (FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) != (level != 0) + || level > 9) { + /* The compression flags are not in the buggy MariaDB + 10.1 format. */ + return(ULINT_UNDEFINED); + } + if (!(~flags & FSP_FLAGS_MASK_ATOMIC_WRITES_MARIADB101)) { + /* The ATOMIC_WRITES flags cannot be 0b11. + (The bits 11..12 should actually never be 0b11, + because in MySQL they would be SHARED|TEMPORARY.) */ + return(ULINT_UNDEFINED); + } + + /* Bits 13..16 are the wrong position for PAGE_SSIZE, and they + should contain one of the values 3,4,6,7, that is, be of the form + 0b0011 or 0b01xx (except 0b0110). + In correct versions, these bits should be 0bc0se + where c is the MariaDB COMPRESSED flag + and e is the MySQL 5.7 ENCRYPTION flag + and s is the MySQL 8.0 SDI flag. MariaDB can only support s=0, e=0. + + Compressed innodb_page_size=16k tables with correct FSP_SPACE_FLAGS + will be properly rejected by older MariaDB 10.1.x because they + would read as PAGE_SSIZE>=8 which is not valid. */ + + const ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE_MARIADB101(flags); + if (ssize == 1 || ssize == 2 || ssize == 5 || ssize & 8) { + /* the page_size is not between 4k and 64k; + 16k should be encoded as 0, not 5 */ + return(ULINT_UNDEFINED); + } + const ulint zssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); + if (zssize == 0) { + /* not ROW_FORMAT=COMPRESSED */ + } else if (zssize > (ssize ? ssize : 5)) { + /* invalid KEY_BLOCK_SIZE */ + return(ULINT_UNDEFINED); + } else if (~flags & (FSP_FLAGS_MASK_POST_ANTELOPE + | FSP_FLAGS_MASK_ATOMIC_BLOBS)) { + /* both these flags should be set for + ROW_FORMAT=COMPRESSED */ + return(ULINT_UNDEFINED); + } + + flags = ((flags & 0x3f) | ssize << FSP_FLAGS_POS_PAGE_SSIZE + | FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) + << FSP_FLAGS_POS_PAGE_COMPRESSION); + ut_ad(fsp_flags_is_valid(flags)); + return(flags); +} + +/** Compare tablespace flags. +@param[in] expected expected flags from dict_tf_to_fsp_flags() +@param[in] actual flags read from FSP_SPACE_FLAGS +@return whether the flags match */ +MY_ATTRIBUTE((warn_unused_result)) +UNIV_INLINE +bool +fsp_flags_match(ulint expected, ulint actual) +{ + expected &= ~FSP_FLAGS_MEM_MASK; + ut_ad(fsp_flags_is_valid(expected)); + + if (actual == expected) { + return(true); + } + + actual = fsp_flags_convert_from_101(actual); + return(actual == expected); +} + /********************************************************************//** Determine if the tablespace is compressed from dict_table_t::flags. @return TRUE if compressed, FALSE if not compressed */ diff --git a/storage/innobase/include/fsp0fsp.ic b/storage/innobase/include/fsp0fsp.ic index 9f09a9d53e1..59c732b7a29 100644 --- a/storage/innobase/include/fsp0fsp.ic +++ b/storage/innobase/include/fsp0fsp.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2012, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -47,107 +47,6 @@ fsp_descr_page( } /********************************************************************//** -Validate and return the tablespace flags, which are stored in the -tablespace header at offset FSP_SPACE_FLAGS. They should be 0 for -ROW_FORMAT=COMPACT and ROW_FORMAT=REDUNDANT. The newer row formats, -COMPRESSED and DYNAMIC, use a file format > Antelope so they should -have a file format number plus the DICT_TF_COMPACT bit set. -@return true if check ok */ -UNIV_INLINE -bool -fsp_flags_is_valid( -/*===============*/ - ulint flags) /*!< in: tablespace flags */ -{ - ulint post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(flags); - ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); - ulint atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(flags); - ulint page_ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); - ulint unused = FSP_FLAGS_GET_UNUSED(flags); - ulint page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(flags); - ulint page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags); - ulint atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(flags); - - DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", return(false);); - - /* fsp_flags is zero unless atomic_blobs is set. */ - /* Make sure there are no bits that we do not know about. */ - if (unused != 0 || flags == 1) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted unused %lu\n", - flags, unused); - return(false); - } else if (post_antelope) { - /* The Antelope row formats REDUNDANT and COMPACT did - not use tablespace flags, so this flag and the entire - 4-byte field is zero for Antelope row formats. */ - - if (!atomic_blobs) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted atomic_blobs %lu\n", - flags, atomic_blobs); - return(false); - } - } - - if (!atomic_blobs) { - /* Barracuda row formats COMPRESSED and DYNAMIC build on - the page structure introduced for the COMPACT row format - by allowing long fields to be broken into prefix and - externally stored parts. */ - - if (post_antelope || zip_ssize != 0) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted zip_ssize %lu atomic_blobs %lu\n", - flags, zip_ssize, atomic_blobs); - return(false); - } - - } else if (!post_antelope || zip_ssize > PAGE_ZIP_SSIZE_MAX) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted zip_ssize %lu max %d\n", - flags, zip_ssize, PAGE_ZIP_SSIZE_MAX); - return(false); - } else if (page_ssize > UNIV_PAGE_SSIZE_MAX) { - - /* The page size field can be used for any row type, or it may - be zero for an original 16k page size. - Validate the page shift size is within allowed range. */ - - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted page_ssize %lu max %lu\n", - flags, page_ssize, UNIV_PAGE_SSIZE_MAX); - return(false); - - } else if (UNIV_PAGE_SIZE != UNIV_PAGE_SIZE_ORIG && !page_ssize) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted page_ssize %lu max %lu:%d\n", - flags, page_ssize, UNIV_PAGE_SIZE, UNIV_PAGE_SIZE_ORIG); - return(false); - } - - /* Page compression level requires page compression and atomic blobs - to be set */ - if (page_compression_level || page_compression) { - if (!page_compression || !atomic_blobs) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted page_compression %lu\n" - "InnoDB: Error: page_compression_level %lu atomic_blobs %lu\n", - flags, page_compression, page_compression_level, atomic_blobs); - return(false); - } - } - - if (atomic_writes > ATOMIC_WRITES_OFF) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted atomic_writes %lu\n", - flags, atomic_writes); - return (false); - } - -#if UNIV_FORMAT_MAX != UNIV_FORMAT_B -# error "UNIV_FORMAT_MAX != UNIV_FORMAT_B, Add more validations." -#endif - - /* The DATA_DIR field can be used for any row type so there is - nothing here to validate. */ - - return(true); -} - -/********************************************************************//** Determine if the tablespace is compressed from dict_table_t::flags. @return TRUE if compressed, FALSE if not compressed */ UNIV_INLINE @@ -191,10 +90,10 @@ UNIV_INLINE ulint fsp_flags_get_page_size( /*====================*/ - ulint flags) /*!< in: tablespace flags */ + ulint flags) /*!< in: tablespace flags */ { - ulint page_size = 0; - ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); + ulint page_size = 0; + ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); /* Convert from a 'log2 minus 9' to a page size in bytes. */ if (UNIV_UNLIKELY(ssize)) { @@ -211,50 +110,6 @@ fsp_flags_get_page_size( } #ifndef UNIV_INNOCHECKSUM - -/********************************************************************//** -Add the page size to the tablespace flags. -@return tablespace flags after page size is added */ -UNIV_INLINE -ulint -fsp_flags_set_page_size( -/*====================*/ - ulint flags, /*!< in: tablespace flags */ - ulint page_size) /*!< in: page size in bytes */ -{ - ulint ssize = 0; - ulint shift; - - /* Page size should be > UNIV_PAGE_SIZE_MIN */ - ut_ad(page_size >= UNIV_PAGE_SIZE_MIN); - ut_ad(page_size <= UNIV_PAGE_SIZE_MAX); - - if (page_size == UNIV_PAGE_SIZE_ORIG) { - ut_ad(0 == FSP_FLAGS_GET_PAGE_SSIZE(flags)); - return(flags); - } - - for (shift = UNIV_PAGE_SIZE_SHIFT_MAX; - shift >= UNIV_PAGE_SIZE_SHIFT_MIN; - shift--) { - ulint mask = (1 << shift); - if (page_size & mask) { - ut_ad(!(page_size & ~mask)); - ssize = shift - UNIV_ZIP_SIZE_SHIFT_MIN + 1; - break; - } - } - - ut_ad(ssize); - ut_ad(ssize <= UNIV_PAGE_SSIZE_MAX); - - flags = FSP_FLAGS_SET_PAGE_SSIZE(flags, ssize); - - ut_ad(fsp_flags_is_valid(flags)); - - return(flags); -} - /********************************************************************//** Calculates the descriptor index within a descriptor page. @return descriptor index */ diff --git a/storage/innobase/include/fsp0pagecompress.h b/storage/innobase/include/fsp0pagecompress.h index 5f943ee2b83..c623d11c326 100644 --- a/storage/innobase/include/fsp0pagecompress.h +++ b/storage/innobase/include/fsp0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -48,15 +48,6 @@ fsp_header_get_compression_level( const page_t* page); /*!< in: first page of a tablespace */ /********************************************************************//** -Determine if the tablespace is page compressed from dict_table_t::flags. -@return TRUE if page compressed, FALSE if not compressed */ -UNIV_INLINE -ibool -fsp_flags_is_page_compressed( -/*=========================*/ - ulint flags); /*!< in: tablespace flags */ - -/********************************************************************//** Extract the page compression level from tablespace flags. A tablespace has only one physical page compression level whether that page is compressed or not. diff --git a/storage/innobase/include/fsp0pagecompress.ic b/storage/innobase/include/fsp0pagecompress.ic index e879aa2c16e..48163277feb 100644 --- a/storage/innobase/include/fsp0pagecompress.ic +++ b/storage/innobase/include/fsp0pagecompress.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -26,18 +26,6 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@mariadb.com ***********************************************************************/ /********************************************************************//** -Determine if the tablespace is page compressed from dict_table_t::flags. -@return TRUE if page compressed, FALSE if not page compressed */ -UNIV_INLINE -ibool -fsp_flags_is_page_compressed( -/*=========================*/ - ulint flags) /*!< in: tablespace flags */ -{ - return(FSP_FLAGS_GET_PAGE_COMPRESSION(flags)); -} - -/********************************************************************//** Determine the tablespace is page compression level from dict_table_t::flags. @return page compression level or 0 if not compressed*/ UNIV_INLINE @@ -125,21 +113,15 @@ Extract the page compression from space. @return true if space is page compressed, false if space is not found or space is not page compressed. */ UNIV_INLINE -ibool +bool fil_space_is_page_compressed( /*=========================*/ ulint id) /*!< in: space id */ { - ulint flags; - - flags = fil_space_get_flags(id); - - if (flags && flags != ULINT_UNDEFINED) { - - return(fsp_flags_is_page_compressed(flags)); - } + ulint flags = fil_space_get_flags(id); - return(0); + return(flags != ULINT_UNDEFINED + && FSP_FLAGS_HAS_PAGE_COMPRESSION(flags)); } #endif /* UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index 3a47e3fc6aa..f54c9d589c2 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -371,8 +371,7 @@ public: m_space(ULINT_UNDEFINED), m_xdes(), m_xdes_page_no(ULINT_UNDEFINED), - m_space_flags(ULINT_UNDEFINED), - m_table_flags(ULINT_UNDEFINED) UNIV_NOTHROW { } + m_space_flags(ULINT_UNDEFINED) UNIV_NOTHROW { } /** Free any extent descriptor instance */ @@ -535,10 +534,6 @@ protected: /** Flags value read from the header page */ ulint m_space_flags; - - /** Derived from m_space_flags and row format type, the row format - type is determined from the page header. */ - ulint m_table_flags; }; /** Determine the page size to use for traversing the tablespace @@ -553,6 +548,19 @@ AbstractCallback::init( const page_t* page = block->frame; m_space_flags = fsp_header_get_flags(page); + if (!fsp_flags_is_valid(m_space_flags)) { + ulint cflags = fsp_flags_convert_from_101(m_space_flags); + if (cflags == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Invalid FSP_SPACE_FLAGS=0x%x", + int(m_space_flags)); + return(DB_CORRUPTION); + } + m_space_flags = cflags; + } + + /* Clear the DATA_DIR flag, which is basically garbage. */ + m_space_flags &= ~(1U << FSP_FLAGS_POS_RESERVED); /* Since we don't know whether it is a compressed table or not, the data is always read into the block->frame. */ @@ -641,46 +649,6 @@ struct FetchIndexRootPages : public AbstractCallback { } /** - Check if the .ibd file row format is the same as the table's. - @param ibd_table_flags - determined from space and page. - @return DB_SUCCESS or error code. */ - dberr_t check_row_format(ulint ibd_table_flags) UNIV_NOTHROW - { - dberr_t err; - rec_format_t ibd_rec_format; - rec_format_t table_rec_format; - - if (!dict_tf_is_valid(ibd_table_flags)) { - - ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - ".ibd file has invlad table flags: %lx", - ibd_table_flags); - - return(DB_CORRUPTION); - } - - ibd_rec_format = dict_tf_get_rec_format(ibd_table_flags); - table_rec_format = dict_tf_get_rec_format(m_table->flags); - - if (table_rec_format != ibd_rec_format) { - - ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Table has %s row format, .ibd " - "file has %s row format.", - dict_tf_to_row_format_string(m_table->flags), - dict_tf_to_row_format_string(ibd_table_flags)); - - err = DB_CORRUPTION; - } else { - err = DB_SUCCESS; - } - - return(err); - } - - /** Called for each block as it is read from the file. @param offset - physical offset in the file @param block - block to convert, it is not from the buffer pool. @@ -743,12 +711,17 @@ FetchIndexRootPages::operator() ( m_indexes.push_back(Index(id, page_no)); if (m_indexes.size() == 1) { - - m_table_flags = dict_sys_tables_type_to_tf( - m_space_flags, - page_is_comp(page) ? DICT_N_COLS_COMPACT : 0); - - err = check_row_format(m_table_flags); + /* Check that the tablespace flags match the table flags. */ + ulint expected = dict_tf_to_fsp_flags(m_table->flags); + if (!fsp_flags_match(expected, m_space_flags)) { + ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, + ER_TABLE_SCHEMA_MISMATCH, + "Expected FSP_SPACE_FLAGS=0x%x, .ibd " + "file contains 0x%x.", + unsigned(expected), + unsigned(m_space_flags)); + return(DB_CORRUPTION); + } } } @@ -1965,21 +1938,14 @@ PageConverter::update_header( "- ignored"); } - ulint space_flags = fsp_header_get_flags(get_frame(block)); - - if (!fsp_flags_is_valid(space_flags)) { - - ib_logf(IB_LOG_LEVEL_ERROR, - "Unsupported tablespace format %lu", - (ulong) space_flags); - - return(DB_UNSUPPORTED); - } - mach_write_to_8( get_frame(block) + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, m_current_lsn); + /* Write back the adjusted flags. */ + mach_write_to_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + + get_frame(block), m_space_flags); + /* Write space_id to the tablespace header, page 0. */ mach_write_to_4( get_frame(block) + FSP_HEADER_OFFSET + FSP_SPACE_ID, diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 1c02590933f..0e539f53641 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -4335,6 +4335,7 @@ row_drop_table_for_mysql( switch (err) { ibool is_temp; + ulint table_flags; case DB_SUCCESS: /* Clone the name, in case it has been allocated @@ -4343,6 +4344,7 @@ row_drop_table_for_mysql( space_id = table->space; ibd_file_missing = table->ibd_file_missing; + table_flags = table->flags; is_temp = DICT_TF2_FLAG_IS_SET(table, DICT_TF2_TEMPORARY); /* If there is a temp path then the temp flag is set. @@ -4358,9 +4360,9 @@ row_drop_table_for_mysql( } /* We do not allow temporary tables with a remote path. */ - ut_a(!(is_temp && DICT_TF_HAS_DATA_DIR(table->flags))); + ut_a(!(is_temp && DICT_TF_HAS_DATA_DIR(table_flags))); - if (space_id && DICT_TF_HAS_DATA_DIR(table->flags)) { + if (space_id && DICT_TF_HAS_DATA_DIR(table_flags)) { dict_get_and_save_data_dir_path(table, true); ut_a(table->data_dir_path); @@ -4426,8 +4428,9 @@ row_drop_table_for_mysql( if (err == DB_SUCCESS && space_id > TRX_SYS_SPACE) { if (!is_temp && !fil_space_for_table_exists_in_mem( - space_id, tablename, FALSE, - print_msg, false, NULL, 0)) { + space_id, tablename, + print_msg, false, NULL, 0, + table_flags)) { /* This might happen if we are dropping a discarded tablespace */ err = DB_SUCCESS; diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 40154c11ce1..2fec8b7bc2f 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -673,8 +673,7 @@ create_log_files( sprintf(logfilename + dirnamelen, "ib_logfile%u", INIT_LOG_FILE0); fil_space_create( - logfilename, SRV_LOG_SPACE_FIRST_ID, - fsp_flags_set_page_size(0, UNIV_PAGE_SIZE), + logfilename, SRV_LOG_SPACE_FIRST_ID, 0, FIL_LOG, NULL /* no encryption yet */, true /* this is create */); @@ -1159,7 +1158,7 @@ check_first_page: crypt_data = fil_space_create_crypt_data(FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY); } - flags = fsp_flags_set_page_size(0, UNIV_PAGE_SIZE); + flags = FSP_FLAGS_PAGE_SSIZE(); fil_space_create(name, 0, flags, FIL_TABLESPACE, crypt_data, (*create_new_db) == true); @@ -1307,7 +1306,7 @@ srv_undo_tablespace_open( fil_set_max_space_id_if_bigger(space); /* Set the compressed page size to 0 (non-compressed) */ - flags = fsp_flags_set_page_size(0, UNIV_PAGE_SIZE); + flags = FSP_FLAGS_PAGE_SSIZE(); fil_space_create(name, space, flags, FIL_TABLESPACE, NULL /* no encryption */, true /* create */); @@ -2296,9 +2295,7 @@ innobase_start_or_create_for_mysql(void) sprintf(logfilename + dirnamelen, "ib_logfile%u", 0); fil_space_create(logfilename, - SRV_LOG_SPACE_FIRST_ID, - fsp_flags_set_page_size(0, UNIV_PAGE_SIZE), - FIL_LOG, + SRV_LOG_SPACE_FIRST_ID, 0, FIL_LOG, NULL /* no encryption yet */, true /* create */); @@ -2714,6 +2711,13 @@ files_checked: } if (!srv_read_only_mode) { + const ulint flags = FSP_FLAGS_PAGE_SSIZE(); + for (ulint id = 0; id <= srv_undo_tablespaces; id++) { + if (fil_space_get(id)) { + fsp_flags_try_adjust(id, flags); + } + } + /* Create the thread which watches the timeouts for lock waits */ thread_handles[2 + SRV_MAX_N_IO_THREADS] = os_thread_create( diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index 9c7eea6410a..68bb83e4903 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -589,6 +589,21 @@ buf_dblwr_process() continue; } + if (page_no == 0) { + /* Check the FSP_SPACE_FLAGS. */ + ulint flags = fsp_header_get_flags(page); + if (!fsp_flags_is_valid(flags) + && fsp_flags_convert_from_101(flags) + == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_WARN, + "Ignoring a doublewrite copy of page " + ULINTPF ":0 due to invalid flags 0x%x", + space_id, int(flags)); + continue; + } + /* The flags on the page should be converted later. */ + } + /* Write the good page from the doublewrite buffer to the intended position. */ diff --git a/storage/xtradb/dict/dict0load.cc b/storage/xtradb/dict/dict0load.cc index 39a6ae993c8..776bb873c5c 100644 --- a/storage/xtradb/dict/dict0load.cc +++ b/storage/xtradb/dict/dict0load.cc @@ -1052,8 +1052,6 @@ loop: btr_pcur_store_position(&pcur, &mtr); - mtr_commit(&mtr); - /* For tables created with old versions of InnoDB, SYS_TABLES.MIX_LEN may contain garbage. Such tables would always be in ROW_FORMAT=REDUNDANT. Pretend that @@ -1087,16 +1085,19 @@ loop: if (space_id == 0) { /* The system tablespace always exists. */ ut_ad(!discarded); - goto next_tablespace; + mem_free(name); + goto loop; } + mtr_commit(&mtr); + switch (dict_check) { case DICT_CHECK_ALL_LOADED: /* All tablespaces should have been found in fil_load_single_table_tablespaces(). */ if (fil_space_for_table_exists_in_mem( - space_id, name, TRUE, !(is_temp || discarded), - false, NULL, 0) + space_id, name, !(is_temp || discarded), + false, NULL, 0, flags) && !(is_temp || discarded)) { /* If user changes the path of .ibd files in *.isl files before doing crash recovery , @@ -1128,8 +1129,8 @@ loop: /* Some tablespaces may have been opened in trx_resurrect_table_locks(). */ if (fil_space_for_table_exists_in_mem( - space_id, name, FALSE, FALSE, - false, NULL, 0)) { + space_id, name, false, + false, NULL, 0, flags)) { break; } /* fall through */ @@ -1191,7 +1192,6 @@ loop: max_space_id = space_id; } -next_tablespace: mem_free(name); mtr_start(&mtr); @@ -2383,8 +2383,8 @@ err_exit: table->ibd_file_missing = TRUE; } else if (!fil_space_for_table_exists_in_mem( - table->space, name, FALSE, FALSE, true, heap, - table->id)) { + table->space, name, false, true, heap, + table->id, table->flags)) { if (DICT_TF2_FLAG_IS_SET(table, DICT_TF2_TEMPORARY)) { /* Do not bother to retry opening temporary tables. */ diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 133960ae8b4..f4301d47028 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -601,10 +601,7 @@ fil_node_open_file( ibool success; byte* buf2; byte* page; - ulint space_id; - ulint flags=0; ulint page_size; - ulint atomic_writes=0; ut_ad(mutex_own(&(system->mutex))); ut_a(node->n_pending == 0); @@ -626,8 +623,6 @@ fil_node_open_file( /* The following call prints an error message */ os_file_get_last_error(true); - ut_print_timestamp(stderr); - ib_logf(IB_LOG_LEVEL_WARN, "InnoDB: Error: cannot " "open %s\n. InnoDB: Have you deleted .ibd " "files under a running mysqld server?\n", @@ -653,17 +648,13 @@ fil_node_open_file( ut_a(fil_is_user_tablespace_id(space->id)); if (size_bytes < FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE) { - fprintf(stderr, - "InnoDB: Error: the size of single-table" - " tablespace file %s\n" - "InnoDB: is only " UINT64PF "," - " should be at least %lu!\n", - node->name, - size_bytes, - (ulong) (FIL_IBD_FILE_INITIAL_SIZE - * UNIV_PAGE_SIZE)); - - ut_a(0); + ib_logf(IB_LOG_LEVEL_ERROR, + "The size of the file %s is only " UINT64PF + " bytes, should be at least " ULINTPF, + node->name, size_bytes, + FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE); + os_file_close(node->handle); + return(false); } /* Read the first page of the tablespace */ @@ -676,77 +667,34 @@ fil_node_open_file( success = os_file_read(node->handle, page, 0, UNIV_PAGE_SIZE); srv_stats.page0_read.add(1); - space_id = fsp_header_get_space_id(page); - flags = fsp_header_get_flags(page); - - page_size = fsp_flags_get_page_size(flags); - atomic_writes = fsp_flags_get_atomic_writes(flags); + const ulint space_id = fsp_header_get_space_id(page); + ulint flags = fsp_header_get_flags(page); ut_free(buf2); - - /* Close the file now that we have read the space id from it */ - os_file_close(node->handle); - if (UNIV_UNLIKELY(space_id != space->id)) { - fprintf(stderr, - "InnoDB: Error: tablespace id is %lu" - " in the data dictionary\n" - "InnoDB: but in file %s it is %lu!\n", - space->id, node->name, space_id); - - ut_error; - } - - if (UNIV_UNLIKELY(space_id == ULINT_UNDEFINED - || space_id == 0)) { - fprintf(stderr, - "InnoDB: Error: tablespace id %lu" - " in file %s is not sensible\n", - (ulong) space_id, node->name); - - ut_error; - } - - if (UNIV_UNLIKELY(fsp_flags_get_page_size(space->flags) - != page_size)) { - fprintf(stderr, - "InnoDB: Error: tablespace file %s" - " has page size 0x%lx\n" - "InnoDB: but the data dictionary" - " expects page size 0x%lx!\n", - node->name, flags, - fsp_flags_get_page_size(space->flags)); + if (!fsp_flags_is_valid(flags)) { + ulint cflags = fsp_flags_convert_from_101(flags); + if (cflags == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Expected tablespace flags 0x%x" + " but found 0x%x in the file %s", + int(space->flags), int(flags), + node->name); + return(false); + } - ut_error; + flags = cflags; } - if (UNIV_UNLIKELY(space->flags != flags)) { - ulint sflags = (space->flags & ~FSP_FLAGS_MASK_DATA_DIR); - ulint fflags = (flags & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - - /* DATA_DIR option is on different place on MariaDB - compared to MySQL. If this is the difference. Fix - it. */ - - if (sflags == fflags) { - fprintf(stderr, - "InnoDB: Warning: Table flags 0x%lx" - " in the data dictionary but in file %s are 0x%lx!\n" - " Temporally corrected because DATA_DIR option to 0x%lx.\n", - space->flags, node->name, flags, space->flags); - - flags = space->flags; - } + page_size = fsp_flags_get_page_size(flags); - if (!dict_tf_verify_flags(space->flags, flags)) { - fprintf(stderr, - "InnoDB: Error: table flags are 0x%lx" - " in the data dictionary\n" - "InnoDB: but the flags in file %s are 0x%lx!\n", - space->flags, node->name, flags); - ut_error; - } + if (UNIV_UNLIKELY(space_id != space->id)) { + ib_logf(IB_LOG_LEVEL_ERROR, + "tablespace id is " ULINTPF " in the data dictionary" + " but in file %s it is " ULINTPF "!\n", + space->id, node->name, space_id); + return(false); } if (size_bytes >= (1024*1024)) { @@ -768,7 +716,7 @@ add_size: space->size += node->size; } - atomic_writes = fsp_flags_get_atomic_writes(space->flags); + ulint atomic_writes = fsp_flags_get_atomic_writes(space->flags); /* printf("Opening file %s\n", node->name); */ @@ -1572,7 +1520,6 @@ fil_space_create( fil_system->tablespace_version++; space->tablespace_version = fil_system->tablespace_version; - space->mark = FALSE; if (purpose == FIL_TABLESPACE && !recv_recovery_on && id > fil_system->max_assigned_id) { @@ -2324,27 +2271,21 @@ fil_write_flushed_lsn_to_data_files( return(DB_SUCCESS); } -/*******************************************************************//** -Checks the consistency of the first data page of a tablespace +/** Check the consistency of the first data page of a tablespace at database startup. +@param[in] page page frame +@param[in] space_id tablespace identifier +@param[in] flags tablespace flags @retval NULL on success, or if innodb_force_recovery is set @return pointer to an error message string */ static MY_ATTRIBUTE((warn_unused_result)) const char* -fil_check_first_page( -/*=================*/ - const page_t* page) /*!< in: data page */ +fil_check_first_page(const page_t* page, ulint space_id, ulint flags) { - ulint space_id; - ulint flags; - if (srv_force_recovery >= SRV_FORCE_IGNORE_CORRUPT) { return(NULL); } - space_id = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_ID + page); - flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); - if (UNIV_PAGE_SIZE != fsp_flags_get_page_size(flags)) { fprintf(stderr, "InnoDB: Error: Current page size %lu != " @@ -2393,7 +2334,7 @@ fil_read_first_page( ibool one_read_already, /*!< in: TRUE if min and max parameters below already contain sensible data */ - ulint* flags, /*!< out: tablespace flags */ + ulint* flags, /*!< out: FSP_SPACE_FLAGS */ ulint* space_id, /*!< out: tablespace ID */ lsn_t* min_flushed_lsn, /*!< out: min of flushed lsn values in data files */ @@ -2423,12 +2364,22 @@ fil_read_first_page( *flags and *space_id as they were read from the first file and do not validate the first page. */ if (!one_read_already) { - *flags = fsp_header_get_flags(page); *space_id = fsp_header_get_space_id(page); - } + *flags = fsp_header_get_flags(page); - if (!one_read_already) { - check_msg = fil_check_first_page(page); + if (!fsp_flags_is_valid(*flags)) { + ulint cflags = fsp_flags_convert_from_101(*flags); + if (cflags == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Invalid flags 0x%x in tablespace %u", + unsigned(*flags), unsigned(*space_id)); + return "invalid tablespace flags"; + } else { + *flags = cflags; + } + } + + check_msg = fil_check_first_page(page, *space_id, *flags); } flushed_lsn = mach_read_from_8(page + @@ -2612,6 +2563,7 @@ fil_op_write_log( ulint len; log_ptr = mlog_open(mtr, 11 + 2 + 1); + ut_ad(fsp_flags_is_valid(flags)); if (!log_ptr) { /* Logging in mtr is switched off during crash recovery: @@ -3823,7 +3775,7 @@ fil_create_new_single_table_tablespace( ibool success; /* TRUE if a table is created with CREATE TEMPORARY TABLE */ bool is_temp = !!(flags2 & DICT_TF2_TEMPORARY); - bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags); + bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags) != 0; ulint atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(flags); fil_space_crypt_t *crypt_data = NULL; @@ -3831,7 +3783,7 @@ fil_create_new_single_table_tablespace( ut_ad(!srv_read_only_mode); ut_a(space_id < SRV_LOG_SPACE_FIRST_ID); ut_a(size >= FIL_IBD_FILE_INITIAL_SIZE); - ut_a(fsp_flags_is_valid(flags)); + ut_a(fsp_flags_is_valid(flags & ~FSP_FLAGS_MEM_MASK)); if (is_temp) { /* Temporary table filepath */ @@ -3924,12 +3876,9 @@ fil_create_new_single_table_tablespace( memset(page, '\0', UNIV_PAGE_SIZE); - /* Add the UNIV_PAGE_SIZE to the table flags and write them to the - tablespace header. */ - flags = fsp_flags_set_page_size(flags, UNIV_PAGE_SIZE); + flags |= FSP_FLAGS_PAGE_SSIZE(); fsp_header_init_fields(page, space_id, flags); mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id); - ut_ad(fsp_flags_is_valid(flags)); if (!(fsp_flags_is_compressed(flags))) { buf_flush_init_for_writing(page, NULL, 0); @@ -4008,7 +3957,8 @@ fil_create_new_single_table_tablespace( fil_op_write_log(flags ? MLOG_FILE_CREATE2 : MLOG_FILE_CREATE, - space_id, mlog_file_flag, flags, + space_id, mlog_file_flag, + flags & ~FSP_FLAGS_MEM_MASK, tablename, NULL, &mtr); mtr_commit(&mtr); @@ -4072,6 +4022,39 @@ fil_report_bad_tablespace( (ulong) expected_id, (ulong) expected_flags); } +/** Try to adjust FSP_SPACE_FLAGS if they differ from the expectations. +(Typically when upgrading from MariaDB 10.1.0..10.1.20.) +@param[in] space_id tablespace ID +@param[in] flags desired tablespace flags */ +UNIV_INTERN +void +fsp_flags_try_adjust(ulint space_id, ulint flags) +{ + ut_ad(!srv_read_only_mode); + ut_ad(fsp_flags_is_valid(flags)); + + mtr_t mtr; + mtr_start(&mtr); + if (buf_block_t* b = buf_page_get( + space_id, fsp_flags_get_zip_size(flags), 0, RW_X_LATCH, + &mtr)) { + ulint f = fsp_header_get_flags(b->frame); + /* Suppress the message if only the DATA_DIR flag to differs. */ + if ((f ^ flags) & ~(1U << FSP_FLAGS_POS_RESERVED)) { + ib_logf(IB_LOG_LEVEL_WARN, + "adjusting FSP_SPACE_FLAGS of tablespace " + ULINTPF " from 0x%x to 0x%x", + space_id, int(f), int(flags)); + } + if (f != flags) { + mlog_write_ulint(FSP_HEADER_OFFSET + + FSP_SPACE_FLAGS + b->frame, + flags, MLOG_4BYTES, &mtr); + } + } + mtr_commit(&mtr); +} + /********************************************************************//** Tries to open a single-table tablespace and optionally checks that the space id in it is correct. If this does not succeed, print an error message @@ -4101,7 +4084,7 @@ fil_open_single_table_tablespace( bool validate, /*!< in: Do we validate tablespace? */ bool fix_dict, /*!< in: Can we fix the dictionary? */ ulint id, /*!< in: space id */ - ulint flags, /*!< in: tablespace flags */ + ulint flags, /*!< in: expected FSP_SPACE_FLAGS */ const char* tablename, /*!< in: table name in the databasename/tablename format */ const char* path_in, /*!< in: tablespace filepath */ @@ -4126,20 +4109,13 @@ fil_open_single_table_tablespace( /* Table flags can be ULINT_UNDEFINED if dict_tf_to_fsp_flags_failure is set. */ - if (flags != ULINT_UNDEFINED) { - if (!fsp_flags_is_valid(flags)) { - return(DB_CORRUPTION); - } - } else { + if (flags == ULINT_UNDEFINED) { return(DB_CORRUPTION); } + ut_ad(fsp_flags_is_valid(flags & ~FSP_FLAGS_MEM_MASK)); atomic_writes = fsp_flags_get_atomic_writes(flags); - /* If the tablespace was relocated, we do not - compare the DATA_DIR flag */ - ulint mod_flags = flags & ~FSP_FLAGS_MASK_DATA_DIR; - memset(&def, 0, sizeof(def)); memset(&dict, 0, sizeof(dict)); memset(&remote, 0, sizeof(remote)); @@ -4217,31 +4193,17 @@ fil_open_single_table_tablespace( def.check_msg = fil_read_first_page( def.file, FALSE, &def.flags, &def.id, &def.lsn, &def.lsn, &def.crypt_data); - def.valid = !def.check_msg; if (table) { table->crypt_data = def.crypt_data; table->page_0_read = true; } - /* Validate this single-table-tablespace with SYS_TABLES, - but do not compare the DATA_DIR flag, in case the - tablespace was relocated. */ - - ulint newf = def.flags; - if (newf != mod_flags) { - if (FSP_FLAGS_HAS_DATA_DIR(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR); - } else if(FSP_FLAGS_HAS_DATA_DIR_ORACLE(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - } - } - - if (def.valid && def.id == id - && newf == mod_flags) { + def.valid = !def.check_msg && def.id == id + && fsp_flags_match(flags, def.flags); + if (def.valid) { valid_tablespaces_found++; } else { - def.valid = false; /* Do not use this tablespace. */ fil_report_bad_tablespace( def.filepath, def.check_msg, def.id, @@ -4254,30 +4216,18 @@ fil_open_single_table_tablespace( remote.check_msg = fil_read_first_page( remote.file, FALSE, &remote.flags, &remote.id, &remote.lsn, &remote.lsn, &remote.crypt_data); - remote.valid = !remote.check_msg; if (table) { table->crypt_data = remote.crypt_data; table->page_0_read = true; } - /* Validate this single-table-tablespace with SYS_TABLES, - but do not compare the DATA_DIR flag, in case the - tablespace was relocated. */ - ulint newf = remote.flags; - if (newf != mod_flags) { - if (FSP_FLAGS_HAS_DATA_DIR(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR); - } else if(FSP_FLAGS_HAS_DATA_DIR_ORACLE(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - } - } - - if (remote.valid && remote.id == id - && newf == mod_flags) { + /* Validate this single-table-tablespace with SYS_TABLES. */ + remote.valid = !remote.check_msg && remote.id == id + && fsp_flags_match(flags, remote.flags); + if (remote.valid) { valid_tablespaces_found++; } else { - remote.valid = false; /* Do not use this linked tablespace. */ fil_report_bad_tablespace( remote.filepath, remote.check_msg, remote.id, @@ -4291,30 +4241,19 @@ fil_open_single_table_tablespace( dict.check_msg = fil_read_first_page( dict.file, FALSE, &dict.flags, &dict.id, &dict.lsn, &dict.lsn, &dict.crypt_data); - dict.valid = !dict.check_msg; if (table) { table->crypt_data = dict.crypt_data; table->page_0_read = true; } - /* Validate this single-table-tablespace with SYS_TABLES, - but do not compare the DATA_DIR flag, in case the - tablespace was relocated. */ - ulint newf = dict.flags; - if (newf != mod_flags) { - if (FSP_FLAGS_HAS_DATA_DIR(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR); - } else if(FSP_FLAGS_HAS_DATA_DIR_ORACLE(newf)) { - newf = (newf & ~FSP_FLAGS_MASK_DATA_DIR_ORACLE); - } - } + /* Validate this single-table-tablespace with SYS_TABLES. */ + dict.valid = !dict.check_msg && dict.id == id + && fsp_flags_match(flags, dict.flags); - if (dict.valid && dict.id == id - && newf == mod_flags) { + if (dict.valid) { valid_tablespaces_found++; } else { - dict.valid = false; /* Do not use this tablespace. */ fil_report_bad_tablespace( dict.filepath, dict.check_msg, dict.id, @@ -4512,6 +4451,10 @@ cleanup_and_exit: mem_free(def.filepath); + if (err == DB_SUCCESS && !srv_read_only_mode) { + fsp_flags_try_adjust(id, flags & ~FSP_FLAGS_MEM_MASK); + } + return(err); } #endif /* !UNIV_HOTBACKUP */ @@ -4693,7 +4636,23 @@ fil_user_tablespace_restore_page( goto out; } - flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); + flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); + + if (!fsp_flags_is_valid(flags)) { + ulint cflags = fsp_flags_convert_from_101(flags); + if (cflags == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_WARN, + "Ignoring a doublewrite copy of page " + ULINTPF ":" ULINTPF + " due to invalid flags 0x%x", + fsp->id, page_no, int(flags)); + err = false; + goto out; + } + flags = cflags; + /* The flags on the page should be converted later. */ + } + zip_size = fsp_flags_get_zip_size(flags); page_size = fsp_flags_get_page_size(flags); @@ -5071,6 +5030,16 @@ will_not_choose: } mutex_exit(&fil_system->mutex); #endif /* UNIV_HOTBACKUP */ + /* Adjust the memory-based flags that would normally be set by + dict_tf_to_fsp_flags(). In recovery, we have no data dictionary. */ + if (FSP_FLAGS_HAS_PAGE_COMPRESSION(fsp->flags)) { + fsp->flags |= page_zip_level + << FSP_FLAGS_MEM_COMPRESSION_LEVEL; + } + remote.flags |= 1U << FSP_FLAGS_MEM_DATA_DIR; + /* We will leave atomic_writes at ATOMIC_WRITES_DEFAULT. + That will be adjusted in fil_space_for_table_exists_in_mem(). */ + ibool file_space_create_success = fil_space_create( tablename, fsp->id, fsp->flags, FIL_TABLESPACE, fsp->crypt_data, false); @@ -5380,13 +5349,12 @@ fil_report_missing_tablespace( name, space_id); } -/*******************************************************************//** -Returns TRUE if a matching tablespace exists in the InnoDB tablespace memory +/** Check if a matching tablespace exists in the InnoDB tablespace memory cache. Note that if we have not done a crash recovery at the database startup, there may be many tablespaces which are not yet in the memory cache. -@return TRUE if a matching tablespace exists in the memory cache */ +@return whether a matching tablespace exists in the memory cache */ UNIV_INTERN -ibool +bool fil_space_for_table_exists_in_mem( /*==============================*/ ulint id, /*!< in: space id */ @@ -5394,13 +5362,7 @@ fil_space_for_table_exists_in_mem( fil_space_create(). Either the standard 'dbname/tablename' format or table->dir_path_of_temp_table */ - ibool mark_space, /*!< in: in crash recovery, at database - startup we mark all spaces which have - an associated table in the InnoDB - data dictionary, so that - we can print a warning about orphaned - tablespaces */ - ibool print_error_if_does_not_exist, + bool print_error_if_does_not_exist, /*!< in: print detailed error information to the .err log if a matching tablespace is not found from @@ -5408,12 +5370,13 @@ fil_space_for_table_exists_in_mem( bool adjust_space, /*!< in: whether to adjust space id when find table space mismatch */ mem_heap_t* heap, /*!< in: heap memory */ - table_id_t table_id) /*!< in: table id */ + table_id_t table_id, /*!< in: table id */ + ulint table_flags) /*!< in: table flags */ { fil_space_t* fnamespace; fil_space_t* space; - ut_ad(fil_system); + const ulint expected_flags = dict_tf_to_fsp_flags(table_flags); mutex_enter(&fil_system->mutex); @@ -5425,42 +5388,31 @@ fil_space_for_table_exists_in_mem( directory path from the datadir to the file */ fnamespace = fil_space_get_by_name(name); - if (space && space == fnamespace) { - /* Found */ - - if (mark_space) { - space->mark = TRUE; - } - - mutex_exit(&fil_system->mutex); - - return(TRUE); - } - - /* Info from "fnamespace" comes from the ibd file itself, it can - be different from data obtained from System tables since it is - not transactional. If adjust_space is set, and the mismatching - space are between a user table and its temp table, we shall - adjust the ibd file name according to system table info */ - if (adjust_space - && space != NULL - && row_is_mysql_tmp_table_name(space->name) - && !row_is_mysql_tmp_table_name(name)) { + bool valid = space && !((space->flags ^ expected_flags) + & ~FSP_FLAGS_MEM_MASK); + if (!space) { + } else if (!valid || space == fnamespace) { + /* Found with the same file name, or got a flag mismatch. */ + goto func_exit; + } else if (adjust_space + && row_is_mysql_tmp_table_name(space->name) + && !row_is_mysql_tmp_table_name(name)) { + /* Info from fnamespace comes from the ibd file + itself, it can be different from data obtained from + System tables since renaming files is not + transactional. We shall adjust the ibd file name + according to system table info. */ mutex_exit(&fil_system->mutex); DBUG_EXECUTE_IF("ib_crash_before_adjust_fil_space", DBUG_SUICIDE();); - if (fnamespace) { - char* tmp_name; + char* tmp_name = dict_mem_create_temporary_tablename( + heap, name, table_id); - tmp_name = dict_mem_create_temporary_tablename( - heap, name, table_id); - - fil_rename_tablespace(fnamespace->name, fnamespace->id, - tmp_name, NULL); - } + fil_rename_tablespace(fnamespace->name, fnamespace->id, + tmp_name, NULL); DBUG_EXECUTE_IF("ib_crash_after_adjust_one_fil_space", DBUG_SUICIDE();); @@ -5473,16 +5425,12 @@ fil_space_for_table_exists_in_mem( mutex_enter(&fil_system->mutex); fnamespace = fil_space_get_by_name(name); ut_ad(space == fnamespace); - mutex_exit(&fil_system->mutex); - - return(TRUE); + goto func_exit; } if (!print_error_if_does_not_exist) { - - mutex_exit(&fil_system->mutex); - - return(FALSE); + valid = false; + goto func_exit; } if (space == NULL) { @@ -5509,10 +5457,8 @@ error_exit: fputs("InnoDB: Please refer to\n" "InnoDB: " REFMAN "innodb-troubleshooting-datadict.html\n" "InnoDB: for how to resolve the issue.\n", stderr); - - mutex_exit(&fil_system->mutex); - - return(FALSE); + valid = false; + goto func_exit; } if (0 != strcmp(space->name, name)) { @@ -5539,9 +5485,19 @@ error_exit: goto error_exit; } +func_exit: + if (valid) { + /* Adjust the flags that are in FSP_FLAGS_MEM_MASK. + FSP_SPACE_FLAGS will not be written back here. */ + space->flags = expected_flags; + } mutex_exit(&fil_system->mutex); - return(FALSE); + if (valid && !srv_read_only_mode) { + fsp_flags_try_adjust(id, expected_flags & ~FSP_FLAGS_MEM_MASK); + } + + return(valid); } /*******************************************************************//** @@ -7444,9 +7400,8 @@ fil_space_get_crypt_data( byte *page = static_cast<byte*>(ut_align(buf, UNIV_PAGE_SIZE)); fil_read(true, space_id, 0, 0, 0, UNIV_PAGE_SIZE, page, NULL, NULL); - ulint flags = fsp_header_get_flags(page); ulint offset = fsp_header_get_crypt_offset( - fsp_flags_get_zip_size(flags), NULL); + fsp_header_get_zip_size(page), NULL); space->crypt_data = fil_space_read_crypt_data(space_id, page, offset); ut_free(buf); diff --git a/storage/xtradb/fsp/fsp0fsp.cc b/storage/xtradb/fsp/fsp0fsp.cc index 4acfd134c1f..c32fddaabbe 100644 --- a/storage/xtradb/fsp/fsp0fsp.cc +++ b/storage/xtradb/fsp/fsp0fsp.cc @@ -663,6 +663,7 @@ fsp_header_init_fields( ulint space_id, /*!< in: space id */ ulint flags) /*!< in: tablespace flags (FSP_SPACE_FLAGS) */ { + flags &= ~FSP_FLAGS_MEM_MASK; ut_a(fsp_flags_is_valid(flags)); mach_write_to_4(FSP_HEADER_OFFSET + FSP_SPACE_ID + page, @@ -713,7 +714,7 @@ fsp_header_init( mlog_write_ulint(header + FSP_SIZE, size, MLOG_4BYTES, mtr); mlog_write_ulint(header + FSP_FREE_LIMIT, 0, MLOG_4BYTES, mtr); - mlog_write_ulint(header + FSP_SPACE_FLAGS, flags, + mlog_write_ulint(header + FSP_SPACE_FLAGS, flags & ~FSP_FLAGS_MEM_MASK, MLOG_4BYTES, mtr); mlog_write_ulint(header + FSP_FRAG_N_USED, 0, MLOG_4BYTES, mtr); diff --git a/storage/xtradb/include/dict0dict.ic b/storage/xtradb/include/dict0dict.ic index d9d40b809d1..2b63ddea51d 100644 --- a/storage/xtradb/include/dict0dict.ic +++ b/storage/xtradb/include/dict0dict.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -918,10 +918,13 @@ dict_tf_to_fsp_flags( ulint table_flags) /*!< in: dict_table_t::flags */ { ulint fsp_flags; - ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); - ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); + ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL( + table_flags); ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); + ut_ad((DICT_TF_GET_PAGE_COMPRESSION(table_flags) == 0) + == (page_compression_level == 0)); + DBUG_EXECUTE_IF("dict_tf_to_fsp_flags_failure", return(ULINT_UNDEFINED);); @@ -929,30 +932,23 @@ dict_tf_to_fsp_flags( fsp_flags = DICT_TF_HAS_ATOMIC_BLOBS(table_flags) ? 1 : 0; /* ZIP_SSIZE and ATOMIC_BLOBS are at the same position. */ - fsp_flags |= table_flags & DICT_TF_MASK_ZIP_SSIZE; - fsp_flags |= table_flags & DICT_TF_MASK_ATOMIC_BLOBS; - - /* In addition, tablespace flags also contain the page size. */ - fsp_flags |= fsp_flags_set_page_size(fsp_flags, UNIV_PAGE_SIZE); + fsp_flags |= table_flags + & (DICT_TF_MASK_ZIP_SSIZE | DICT_TF_MASK_ATOMIC_BLOBS); - /* The DATA_DIR flag is in a different position in fsp_flag */ - fsp_flags |= DICT_TF_HAS_DATA_DIR(table_flags) - ? FSP_FLAGS_MASK_DATA_DIR : 0; + fsp_flags |= FSP_FLAGS_PAGE_SSIZE(); - /* In addition, tablespace flags also contain if the page - compression is used for this table. */ - fsp_flags |= FSP_FLAGS_SET_PAGE_COMPRESSION(fsp_flags, page_compression); + if (page_compression_level) { + fsp_flags |= FSP_FLAGS_MASK_PAGE_COMPRESSION; + } - /* In addition, tablespace flags also contain page compression level - if page compression is used for this table. */ - fsp_flags |= FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(fsp_flags, page_compression_level); + ut_a(fsp_flags_is_valid(fsp_flags)); - /* In addition, tablespace flags also contain flag if atomic writes - is used for this table */ - fsp_flags |= FSP_FLAGS_SET_ATOMIC_WRITES(fsp_flags, atomic_writes); + if (DICT_TF_HAS_DATA_DIR(table_flags)) { + fsp_flags |= 1U << FSP_FLAGS_MEM_DATA_DIR; + } - ut_a(fsp_flags_is_valid(fsp_flags)); - ut_a(dict_tf_verify_flags(table_flags, fsp_flags)); + fsp_flags |= atomic_writes << FSP_FLAGS_MEM_ATOMIC_WRITES; + fsp_flags |= page_compression_level << FSP_FLAGS_MEM_COMPRESSION_LEVEL; return(fsp_flags); } diff --git a/storage/xtradb/include/dict0pagecompress.h b/storage/xtradb/include/dict0pagecompress.h index 19a2a6c52f3..6503c86ffa2 100644 --- a/storage/xtradb/include/dict0pagecompress.h +++ b/storage/xtradb/include/dict0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013 SkySQL Ab. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -57,17 +57,6 @@ dict_table_page_compression_level( __attribute__((const)); /********************************************************************//** -Verify that dictionary flags match tablespace flags -@return true if flags match, false if not */ -UNIV_INLINE -ibool -dict_tf_verify_flags( -/*=================*/ - ulint table_flags, /*!< in: dict_table_t::flags */ - ulint fsp_flags) /*!< in: fil_space_t::flags */ - __attribute__((const)); - -/********************************************************************//** Extract the atomic writes flag from table flags. @return true if atomic writes are used, false if not used */ UNIV_INLINE diff --git a/storage/xtradb/include/dict0pagecompress.ic b/storage/xtradb/include/dict0pagecompress.ic index 811976434a8..13c2b46c51c 100644 --- a/storage/xtradb/include/dict0pagecompress.ic +++ b/storage/xtradb/include/dict0pagecompress.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013 SkySQL Ab. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -25,92 +25,6 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com ***********************************************************************/ /********************************************************************//** -Verify that dictionary flags match tablespace flags -@return true if flags match, false if not */ -UNIV_INLINE -ibool -dict_tf_verify_flags( -/*=================*/ - ulint table_flags, /*!< in: dict_table_t::flags */ - ulint fsp_flags) /*!< in: fil_space_t::flags */ -{ - ulint table_unused = DICT_TF_GET_UNUSED(table_flags); - ulint compact = DICT_TF_GET_COMPACT(table_flags); - ulint ssize = DICT_TF_GET_ZIP_SSIZE(table_flags); - ulint atomic_blobs = DICT_TF_HAS_ATOMIC_BLOBS(table_flags); - ulint data_dir = DICT_TF_HAS_DATA_DIR(table_flags); - ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); - ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); - ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); - ulint post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(fsp_flags); - ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(fsp_flags); - ulint fsp_atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(fsp_flags); - ulint page_ssize = FSP_FLAGS_GET_PAGE_SSIZE(fsp_flags); - ulint fsp_unused = FSP_FLAGS_GET_UNUSED(fsp_flags); - ulint fsp_page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(fsp_flags); - ulint fsp_page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(fsp_flags); - ulint fsp_atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(fsp_flags); - - DBUG_EXECUTE_IF("dict_tf_verify_flags_failure", - return(ULINT_UNDEFINED);); - - ut_a(!table_unused); - ut_a(!fsp_unused); - ut_a(page_ssize == 0 || page_ssize != 0); /* silence compiler */ - ut_a(compact == 0 || compact == 1); /* silence compiler */ - ut_a(data_dir == 0 || data_dir == 1); /* silence compiler */ - ut_a(post_antelope == 0 || post_antelope == 1); /* silence compiler */ - - if (ssize != zip_ssize) { - fprintf(stderr, - "InnoDB: Error: table flags has zip_ssize %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has zip_ssize %ld\n", - ssize, zip_ssize); - return (FALSE); - } - if (atomic_blobs != fsp_atomic_blobs) { - fprintf(stderr, - "InnoDB: Error: table flags has atomic_blobs %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has atomic_blobs %ld\n", - atomic_blobs, fsp_atomic_blobs); - - return (FALSE); - } - if (page_compression != fsp_page_compression) { - fprintf(stderr, - "InnoDB: Error: table flags has page_compression %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file ahas page_compression %ld\n", - page_compression, fsp_page_compression); - - return (FALSE); - } - if (page_compression_level != fsp_page_compression_level) { - fprintf(stderr, - "InnoDB: Error: table flags has page_compression_level %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has page_compression_level %ld\n", - page_compression_level, fsp_page_compression_level); - - return (FALSE); - } - - if (atomic_writes != fsp_atomic_writes) { - fprintf(stderr, - "InnoDB: Error: table flags has atomic writes %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has atomic_writes %ld\n", - atomic_writes, fsp_atomic_writes); - - return (FALSE); - } - - return(TRUE); -} - -/********************************************************************//** Extract the page compression level from dict_table_t::flags. These flags are in memory, so assert that they are valid. @return page compression level, or 0 if not compressed */ diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 38cc09bced3..41e38794ea9 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -45,7 +45,6 @@ Created 10/25/1995 Heikki Tuuri // Forward declaration struct trx_t; -struct fil_space_t; typedef std::list<const char*> space_name_list_t; @@ -264,10 +263,6 @@ struct fil_space_t { an insert buffer merge request for a page because it actually was for the previous incarnation of the space */ - ibool mark; /*!< this is set to TRUE at database startup if - the space corresponds to a table in the InnoDB - data dictionary; so we can print a warning of - orphaned tablespaces */ ibool stop_ios;/*!< TRUE if we want to rename the .ibd file of tablespace and want to stop temporarily posting of new i/o @@ -296,7 +291,8 @@ struct fil_space_t { /*!< recovered tablespace size in pages; 0 if no size change was read from the redo log, or if the size change was implemented */ - ulint flags; /*!< tablespace flags; see + ulint flags; /*!< FSP_SPACE_FLAGS and FSP_FLAGS_MEM_ flags; + see fsp0fsp.h, fsp_flags_is_valid(), fsp_flags_get_zip_size() */ ulint n_reserved_extents; @@ -449,6 +445,7 @@ fil_node_create( ibool is_raw) /*!< in: TRUE if a raw device or a raw disk partition */ MY_ATTRIBUTE((nonnull, warn_unused_result)); + #ifdef UNIV_LOG_ARCHIVE /****************************************************************//** Drops files from the start of a file space, so that its size is cut by @@ -620,7 +617,7 @@ fil_read_first_page( ibool one_read_already, /*!< in: TRUE if min and max parameters below already contain sensible data */ - ulint* flags, /*!< out: tablespace flags */ + ulint* flags, /*!< out: FSP_SPACE_FLAGS */ ulint* space_id, /*!< out: tablespace ID */ lsn_t* min_flushed_lsn, /*!< out: min of flushed lsn values in data files */ @@ -832,6 +829,14 @@ fil_create_new_single_table_tablespace( ulint key_id) /*!< in: encryption key_id */ __attribute__((nonnull, warn_unused_result)); #ifndef UNIV_HOTBACKUP +/** Try to adjust FSP_SPACE_FLAGS if they differ from the expectations. +(Typically when upgrading from MariaDB 10.1.0..10.1.20.) +@param[in] space_id tablespace ID +@param[in] flags desired tablespace flags */ +UNIV_INTERN +void +fsp_flags_try_adjust(ulint space_id, ulint flags); + /********************************************************************//** Tries to open a single-table tablespace and optionally checks the space id is right in it. If does not succeed, prints an error message to the .err log. This @@ -860,7 +865,7 @@ fil_open_single_table_tablespace( bool validate, /*!< in: Do we validate tablespace? */ bool fix_dict, /*!< in: Can we fix the dictionary? */ ulint id, /*!< in: space id */ - ulint flags, /*!< in: tablespace flags */ + ulint flags, /*!< in: expected FSP_SPACE_FLAGS */ const char* tablename, /*!< in: table name in the databasename/tablename format */ const char* filepath, /*!< in: tablespace filepath */ @@ -901,25 +906,18 @@ fil_tablespace_exists_in_mem( /*=========================*/ ulint id); /*!< in: space id */ #ifndef UNIV_HOTBACKUP -/*******************************************************************//** -Returns TRUE if a matching tablespace exists in the InnoDB tablespace memory +/** Check if a matching tablespace exists in the InnoDB tablespace memory cache. Note that if we have not done a crash recovery at the database startup, there may be many tablespaces which are not yet in the memory cache. -@return TRUE if a matching tablespace exists in the memory cache */ +@return whether a matching tablespace exists in the memory cache */ UNIV_INTERN -ibool +bool fil_space_for_table_exists_in_mem( /*==============================*/ ulint id, /*!< in: space id */ const char* name, /*!< in: table name in the standard 'databasename/tablename' format */ - ibool mark_space, /*!< in: in crash recovery, at database - startup we mark all spaces which have - an associated table in the InnoDB - data dictionary, so that - we can print a warning about orphaned - tablespaces */ - ibool print_error_if_does_not_exist, + bool print_error_if_does_not_exist, /*!< in: print detailed error information to the .err log if a matching tablespace is not found from @@ -927,7 +925,8 @@ fil_space_for_table_exists_in_mem( bool adjust_space, /*!< in: whether to adjust space id when find table space mismatch */ mem_heap_t* heap, /*!< in: heap memory */ - table_id_t table_id); /*!< in: table id */ + table_id_t table_id, /*!< in: table id */ + ulint table_flags); /*!< in: table flags */ #else /* !UNIV_HOTBACKUP */ /********************************************************************//** Extends all tablespaces to the size stored in the space header. During the diff --git a/storage/xtradb/include/fil0pagecompress.h b/storage/xtradb/include/fil0pagecompress.h index 10db59fb218..1fe5cb66bf6 100644 --- a/storage/xtradb/include/fil0pagecompress.h +++ b/storage/xtradb/include/fil0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2016 MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017 MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -44,20 +44,11 @@ Returns the page compression flag of the space, or false if the space is not compressed. The tablespace must be cached in the memory cache. @return true if page compressed, false if not or space not found */ UNIV_INLINE -ibool +bool fil_space_is_page_compressed( /*=========================*/ ulint id); /*!< in: space id */ /*******************************************************************//** -Returns the page compression flag of the space, or false if the space -is not compressed. The tablespace must be cached in the memory cache. -@return true if page compressed, false if not or space not found */ -UNIV_INTERN -ibool -fil_space_get_page_compressed( -/*=========================*/ - fil_space_t* space); /*!< in: space id */ -/*******************************************************************//** Returns the atomic writes flag of the space, or false if the space is not using atomic writes. The tablespace must be cached in the memory cache. @return atomic write table option value */ diff --git a/storage/xtradb/include/fsp0fsp.h b/storage/xtradb/include/fsp0fsp.h index 551a8c9ef97..93f98eb0b0b 100644 --- a/storage/xtradb/include/fsp0fsp.h +++ b/storage/xtradb/include/fsp0fsp.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -51,28 +51,67 @@ to the two Barracuda row formats COMPRESSED and DYNAMIC. */ #define FSP_FLAGS_WIDTH_ATOMIC_BLOBS 1 /** Number of flag bits used to indicate the tablespace page size */ #define FSP_FLAGS_WIDTH_PAGE_SSIZE 4 -/** Width of the DATA_DIR flag. This flag indicates that the tablespace -is found in a remote location, not the default data directory. */ -#define FSP_FLAGS_WIDTH_DATA_DIR 1 -/** Number of flag bits used to indicate the page compression and compression level */ -#define FSP_FLAGS_WIDTH_PAGE_COMPRESSION 1 -#define FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL 4 +/** Number of reserved bits */ +#define FSP_FLAGS_WIDTH_RESERVED 6 +/** Number of flag bits used to indicate the page compression */ +#define FSP_FLAGS_WIDTH_PAGE_COMPRESSION 1 -/** Number of flag bits used to indicate atomic writes for this tablespace */ -#define FSP_FLAGS_WIDTH_ATOMIC_WRITES 2 - -/** Width of all the currently known tablespace flags */ +/** Width of all the currently known persistent tablespace flags */ #define FSP_FLAGS_WIDTH (FSP_FLAGS_WIDTH_POST_ANTELOPE \ + FSP_FLAGS_WIDTH_ZIP_SSIZE \ + FSP_FLAGS_WIDTH_ATOMIC_BLOBS \ + FSP_FLAGS_WIDTH_PAGE_SSIZE \ - + FSP_FLAGS_WIDTH_DATA_DIR \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES ) - -/** A mask of all the known/used bits in tablespace flags */ -#define FSP_FLAGS_MASK (~(~0 << FSP_FLAGS_WIDTH)) + + FSP_FLAGS_WIDTH_RESERVED \ + + FSP_FLAGS_WIDTH_PAGE_COMPRESSION) + +/** A mask of all the known/used bits in FSP_SPACE_FLAGS */ +#define FSP_FLAGS_MASK (~(~0U << FSP_FLAGS_WIDTH)) + +/* FSP_SPACE_FLAGS position and name in MySQL 5.6/MariaDB 10.0 or older +and MariaDB 10.1.20 or older MariaDB 10.1 and in MariaDB 10.1.21 +or newer. +MySQL 5.6 MariaDB 10.1.x MariaDB 10.1.21 +==================================================================== +Below flags in same offset +==================================================================== +0: POST_ANTELOPE 0:POST_ANTELOPE 0: POST_ANTELOPE +1..4: ZIP_SSIZE(0..5) 1..4:ZIP_SSIZE(0..5) 1..4: ZIP_SSIZE(0..5) +(NOTE: bit 4 is always 0) +5: ATOMIC_BLOBS 5:ATOMIC_BLOBS 5: ATOMIC_BLOBS +===================================================================== +Below note the order difference: +===================================================================== +6..9: PAGE_SSIZE(3..7) 6: COMPRESSION 6..9: PAGE_SSIZE(3..7) +10: DATA_DIR 7..10: COMP_LEVEL(0..9) 10: RESERVED (5.6 DATA_DIR) +===================================================================== +The flags below were in incorrect position in MariaDB 10.1, +or have been introduced in MySQL 5.7 or 8.0: +===================================================================== +11: UNUSED 11..12:ATOMIC_WRITES 11: RESERVED (5.7 SHARED) + 12: RESERVED (5.7 TEMPORARY) + 13..15:PAGE_SSIZE(3..7) 13: RESERVED (5.7 ENCRYPTION) + 14: RESERVED (8.0 SDI) + 15: RESERVED + 16: PAGE_SSIZE_msb(0) 16: COMPRESSION + 17: DATA_DIR 17: UNUSED + 18: UNUSED +===================================================================== +The flags below only exist in fil_space_t::flags, not in FSP_SPACE_FLAGS: +===================================================================== + 25: DATA_DIR + 26..27: ATOMIC_WRITES + 28..31: COMPRESSION_LEVEL +*/ + +/** A mask of the memory-only flags in fil_space_t::flags */ +#define FSP_FLAGS_MEM_MASK (~0U << FSP_FLAGS_MEM_DATA_DIR) + +/** Zero relative shift position of the DATA_DIR flag */ +#define FSP_FLAGS_MEM_DATA_DIR 25 +/** Zero relative shift position of the ATOMIC_WRITES field */ +#define FSP_FLAGS_MEM_ATOMIC_WRITES 26 +/** Zero relative shift position of the COMPRESSION_LEVEL field */ +#define FSP_FLAGS_MEM_COMPRESSION_LEVEL 28 /** Zero relative shift position of the POST_ANTELOPE field */ #define FSP_FLAGS_POS_POST_ANTELOPE 0 @@ -82,29 +121,16 @@ is found in a remote location, not the default data directory. */ /** Zero relative shift position of the ATOMIC_BLOBS field */ #define FSP_FLAGS_POS_ATOMIC_BLOBS (FSP_FLAGS_POS_ZIP_SSIZE \ + FSP_FLAGS_WIDTH_ZIP_SSIZE) -/** Note that these need to be before the page size to be compatible with -dictionary */ -/** Zero relative shift position of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_POS_PAGE_COMPRESSION (FSP_FLAGS_POS_ATOMIC_BLOBS \ - + FSP_FLAGS_WIDTH_ATOMIC_BLOBS) -/** Zero relative shift position of the PAGE_COMPRESSION_LEVEL field */ -#define FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL (FSP_FLAGS_POS_PAGE_COMPRESSION \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION) -/** Zero relative shift position of the ATOMIC_WRITES field */ -#define FSP_FLAGS_POS_ATOMIC_WRITES (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL \ - + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL) -/** Zero relative shift position of the PAGE_SSIZE field */ -#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_ATOMIC_WRITES \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES) -/** Zero relative shift position of the start of the DATA DIR bits */ -#define FSP_FLAGS_POS_DATA_DIR (FSP_FLAGS_POS_PAGE_SSIZE \ - + FSP_FLAGS_WIDTH_PAGE_SSIZE) -#define FSP_FLAGS_POS_DATA_DIR_ORACLE (FSP_FLAGS_POS_ATOMIC_BLOBS \ - + FSP_FLAGS_WIDTH_ATOMIC_BLOBS \ +/** Zero relative shift position of the start of the PAGE_SSIZE bits */ +#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_ATOMIC_BLOBS \ + + FSP_FLAGS_WIDTH_ATOMIC_BLOBS) +/** Zero relative shift position of the start of the RESERVED bits +these are only used in MySQL 5.7 and used for compatibility. */ +#define FSP_FLAGS_POS_RESERVED (FSP_FLAGS_POS_PAGE_SSIZE \ + FSP_FLAGS_WIDTH_PAGE_SSIZE) -/** Zero relative shift position of the start of the UNUSED bits */ -#define FSP_FLAGS_POS_UNUSED (FSP_FLAGS_POS_DATA_DIR\ - + FSP_FLAGS_WIDTH_DATA_DIR) +/** Zero relative shift position of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_POS_PAGE_COMPRESSION (FSP_FLAGS_POS_RESERVED \ + + FSP_FLAGS_WIDTH_RESERVED) /** Bit mask of the POST_ANTELOPE field */ #define FSP_FLAGS_MASK_POST_ANTELOPE \ @@ -122,26 +148,23 @@ dictionary */ #define FSP_FLAGS_MASK_PAGE_SSIZE \ ((~(~0U << FSP_FLAGS_WIDTH_PAGE_SSIZE)) \ << FSP_FLAGS_POS_PAGE_SSIZE) -/** Bit mask of the DATA_DIR field */ -#define FSP_FLAGS_MASK_DATA_DIR \ - ((~(~0U << FSP_FLAGS_WIDTH_DATA_DIR)) \ - << FSP_FLAGS_POS_DATA_DIR) -/** Bit mask of the DATA_DIR field */ -#define FSP_FLAGS_MASK_DATA_DIR_ORACLE \ - ((~(~0U << FSP_FLAGS_WIDTH_DATA_DIR)) \ - << FSP_FLAGS_POS_DATA_DIR_ORACLE) +/** Bit mask of the RESERVED1 field */ +#define FSP_FLAGS_MASK_RESERVED \ + ((~(~0U << FSP_FLAGS_WIDTH_RESERVED)) \ + << FSP_FLAGS_POS_RESERVED) /** Bit mask of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_MASK_PAGE_COMPRESSION \ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION \ ((~(~0U << FSP_FLAGS_WIDTH_PAGE_COMPRESSION)) \ << FSP_FLAGS_POS_PAGE_COMPRESSION) -/** Bit mask of the PAGE_COMPRESSION_LEVEL field */ -#define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL \ - ((~(~0U << FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL)) \ - << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL) -/** Bit mask of the ATOMIC_WRITES field */ -#define FSP_FLAGS_MASK_ATOMIC_WRITES \ - ((~(~0U << FSP_FLAGS_WIDTH_ATOMIC_WRITES)) \ - << FSP_FLAGS_POS_ATOMIC_WRITES) + +/** Bit mask of the in-memory ATOMIC_WRITES field */ +#define FSP_FLAGS_MASK_MEM_ATOMIC_WRITES \ + (3U << FSP_FLAGS_MEM_ATOMIC_WRITES) + +/** Bit mask of the in-memory COMPRESSION_LEVEL field */ +#define FSP_FLAGS_MASK_MEM_COMPRESSION_LEVEL \ + (15U << FSP_FLAGS_MEM_COMPRESSION_LEVEL) + /** Return the value of the POST_ANTELOPE field */ #define FSP_FLAGS_GET_POST_ANTELOPE(flags) \ ((flags & FSP_FLAGS_MASK_POST_ANTELOPE) \ @@ -158,48 +181,78 @@ dictionary */ #define FSP_FLAGS_GET_PAGE_SSIZE(flags) \ ((flags & FSP_FLAGS_MASK_PAGE_SSIZE) \ >> FSP_FLAGS_POS_PAGE_SSIZE) -/** Return the value of the DATA_DIR field */ -#define FSP_FLAGS_HAS_DATA_DIR(flags) \ - ((flags & FSP_FLAGS_MASK_DATA_DIR) \ - >> FSP_FLAGS_POS_DATA_DIR) -#define FSP_FLAGS_HAS_DATA_DIR_ORACLE(flags) \ - ((flags & FSP_FLAGS_MASK_DATA_DIR_ORACLE) \ - >> FSP_FLAGS_POS_DATA_DIR_ORACLE) +/** @return the RESERVED flags */ +#define FSP_FLAGS_GET_RESERVED(flags) \ + ((flags & FSP_FLAGS_MASK_RESERVED) \ + >> FSP_FLAGS_POS_RESERVED) +/** @return the PAGE_COMPRESSION flag */ +#define FSP_FLAGS_HAS_PAGE_COMPRESSION(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION) \ + >> FSP_FLAGS_POS_PAGE_COMPRESSION) + /** Return the contents of the UNUSED bits */ #define FSP_FLAGS_GET_UNUSED(flags) \ (flags >> FSP_FLAGS_POS_UNUSED) + +/** @return the PAGE_SSIZE flags for the current innodb_page_size */ +#define FSP_FLAGS_PAGE_SSIZE() \ + ((UNIV_PAGE_SIZE == UNIV_PAGE_SIZE_ORIG) ? \ + 0 : (UNIV_PAGE_SIZE_SHIFT - UNIV_ZIP_SIZE_SHIFT_MIN + 1) \ + << FSP_FLAGS_POS_PAGE_SSIZE) + +/** @return the value of the DATA_DIR field */ +#define FSP_FLAGS_HAS_DATA_DIR(flags) \ + (flags & 1U << FSP_FLAGS_MEM_DATA_DIR) +/** @return the COMPRESSION_LEVEL field */ +#define FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags) \ + ((flags & FSP_FLAGS_MASK_MEM_COMPRESSION_LEVEL) \ + >> FSP_FLAGS_MEM_COMPRESSION_LEVEL) +/** @return the ATOMIC_WRITES field */ +#define FSP_FLAGS_GET_ATOMIC_WRITES(flags) \ + ((flags & FSP_FLAGS_MASK_MEM_ATOMIC_WRITES) \ + >> FSP_FLAGS_MEM_ATOMIC_WRITES) + +/* Compatibility macros for MariaDB 10.1.20 or older 10.1 see +table above. */ +/** Zero relative shift position of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101 \ + (FSP_FLAGS_POS_ATOMIC_BLOBS \ + + FSP_FLAGS_WIDTH_ATOMIC_BLOBS) +/** Zero relative shift position of the PAGE_COMPRESSION_LEVEL field */ +#define FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101 \ + (FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101 + 1) +/** Zero relative shift position of the ATOMIC_WRITES field */ +#define FSP_FLAGS_POS_ATOMIC_WRITES_MARIADB101 \ + (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101 + 4) +/** Zero relative shift position of the PAGE_SSIZE field */ +#define FSP_FLAGS_POS_PAGE_SSIZE_MARIADB101 \ + (FSP_FLAGS_POS_ATOMIC_WRITES_MARIADB101 + 2) + +/** Bit mask of the PAGE_COMPRESSION field */ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION_MARIADB101 \ + (1U << FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101) +/** Bit mask of the PAGE_COMPRESSION_LEVEL field */ +#define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL_MARIADB101 \ + (15U << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101) +/** Bit mask of the ATOMIC_WRITES field */ +#define FSP_FLAGS_MASK_ATOMIC_WRITES_MARIADB101 \ + (3U << FSP_FLAGS_POS_ATOMIC_WRITES_MARIADB101) +/** Bit mask of the PAGE_SSIZE field */ +#define FSP_FLAGS_MASK_PAGE_SSIZE_MARIADB101 \ + (15U << FSP_FLAGS_POS_PAGE_SSIZE_MARIADB101) + /** Return the value of the PAGE_COMPRESSION field */ -#define FSP_FLAGS_GET_PAGE_COMPRESSION(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION) \ - >> FSP_FLAGS_POS_PAGE_COMPRESSION) +#define FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION_MARIADB101) \ + >> FSP_FLAGS_POS_PAGE_COMPRESSION_MARIADB101) /** Return the value of the PAGE_COMPRESSION_LEVEL field */ -#define FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags) \ - ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL) \ - >> FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL) -/** Return the value of the ATOMIC_WRITES field */ -#define FSP_FLAGS_GET_ATOMIC_WRITES(flags) \ - ((flags & FSP_FLAGS_MASK_ATOMIC_WRITES) \ - >> FSP_FLAGS_POS_ATOMIC_WRITES) - -/** Set a PAGE_SSIZE into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_SSIZE(flags, ssize) \ - (flags | (ssize << FSP_FLAGS_POS_PAGE_SSIZE)) - -/** Set a PAGE_COMPRESSION into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_COMPRESSION(flags, compression) \ - (flags | (compression << FSP_FLAGS_POS_PAGE_COMPRESSION)) - -/** Set a PAGE_COMPRESSION_LEVEL into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(flags, level) \ - (flags | (level << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL)) - -/** Set a ATOMIC_WRITES into the correct bits in a given -tablespace flags. */ -#define FSP_FLAGS_SET_ATOMIC_WRITES(flags, atomics) \ - (flags | (atomics << FSP_FLAGS_POS_ATOMIC_WRITES)) +#define FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL_MARIADB101(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL_MARIADB101) \ + >> FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL_MARIADB101) +/** Return the value of the PAGE_SSIZE field */ +#define FSP_FLAGS_GET_PAGE_SSIZE_MARIADB101(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_SSIZE_MARIADB101) \ + >> FSP_FLAGS_POS_PAGE_SSIZE_MARIADB101) /* @} */ @@ -733,19 +786,193 @@ fseg_print( mtr_t* mtr); /*!< in/out: mini-transaction */ #endif /* UNIV_BTR_PRINT */ -/********************************************************************//** -Validate and return the tablespace flags, which are stored in the -tablespace header at offset FSP_SPACE_FLAGS. They should be 0 for -ROW_FORMAT=COMPACT and ROW_FORMAT=REDUNDANT. The newer row formats, -COMPRESSED and DYNAMIC, use a file format > Antelope so they should -have a file format number plus the DICT_TF_COMPACT bit set. -@return true if check ok */ +/** Validate the tablespace flags, which are stored in the +tablespace header at offset FSP_SPACE_FLAGS. +@param[in] flags the contents of FSP_SPACE_FLAGS +@return whether the flags are correct (not in the buggy 10.1) format */ +MY_ATTRIBUTE((warn_unused_result, const)) UNIV_INLINE bool -fsp_flags_is_valid( -/*===============*/ - ulint flags) /*!< in: tablespace flags */ - MY_ATTRIBUTE((warn_unused_result, const)); +fsp_flags_is_valid(ulint flags) +{ + DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", + return(false);); + if (flags == 0) { + return(true); + } + if (flags & ~FSP_FLAGS_MASK) { + return(false); + } + if ((flags & (FSP_FLAGS_MASK_POST_ANTELOPE | FSP_FLAGS_MASK_ATOMIC_BLOBS)) + == FSP_FLAGS_MASK_ATOMIC_BLOBS) { + /* If the "atomic blobs" flag (indicating + ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED) flag + is set, then the "post Antelope" (ROW_FORMAT!=REDUNDANT) flag + must also be set. */ + return(false); + } + /* Bits 10..14 should be 0b0000d where d is the DATA_DIR flag + of MySQL 5.6 and MariaDB 10.0, which we ignore. + In the buggy FSP_SPACE_FLAGS written by MariaDB 10.1.0 to 10.1.20, + bits 10..14 would be nonzero 0bsssaa where sss is + nonzero PAGE_SSIZE (3, 4, 6, or 7) + and aa is ATOMIC_WRITES (not 0b11). */ + if (FSP_FLAGS_GET_RESERVED(flags) & ~1) { + return(false); + } + + const ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); + if (ssize == 1 || ssize == 2 || ssize == 5 || ssize & 8) { + /* the page_size is not between 4k and 64k; + 16k should be encoded as 0, not 5 */ + return(false); + } + const ulint zssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); + if (zssize == 0) { + /* not ROW_FORMAT=COMPRESSED */ + } else if (zssize > (ssize ? ssize : 5)) { + /* invalid KEY_BLOCK_SIZE */ + return(false); + } else if (~flags & (FSP_FLAGS_MASK_POST_ANTELOPE + | FSP_FLAGS_MASK_ATOMIC_BLOBS)) { + /* both these flags should be set for + ROW_FORMAT=COMPRESSED */ + return(false); + } + + return(true); +} + +/** Convert FSP_SPACE_FLAGS from the buggy MariaDB 10.1.0..10.1.20 format. +@param[in] flags the contents of FSP_SPACE_FLAGS +@return the flags corrected from the buggy MariaDB 10.1 format +@retval ULINT_UNDEFINED if the flags are not in the buggy 10.1 format */ +MY_ATTRIBUTE((warn_unused_result, const)) +UNIV_INLINE +ulint +fsp_flags_convert_from_101(ulint flags) +{ + DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", + return(ULINT_UNDEFINED);); + if (flags == 0) { + return(flags); + } + + if (flags >> 18) { + /* The most significant FSP_SPACE_FLAGS bit that was ever set + by MariaDB 10.1.0 to 10.1.20 was bit 17 (misplaced DATA_DIR flag). + The flags must be less than 1<<18 in order to be valid. */ + return(ULINT_UNDEFINED); + } + + if ((flags & (FSP_FLAGS_MASK_POST_ANTELOPE | FSP_FLAGS_MASK_ATOMIC_BLOBS)) + == FSP_FLAGS_MASK_ATOMIC_BLOBS) { + /* If the "atomic blobs" flag (indicating + ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED) flag + is set, then the "post Antelope" (ROW_FORMAT!=REDUNDANT) flag + must also be set. */ + return(ULINT_UNDEFINED); + } + + /* Bits 6..10 denote compression in MariaDB 10.1.0 to 10.1.20. + They must be either 0b00000 or 0b00011 through 0b10011. + In correct versions, these bits would be + 0bd0sss where d is the DATA_DIR flag (garbage bit) and + sss is the PAGE_SSIZE (3, 4, 6, or 7). + + NOTE: MariaDB 10.1.0 to 10.1.20 can misinterpret + uncompressed data files with innodb_page_size=4k or 64k as + compressed innodb_page_size=16k files. Below is an exhaustive + state space analysis. + + -0by1zzz: impossible (the bit 4 must be clean; see above) + -0b101xx: DATA_DIR, innodb_page_size>4k: invalid (COMPRESSION_LEVEL>9) + +0bx0011: innodb_page_size=4k: + !!! Misinterpreted as COMPRESSION_LEVEL=9 or 1, COMPRESSION=1. + -0bx0010: impossible, because sss must be 0b011 or 0b1xx + -0bx0001: impossible, because sss must be 0b011 or 0b1xx + -0b10000: DATA_DIR, innodb_page_size=16: + invalid (COMPRESSION_LEVEL=8 but COMPRESSION=0) + +0b00111: no DATA_DIR, innodb_page_size=64k: + !!! Misinterpreted as COMPRESSION_LEVEL=3, COMPRESSION=1. + -0b00101: impossible, because sss must be 0 for 16k, not 0b101 + -0b001x0: no DATA_DIR, innodb_page_size=32k or 8k: + invalid (COMPRESSION_LEVEL=3 but COMPRESSION=0) + +0b00000: innodb_page_size=16k (looks like COMPRESSION=0) + ??? Could actually be compressed; see PAGE_SSIZE below */ + const ulint level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL_MARIADB101( + flags); + if (FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) != (level != 0) + || level > 9) { + /* The compression flags are not in the buggy MariaDB + 10.1 format. */ + return(ULINT_UNDEFINED); + } + if (!(~flags & FSP_FLAGS_MASK_ATOMIC_WRITES_MARIADB101)) { + /* The ATOMIC_WRITES flags cannot be 0b11. + (The bits 11..12 should actually never be 0b11, + because in MySQL they would be SHARED|TEMPORARY.) */ + return(ULINT_UNDEFINED); + } + + /* Bits 13..16 are the wrong position for PAGE_SSIZE, and they + should contain one of the values 3,4,6,7, that is, be of the form + 0b0011 or 0b01xx (except 0b0110). + In correct versions, these bits should be 0bc0se + where c is the MariaDB COMPRESSED flag + and e is the MySQL 5.7 ENCRYPTION flag + and s is the MySQL 8.0 SDI flag. MariaDB can only support s=0, e=0. + + Compressed innodb_page_size=16k tables with correct FSP_SPACE_FLAGS + will be properly rejected by older MariaDB 10.1.x because they + would read as PAGE_SSIZE>=8 which is not valid. */ + + const ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE_MARIADB101(flags); + if (ssize == 1 || ssize == 2 || ssize == 5 || ssize & 8) { + /* the page_size is not between 4k and 64k; + 16k should be encoded as 0, not 5 */ + return(ULINT_UNDEFINED); + } + const ulint zssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); + if (zssize == 0) { + /* not ROW_FORMAT=COMPRESSED */ + } else if (zssize > (ssize ? ssize : 5)) { + /* invalid KEY_BLOCK_SIZE */ + return(ULINT_UNDEFINED); + } else if (~flags & (FSP_FLAGS_MASK_POST_ANTELOPE + | FSP_FLAGS_MASK_ATOMIC_BLOBS)) { + /* both these flags should be set for + ROW_FORMAT=COMPRESSED */ + return(ULINT_UNDEFINED); + } + + flags = ((flags & 0x3f) | ssize << FSP_FLAGS_POS_PAGE_SSIZE + | FSP_FLAGS_GET_PAGE_COMPRESSION_MARIADB101(flags) + << FSP_FLAGS_POS_PAGE_COMPRESSION); + ut_ad(fsp_flags_is_valid(flags)); + return(flags); +} + +/** Compare tablespace flags. +@param[in] expected expected flags from dict_tf_to_fsp_flags() +@param[in] actual flags read from FSP_SPACE_FLAGS +@return whether the flags match */ +MY_ATTRIBUTE((warn_unused_result)) +UNIV_INLINE +bool +fsp_flags_match(ulint expected, ulint actual) +{ + expected &= ~FSP_FLAGS_MEM_MASK; + ut_ad(fsp_flags_is_valid(expected)); + + if (actual == expected) { + return(true); + } + + actual = fsp_flags_convert_from_101(actual); + return(actual == expected); +} + /********************************************************************//** Determine if the tablespace is compressed from dict_table_t::flags. @return TRUE if compressed, FALSE if not compressed */ diff --git a/storage/xtradb/include/fsp0fsp.ic b/storage/xtradb/include/fsp0fsp.ic index ddcb87b0e57..ee4cb1f32c7 100644 --- a/storage/xtradb/include/fsp0fsp.ic +++ b/storage/xtradb/include/fsp0fsp.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2012, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, SkySQL Ab. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -47,107 +47,6 @@ fsp_descr_page( } /********************************************************************//** -Validate and return the tablespace flags, which are stored in the -tablespace header at offset FSP_SPACE_FLAGS. They should be 0 for -ROW_FORMAT=COMPACT and ROW_FORMAT=REDUNDANT. The newer row formats, -COMPRESSED and DYNAMIC, use a file format > Antelope so they should -have a file format number plus the DICT_TF_COMPACT bit set. -@return true if check ok */ -UNIV_INLINE -bool -fsp_flags_is_valid( -/*===============*/ - ulint flags) /*!< in: tablespace flags */ -{ - ulint post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(flags); - ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(flags); - ulint atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(flags); - ulint page_ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); - ulint unused = FSP_FLAGS_GET_UNUSED(flags); - ulint page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(flags); - ulint page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags); - ulint atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(flags); - - DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", return(false);); - - /* fsp_flags is zero unless atomic_blobs is set. */ - /* Make sure there are no bits that we do not know about. */ - if (unused != 0 || flags == 1) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted unused %lu\n", - flags, unused); - return(false); - } else if (post_antelope) { - /* The Antelope row formats REDUNDANT and COMPACT did - not use tablespace flags, so this flag and the entire - 4-byte field is zero for Antelope row formats. */ - - if (!atomic_blobs) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted atomic_blobs %lu\n", - flags, atomic_blobs); - return(false); - } - } - - if (!atomic_blobs) { - /* Barracuda row formats COMPRESSED and DYNAMIC build on - the page structure introduced for the COMPACT row format - by allowing long fields to be broken into prefix and - externally stored parts. */ - - if (post_antelope || zip_ssize != 0) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted zip_ssize %lu atomic_blobs %lu\n", - flags, zip_ssize, atomic_blobs); - return(false); - } - - } else if (!post_antelope || zip_ssize > PAGE_ZIP_SSIZE_MAX) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted zip_ssize %lu max %d\n", - flags, zip_ssize, PAGE_ZIP_SSIZE_MAX); - return(false); - } else if (page_ssize > UNIV_PAGE_SSIZE_MAX) { - - /* The page size field can be used for any row type, or it may - be zero for an original 16k page size. - Validate the page shift size is within allowed range. */ - - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted page_ssize %lu max %lu\n", - flags, page_ssize, UNIV_PAGE_SSIZE_MAX); - return(false); - - } else if (UNIV_PAGE_SIZE != UNIV_PAGE_SIZE_ORIG && !page_ssize) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted page_ssize %lu max %lu:%d\n", - flags, page_ssize, UNIV_PAGE_SIZE, UNIV_PAGE_SIZE_ORIG); - return(false); - } - -#if UNIV_FORMAT_MAX != UNIV_FORMAT_B -# error "UNIV_FORMAT_MAX != UNIV_FORMAT_B, Add more validations." -#endif - - /* Page compression level requires page compression and atomic blobs - to be set */ - if (page_compression_level || page_compression) { - if (!page_compression || !atomic_blobs) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted page_compression %lu\n" - "InnoDB: Error: page_compression_level %lu atomic_blobs %lu\n", - flags, page_compression, page_compression_level, atomic_blobs); - return(false); - } - } - - if (atomic_writes > ATOMIC_WRITES_OFF) { - fprintf(stderr, "InnoDB: Error: Tablespace flags %lu corrupted atomic_writes %lu\n", - flags, atomic_writes); - return (false); - } - - /* The DATA_DIR field can be used for any row type so there is - nothing here to validate. */ - - return(true); -} - -/********************************************************************//** Determine if the tablespace is compressed from dict_table_t::flags. @return TRUE if compressed, FALSE if not compressed */ UNIV_INLINE @@ -191,10 +90,10 @@ UNIV_INLINE ulint fsp_flags_get_page_size( /*====================*/ - ulint flags) /*!< in: tablespace flags */ + ulint flags) /*!< in: tablespace flags */ { - ulint page_size = 0; - ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); + ulint page_size = 0; + ulint ssize = FSP_FLAGS_GET_PAGE_SSIZE(flags); /* Convert from a 'log2 minus 9' to a page size in bytes. */ if (UNIV_UNLIKELY(ssize)) { @@ -211,50 +110,6 @@ fsp_flags_get_page_size( } #ifndef UNIV_INNOCHECKSUM - -/********************************************************************//** -Add the page size to the tablespace flags. -@return tablespace flags after page size is added */ -UNIV_INLINE -ulint -fsp_flags_set_page_size( -/*====================*/ - ulint flags, /*!< in: tablespace flags */ - ulint page_size) /*!< in: page size in bytes */ -{ - ulint ssize = 0; - ulint shift; - - /* Page size should be > UNIV_PAGE_SIZE_MIN */ - ut_ad(page_size >= UNIV_PAGE_SIZE_MIN); - ut_ad(page_size <= UNIV_PAGE_SIZE_MAX); - - if (page_size == UNIV_PAGE_SIZE_ORIG) { - ut_ad(0 == FSP_FLAGS_GET_PAGE_SSIZE(flags)); - return(flags); - } - - for (shift = UNIV_PAGE_SIZE_SHIFT_MAX; - shift >= UNIV_PAGE_SIZE_SHIFT_MIN; - shift--) { - ulint mask = (1 << shift); - if (page_size & mask) { - ut_ad(!(page_size & ~mask)); - ssize = shift - UNIV_ZIP_SIZE_SHIFT_MIN + 1; - break; - } - } - - ut_ad(ssize); - ut_ad(ssize <= UNIV_PAGE_SSIZE_MAX); - - flags = FSP_FLAGS_SET_PAGE_SSIZE(flags, ssize); - - ut_ad(fsp_flags_is_valid(flags)); - - return(flags); -} - /********************************************************************//** Calculates the descriptor index within a descriptor page. @return descriptor index */ diff --git a/storage/xtradb/include/fsp0pagecompress.h b/storage/xtradb/include/fsp0pagecompress.h index 5f943ee2b83..c623d11c326 100644 --- a/storage/xtradb/include/fsp0pagecompress.h +++ b/storage/xtradb/include/fsp0pagecompress.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -48,15 +48,6 @@ fsp_header_get_compression_level( const page_t* page); /*!< in: first page of a tablespace */ /********************************************************************//** -Determine if the tablespace is page compressed from dict_table_t::flags. -@return TRUE if page compressed, FALSE if not compressed */ -UNIV_INLINE -ibool -fsp_flags_is_page_compressed( -/*=========================*/ - ulint flags); /*!< in: tablespace flags */ - -/********************************************************************//** Extract the page compression level from tablespace flags. A tablespace has only one physical page compression level whether that page is compressed or not. diff --git a/storage/xtradb/include/fsp0pagecompress.ic b/storage/xtradb/include/fsp0pagecompress.ic index e879aa2c16e..48163277feb 100644 --- a/storage/xtradb/include/fsp0pagecompress.ic +++ b/storage/xtradb/include/fsp0pagecompress.ic @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -26,18 +26,6 @@ Created 11/12/2013 Jan Lindström jan.lindstrom@mariadb.com ***********************************************************************/ /********************************************************************//** -Determine if the tablespace is page compressed from dict_table_t::flags. -@return TRUE if page compressed, FALSE if not page compressed */ -UNIV_INLINE -ibool -fsp_flags_is_page_compressed( -/*=========================*/ - ulint flags) /*!< in: tablespace flags */ -{ - return(FSP_FLAGS_GET_PAGE_COMPRESSION(flags)); -} - -/********************************************************************//** Determine the tablespace is page compression level from dict_table_t::flags. @return page compression level or 0 if not compressed*/ UNIV_INLINE @@ -125,21 +113,15 @@ Extract the page compression from space. @return true if space is page compressed, false if space is not found or space is not page compressed. */ UNIV_INLINE -ibool +bool fil_space_is_page_compressed( /*=========================*/ ulint id) /*!< in: space id */ { - ulint flags; - - flags = fil_space_get_flags(id); - - if (flags && flags != ULINT_UNDEFINED) { - - return(fsp_flags_is_page_compressed(flags)); - } + ulint flags = fil_space_get_flags(id); - return(0); + return(flags != ULINT_UNDEFINED + && FSP_FLAGS_HAS_PAGE_COMPRESSION(flags)); } #endif /* UNIV_INNOCHECKSUM */ diff --git a/storage/xtradb/row/row0import.cc b/storage/xtradb/row/row0import.cc index d45ce907304..6170eb66195 100644 --- a/storage/xtradb/row/row0import.cc +++ b/storage/xtradb/row/row0import.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -371,8 +371,7 @@ public: m_space(ULINT_UNDEFINED), m_xdes(), m_xdes_page_no(ULINT_UNDEFINED), - m_space_flags(ULINT_UNDEFINED), - m_table_flags(ULINT_UNDEFINED) UNIV_NOTHROW { } + m_space_flags(ULINT_UNDEFINED) UNIV_NOTHROW { } /** Free any extent descriptor instance */ @@ -535,10 +534,6 @@ protected: /** Flags value read from the header page */ ulint m_space_flags; - - /** Derived from m_space_flags and row format type, the row format - type is determined from the page header. */ - ulint m_table_flags; }; /** Determine the page size to use for traversing the tablespace @@ -553,6 +548,19 @@ AbstractCallback::init( const page_t* page = block->frame; m_space_flags = fsp_header_get_flags(page); + if (!fsp_flags_is_valid(m_space_flags)) { + ulint cflags = fsp_flags_convert_from_101(m_space_flags); + if (cflags == ULINT_UNDEFINED) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Invalid FSP_SPACE_FLAGS=0x%x", + int(m_space_flags)); + return(DB_CORRUPTION); + } + m_space_flags = cflags; + } + + /* Clear the DATA_DIR flag, which is basically garbage. */ + m_space_flags &= ~(1U << FSP_FLAGS_POS_RESERVED); /* Since we don't know whether it is a compressed table or not, the data is always read into the block->frame. */ @@ -641,46 +649,6 @@ struct FetchIndexRootPages : public AbstractCallback { } /** - Check if the .ibd file row format is the same as the table's. - @param ibd_table_flags - determined from space and page. - @return DB_SUCCESS or error code. */ - dberr_t check_row_format(ulint ibd_table_flags) UNIV_NOTHROW - { - dberr_t err; - rec_format_t ibd_rec_format; - rec_format_t table_rec_format; - - if (!dict_tf_is_valid(ibd_table_flags)) { - - ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - ".ibd file has invlad table flags: %lx", - ibd_table_flags); - - return(DB_CORRUPTION); - } - - ibd_rec_format = dict_tf_get_rec_format(ibd_table_flags); - table_rec_format = dict_tf_get_rec_format(m_table->flags); - - if (table_rec_format != ibd_rec_format) { - - ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Table has %s row format, .ibd " - "file has %s row format.", - dict_tf_to_row_format_string(m_table->flags), - dict_tf_to_row_format_string(ibd_table_flags)); - - err = DB_CORRUPTION; - } else { - err = DB_SUCCESS; - } - - return(err); - } - - /** Called for each block as it is read from the file. @param offset - physical offset in the file @param block - block to convert, it is not from the buffer pool. @@ -743,12 +711,17 @@ FetchIndexRootPages::operator() ( m_indexes.push_back(Index(id, page_no)); if (m_indexes.size() == 1) { - - m_table_flags = dict_sys_tables_type_to_tf( - m_space_flags, - page_is_comp(page) ? DICT_N_COLS_COMPACT : 0); - - err = check_row_format(m_table_flags); + /* Check that the tablespace flags match the table flags. */ + ulint expected = dict_tf_to_fsp_flags(m_table->flags); + if (!fsp_flags_match(expected, m_space_flags)) { + ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, + ER_TABLE_SCHEMA_MISMATCH, + "Expected FSP_SPACE_FLAGS=0x%x, .ibd " + "file contains 0x%x.", + unsigned(expected), + unsigned(m_space_flags)); + return(DB_CORRUPTION); + } } } @@ -1968,21 +1941,14 @@ PageConverter::update_header( "- ignored"); } - ulint space_flags = fsp_header_get_flags(get_frame(block)); - - if (!fsp_flags_is_valid(space_flags)) { - - ib_logf(IB_LOG_LEVEL_ERROR, - "Unsupported tablespace format %lu", - (ulong) space_flags); - - return(DB_UNSUPPORTED); - } - mach_write_to_8( get_frame(block) + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, m_current_lsn); + /* Write back the adjusted flags. */ + mach_write_to_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + + get_frame(block), m_space_flags); + /* Write space_id to the tablespace header, page 0. */ mach_write_to_4( get_frame(block) + FSP_HEADER_OFFSET + FSP_SPACE_ID, diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index aff7b758249..c81b10b93f1 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -4362,6 +4362,7 @@ row_drop_table_for_mysql( switch (err) { ibool is_temp; + ulint table_flags; case DB_SUCCESS: /* Clone the name, in case it has been allocated @@ -4370,6 +4371,7 @@ row_drop_table_for_mysql( space_id = table->space; ibd_file_missing = table->ibd_file_missing; + table_flags = table->flags; is_temp = DICT_TF2_FLAG_IS_SET(table, DICT_TF2_TEMPORARY); /* If there is a temp path then the temp flag is set. @@ -4385,9 +4387,9 @@ row_drop_table_for_mysql( } /* We do not allow temporary tables with a remote path. */ - ut_a(!(is_temp && DICT_TF_HAS_DATA_DIR(table->flags))); + ut_a(!(is_temp && DICT_TF_HAS_DATA_DIR(table_flags))); - if (space_id && DICT_TF_HAS_DATA_DIR(table->flags)) { + if (space_id && DICT_TF_HAS_DATA_DIR(table_flags)) { dict_get_and_save_data_dir_path(table, true); ut_a(table->data_dir_path); @@ -4453,8 +4455,9 @@ row_drop_table_for_mysql( if (err == DB_SUCCESS && space_id > TRX_SYS_SPACE) { if (!is_temp && !fil_space_for_table_exists_in_mem( - space_id, tablename, FALSE, - print_msg, false, NULL, 0)) { + space_id, tablename, + print_msg, false, NULL, 0, + table_flags)) { /* This might happen if we are dropping a discarded tablespace */ err = DB_SUCCESS; diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index 77cdfe1affb..b933171e683 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -703,8 +703,7 @@ create_log_files( sprintf(logfilename + dirnamelen, "ib_logfile%u", INIT_LOG_FILE0); fil_space_create( - logfilename, SRV_LOG_SPACE_FIRST_ID, - fsp_flags_set_page_size(0, UNIV_PAGE_SIZE), + logfilename, SRV_LOG_SPACE_FIRST_ID, 0, FIL_LOG, NULL /* no encryption yet */, true /* this is create */); @@ -1193,7 +1192,7 @@ check_first_page: crypt_data = fil_space_create_crypt_data(FIL_SPACE_ENCRYPTION_DEFAULT, FIL_DEFAULT_ENCRYPTION_KEY); } - flags = fsp_flags_set_page_size(0, UNIV_PAGE_SIZE); + flags = FSP_FLAGS_PAGE_SSIZE(); fil_space_create(name, 0, flags, FIL_TABLESPACE, crypt_data, (*create_new_db) == true); @@ -1341,7 +1340,7 @@ srv_undo_tablespace_open( fil_set_max_space_id_if_bigger(space); /* Set the compressed page size to 0 (non-compressed) */ - flags = fsp_flags_set_page_size(0, UNIV_PAGE_SIZE); + flags = FSP_FLAGS_PAGE_SSIZE(); fil_space_create(name, space, flags, FIL_TABLESPACE, NULL /* no encryption */, true /* create */); @@ -2374,9 +2373,7 @@ innobase_start_or_create_for_mysql(void) sprintf(logfilename + dirnamelen, "ib_logfile%u", 0); fil_space_create(logfilename, - SRV_LOG_SPACE_FIRST_ID, - fsp_flags_set_page_size(0, UNIV_PAGE_SIZE), - FIL_LOG, + SRV_LOG_SPACE_FIRST_ID, 0, FIL_LOG, NULL /* no encryption yet */, true /* create */); @@ -2771,6 +2768,13 @@ files_checked: } if (!srv_read_only_mode) { + const ulint flags = FSP_FLAGS_PAGE_SSIZE(); + for (ulint id = 0; id <= srv_undo_tablespaces; id++) { + if (fil_space_get(id)) { + fsp_flags_try_adjust(id, flags); + } + } + /* Create the thread which watches the timeouts for lock waits */ thread_handles[2 + SRV_MAX_N_IO_THREADS] = os_thread_create( |