diff options
| author | Marko Mäkelä <marko.makela@mariadb.com> | 2023-04-24 15:59:23 +0300 |
|---|---|---|
| committer | Marko Mäkelä <marko.makela@mariadb.com> | 2023-04-24 15:59:23 +0300 |
| commit | 3c2507789908df103949670fe43412e220718a6e (patch) | |
| tree | 2576f11ac6f7d10547d3d65b8de60ed379d5d4bb | |
| parent | 10147af9e5b25739bada32c6eb4f204d5f2985a1 (diff) | |
| parent | 0976afec889d8914326f9e71b15ea215470dadba (diff) | |
| download | mariadb-git-3c2507789908df103949670fe43412e220718a6e.tar.gz | |
Merge 10.6 into 10.8
53 files changed, 1159 insertions, 487 deletions
diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index 9ece6d4b69a..8aa9b3a4a4f 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -179,34 +179,39 @@ dch -b -D "${LSBNAME}" -v "${VERSION}" "Automatic build with ${LOGSTRING}." --co echo "Creating package version ${VERSION} ... " -BUILDPACKAGE_DPKGCMD="" +BUILDPACKAGE_DPKGCMD=() + +# Fakeroot test +if fakeroot true; then + BUILDPACKAGE_DPKGCMD+=( "fakeroot" "--" ) +fi # Use eatmydata is available to build faster with less I/O, skipping fsync() # during the entire build process (safe because a build can always be restarted) if which eatmydata > /dev/null then - BUILDPACKAGE_DPKGCMD="eatmydata" + BUILDPACKAGE_DPKGCMD+=("eatmydata") fi -BUILDPACKAGE_DPKGCMD+="dpkg-buildpackage" +BUILDPACKAGE_DPKGCMD+=("dpkg-buildpackage") # Using dpkg-buildpackage args # -us Allow unsigned sources # -uc Allow unsigned changes # -I Tar ignore -BUILDPACKAGE_DPKG_ARGS=(-us -uc -I) +BUILDPACKAGE_DPKGCMD+=(-us -uc -I) # There can be also extra flags that are appended to args if [ -n "$BUILDPACKAGE_FLAGS" ] then read -ra BUILDPACKAGE_TMP_ARGS <<< "$BUILDPACKAGE_FLAGS" - BUILDPACKAGE_DPKG_ARGS=("${BUILDPACKAGE_DPKG_ARGS[@]} ${BUILDPACKAGE_TMP_ARGS[@]}") + BUILDPACKAGE_DPKGCMD+=( "${BUILDPACKAGE_TMP_ARGS[@]}" ) fi # Build the package # Pass -I so that .git and other unnecessary temporary and source control files # will be ignored by dpkg-source when creating the tar.gz source package. -fakeroot -- "${BUILDPACKAGE_DPKGCMD}" "${BUILDPACKAGE_DPKG_ARGS[@]}" +"${BUILDPACKAGE_DPKGCMD[@]}" # If the step above fails due to missing dependencies, you can manually run # sudo mk-build-deps debian/control -r -i diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 77bd0a081d4..3c4d4324785 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -3493,21 +3493,20 @@ static void xb_load_single_table_tablespace(const char *dirname, bool is_empty_file = file->exists() && file->is_empty_file(); if (err == DB_SUCCESS && file->space_id() != SRV_TMP_SPACE_ID) { + mysql_mutex_lock(&fil_system.mutex); space = fil_space_t::create( file->space_id(), file->flags(), FIL_TYPE_TABLESPACE, nullptr/* TODO: crypt_data */, FIL_ENCRYPTION_DEFAULT, file->handle() != OS_FILE_CLOSED); - - ut_a(space != NULL); + ut_ad(space); fil_node_t* node= space->add( file->filepath(), skip_node_page0 ? file->detach() : pfs_os_file_t(), 0, false, false); node->deferred= defer; - mysql_mutex_lock(&fil_system.mutex); if (!space->read_page0()) - err= DB_CANNOT_OPEN_FILE; + err = DB_CANNOT_OPEN_FILE; mysql_mutex_unlock(&fil_system.mutex); if (srv_operation == SRV_OPERATION_RESTORE_DELTA @@ -5302,9 +5301,12 @@ exit: ut_ad(fil_space_t::zip_size(flags) == info.zip_size); ut_ad(fil_space_t::physical_size(flags) == info.page_size); - if (fil_space_t::create(info.space_id, flags, - FIL_TYPE_TABLESPACE, 0, FIL_ENCRYPTION_DEFAULT, - true)) { + mysql_mutex_lock(&fil_system.mutex); + fil_space_t* space = fil_space_t::create(info.space_id, flags, + FIL_TYPE_TABLESPACE, 0, + FIL_ENCRYPTION_DEFAULT, true); + mysql_mutex_unlock(&fil_system.mutex); + if (space) { *success = xb_space_create_file(real_name, info.space_id, flags, &file); } else { diff --git a/mysql-test/main/group_min_max.result b/mysql-test/main/group_min_max.result index 2b8b10b29b9..706a4132614 100644 --- a/mysql-test/main/group_min_max.result +++ b/mysql-test/main/group_min_max.result @@ -4083,6 +4083,18 @@ MIN(pk) 1 DROP TABLE t1, t2; # +# MDEV-30605 Wrong result while using index for group-by +# +CREATE TABLE t1 (pk INT primary key, a int, key(a)) engine=innodb; +INSERT INTO t1 VALUES (1,-1),(2,8),(3,5),(4,-1),(5,10), (6,-1); +SELECT MIN(pk), a FROM t1 WHERE pk <> 1 GROUP BY a; +MIN(pk) a +4 -1 +3 5 +2 8 +5 10 +DROP TABLE t1; +# # End of 10.5 tests # # diff --git a/mysql-test/main/group_min_max.test b/mysql-test/main/group_min_max.test index 0fa91e4d72b..1bc334dd3da 100644 --- a/mysql-test/main/group_min_max.test +++ b/mysql-test/main/group_min_max.test @@ -1739,6 +1739,17 @@ SELECT MIN(pk) FROM t1, t2; DROP TABLE t1, t2; --echo # +--echo # MDEV-30605 Wrong result while using index for group-by +--echo # + +CREATE TABLE t1 (pk INT primary key, a int, key(a)) engine=innodb; +INSERT INTO t1 VALUES (1,-1),(2,8),(3,5),(4,-1),(5,10), (6,-1); + +SELECT MIN(pk), a FROM t1 WHERE pk <> 1 GROUP BY a; + +DROP TABLE t1; + +--echo # --echo # End of 10.5 tests --echo # diff --git a/mysql-test/main/mysql_tzinfo_to_sql_symlink.result b/mysql-test/main/mysql_tzinfo_to_sql_symlink.result index b6b35e44988..97548768a2d 100644 --- a/mysql-test/main/mysql_tzinfo_to_sql_symlink.result +++ b/mysql-test/main/mysql_tzinfo_to_sql_symlink.result @@ -10,13 +10,13 @@ CREATE TABLE time_zone_leap_second LIKE mysql.time_zone_leap_second; set @wsrep_is_on=(select coalesce(sum(SESSION_VALUE='ON'), 0) from information_schema.SYSTEM_VARIABLES WHERE VARIABLE_NAME='wsrep_on'); SET STATEMENT SQL_MODE='' FOR SELECT concat('%', GROUP_CONCAT(OPTION), '%') INTO @replicate_opt FROM (SELECT DISTINCT concat('REPLICATE_', UPPER(ENGINE)) AS OPTION FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN ('time_zone', 'time_zone_name', 'time_zone_transition', 'time_zone_transition_type', 'time_zone_leap_second') AND ENGINE in ('MyISAM', 'Aria')) AS o ORDER BY OPTION DESC; set @wsrep_cannot_replicate_tz=@wsrep_is_on AND (select coalesce(sum(GLOBAL_VALUE NOT LIKE @replicate_opt), 0) from information_schema.SYSTEM_VARIABLES WHERE VARIABLE_NAME='wsrep_mode'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_name_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_name'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_name_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_name''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_name ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_transition_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_transition'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_transition_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_transition''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_transition ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_transition_type_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_transition_type'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_transition_type_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_transition_type''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_transition_type ENGINE=InnoDB', 'do 0'); TRUNCATE TABLE time_zone; TRUNCATE TABLE time_zone_name; @@ -59,13 +59,13 @@ execute immediate if(@wsrep_cannot_replicate_tz, concat('ALTER TABLE time_zone_t set @wsrep_is_on=(select coalesce(sum(SESSION_VALUE='ON'), 0) from information_schema.SYSTEM_VARIABLES WHERE VARIABLE_NAME='wsrep_on'); SET STATEMENT SQL_MODE='' FOR SELECT concat('%', GROUP_CONCAT(OPTION), '%') INTO @replicate_opt FROM (SELECT DISTINCT concat('REPLICATE_', UPPER(ENGINE)) AS OPTION FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN ('time_zone', 'time_zone_name', 'time_zone_transition', 'time_zone_transition_type', 'time_zone_leap_second') AND ENGINE in ('MyISAM', 'Aria')) AS o ORDER BY OPTION DESC; set @wsrep_cannot_replicate_tz=@wsrep_is_on AND (select coalesce(sum(GLOBAL_VALUE NOT LIKE @replicate_opt), 0) from information_schema.SYSTEM_VARIABLES WHERE VARIABLE_NAME='wsrep_mode'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_name_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_name'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_name_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_name''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_name ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_transition_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_transition'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_transition_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_transition''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_transition ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_transition_type_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_transition_type'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_transition_type_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_transition_type''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_transition_type ENGINE=InnoDB', 'do 0'); TRUNCATE TABLE time_zone; TRUNCATE TABLE time_zone_name; @@ -191,13 +191,13 @@ TRUNCATE TABLE time_zone_leap_second; set @wsrep_is_on=(select coalesce(sum(SESSION_VALUE='ON'), 0) from information_schema.SYSTEM_VARIABLES WHERE VARIABLE_NAME='wsrep_on'); SET STATEMENT SQL_MODE='' FOR SELECT concat('%', GROUP_CONCAT(OPTION), '%') INTO @replicate_opt FROM (SELECT DISTINCT concat('REPLICATE_', UPPER(ENGINE)) AS OPTION FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN ('time_zone', 'time_zone_name', 'time_zone_transition', 'time_zone_transition_type', 'time_zone_leap_second') AND ENGINE in ('MyISAM', 'Aria')) AS o ORDER BY OPTION DESC; set @wsrep_cannot_replicate_tz=@wsrep_is_on AND (select coalesce(sum(GLOBAL_VALUE NOT LIKE @replicate_opt), 0) from information_schema.SYSTEM_VARIABLES WHERE VARIABLE_NAME='wsrep_mode'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_name_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_name'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_name_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_name''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_name ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_transition_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_transition'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_transition_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_transition''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_transition ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_transition_type_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_transition_type'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_transition_type_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_transition_type''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_transition_type ENGINE=InnoDB', 'do 0'); /*M!100602 execute immediate if(@wsrep_cannot_replicate_tz, 'start transaction', 'LOCK TABLES time_zone WRITE, time_zone_leap_second WRITE, @@ -313,20 +313,20 @@ TRUNCATE TABLE time_zone_leap_second; set @wsrep_is_on=(select coalesce(sum(SESSION_VALUE='ON'), 0) from information_schema.SYSTEM_VARIABLES WHERE VARIABLE_NAME='wsrep_on'); SET STATEMENT SQL_MODE='' FOR SELECT concat('%', GROUP_CONCAT(OPTION), '%') INTO @replicate_opt FROM (SELECT DISTINCT concat('REPLICATE_', UPPER(ENGINE)) AS OPTION FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN ('time_zone', 'time_zone_name', 'time_zone_transition', 'time_zone_transition_type', 'time_zone_leap_second') AND ENGINE in ('MyISAM', 'Aria')) AS o ORDER BY OPTION DESC; set @wsrep_cannot_replicate_tz=@wsrep_is_on AND (select coalesce(sum(GLOBAL_VALUE NOT LIKE @replicate_opt), 0) from information_schema.SYSTEM_VARIABLES WHERE VARIABLE_NAME='wsrep_mode'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_name_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_name'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_name_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_name''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_name ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_transition_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_transition'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_transition_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_transition''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_transition ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_transition_type_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_transition_type'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_transition_type_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_transition_type''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_transition_type ENGINE=InnoDB', 'do 0'); /*M!100602 execute immediate if(@wsrep_cannot_replicate_tz, 'start transaction', 'LOCK TABLES time_zone WRITE, time_zone_leap_second WRITE, time_zone_name WRITE, time_zone_transition WRITE, time_zone_transition_type WRITE')*/; -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_leap_second_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_leap_second'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_leap_second_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_leap_second''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_leap_second ENGINE=InnoDB', 'do 0'); TRUNCATE TABLE time_zone_leap_second; execute immediate if(@wsrep_cannot_replicate_tz, concat('ALTER TABLE time_zone_leap_second ENGINE=', @time_zone_leap_second_engine), 'do 0'); @@ -497,13 +497,13 @@ set sql_mode=default; set @wsrep_is_on=(select coalesce(sum(SESSION_VALUE='ON'), 0) from information_schema.SYSTEM_VARIABLES WHERE VARIABLE_NAME='wsrep_on'); SET STATEMENT SQL_MODE='' FOR SELECT concat('%', GROUP_CONCAT(OPTION), '%') INTO @replicate_opt FROM (SELECT DISTINCT concat('REPLICATE_', UPPER(ENGINE)) AS OPTION FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN ('time_zone', 'time_zone_name', 'time_zone_transition', 'time_zone_transition_type', 'time_zone_leap_second') AND ENGINE in ('MyISAM', 'Aria')) AS o ORDER BY OPTION DESC; set @wsrep_cannot_replicate_tz=@wsrep_is_on AND (select coalesce(sum(GLOBAL_VALUE NOT LIKE @replicate_opt), 0) from information_schema.SYSTEM_VARIABLES WHERE VARIABLE_NAME='wsrep_mode'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_name_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_name'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_name_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_name''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_name ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_transition_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_transition'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_transition_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_transition''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_transition ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_transition_type_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_transition_type'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_transition_type_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_transition_type''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_transition_type ENGINE=InnoDB', 'do 0'); TRUNCATE TABLE time_zone; TRUNCATE TABLE time_zone_name; diff --git a/mysql-test/mariadb-test-run.pl b/mysql-test/mariadb-test-run.pl index 670f63a990f..4fc4d9f050e 100755 --- a/mysql-test/mariadb-test-run.pl +++ b/mysql-test/mariadb-test-run.pl @@ -145,7 +145,6 @@ my $opt_start_exit; my $start_only; my $file_wsrep_provider; my $num_saved_cores= 0; # Number of core files saved in vardir/log/ so far. -my $test_name_for_report; our @global_suppressions; @@ -402,6 +401,11 @@ sub main { mtr_report("Collecting tests..."); my $tests= collect_test_cases($opt_reorder, $opt_suites, \@opt_cases, \@opt_skip_test_list); + if (@$tests == 0) { + mtr_report("No tests to run..."); + exit 0; + } + mark_time_used('collect'); mysql_install_db(default_mysqld(), "$opt_vardir/install.db") unless using_extern(); @@ -516,13 +520,13 @@ sub main { } if ( not @$completed ) { - if ($test_name_for_report) - { - my $tinfo = My::Test->new(name => $test_name_for_report); - $tinfo->{result}= 'MTR_RES_FAILED'; - $tinfo->{comment}=' '; - mtr_report_test($tinfo); - } + my $test_name= mtr_grab_file($path_testlog); + $test_name =~ s/^CURRENT_TEST:\s//; + chomp($test_name); + my $tinfo = My::Test->new(name => $test_name); + $tinfo->{result}= 'MTR_RES_FAILED'; + $tinfo->{comment}=' '; + mtr_report_test($tinfo); mtr_error("Test suite aborted"); } @@ -3741,8 +3745,8 @@ sub resfile_report_test ($) { sub run_testcase ($$) { my ($tinfo, $server_socket)= @_; my $print_freq=20; - $test_name_for_report= $tinfo->{name}; - mtr_verbose("Running test:", $test_name_for_report); + + mtr_verbose("Running test:", $tinfo->{name}); $ENV{'MTR_TEST_NAME'} = $tinfo->{name}; resfile_report_test($tinfo) if $opt_resfile; @@ -5131,10 +5135,12 @@ sub mysqld_start ($$) { if (!$rc) { # Report failure about the last test case before exit - my $tinfo = My::Test->new(name => $test_name_for_report); + my $test_name= mtr_grab_file($path_current_testlog); + $test_name =~ s/^CURRENT_TEST:\s//; + my $tinfo = My::Test->new(name => $test_name); $tinfo->{result}= 'MTR_RES_FAILED'; $tinfo->{failures}= 1; - $tinfo->{logfile}=get_log_from_proc($mysqld->{'proc'}, $test_name_for_report); + $tinfo->{logfile}=get_log_from_proc($mysqld->{'proc'}, $tinfo->{name}); report_option('verbose', 1); mtr_report_test($tinfo); } diff --git a/mysql-test/suite/binlog/t/binlog_truncate_multi_engine.inc b/mysql-test/suite/binlog/t/binlog_truncate_multi_engine.inc index 52ce4741eaa..f3801070851 100644 --- a/mysql-test/suite/binlog/t/binlog_truncate_multi_engine.inc +++ b/mysql-test/suite/binlog/t/binlog_truncate_multi_engine.inc @@ -20,7 +20,9 @@ connect(con1,localhost,root,,); --source include/show_binary_logs.inc INSERT INTO t1 VALUES (1, REPEAT("x", 1)); INSERT INTO t2 VALUES (1, REPEAT("x", 1)); -if (`SELECT $case = "B"`) +--let $is_case_B=`SELECT $case = "B"` + +if ($is_case_B) { --write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect wait-binlog_truncate_multi_engine.test @@ -39,12 +41,12 @@ if (`SELECT $debug_sync_action != ""`) send COMMIT; --connection default -if (`SELECT $case = "B"`) +if ($is_case_B) { --source include/wait_until_disconnected.inc --source include/start_mysqld.inc } -if (`SELECT $case != "B"`) +if (!$is_case_B) { SET DEBUG_SYNC= "now WAIT_FOR con1_ready"; --echo List of binary logs after rotation diff --git a/mysql-test/suite/galera/r/MDEV-30804.result b/mysql-test/suite/galera/r/MDEV-30804.result new file mode 100644 index 00000000000..2bf323d19f8 --- /dev/null +++ b/mysql-test/suite/galera/r/MDEV-30804.result @@ -0,0 +1,11 @@ +connection node_2; +connection node_1; +CREATE TABLE t (a INT) ENGINE=Aria; +CREATE TABLE t1 (a INT) ENGINE=InnoDB; +START TRANSACTION; +INSERT INTO t VALUES ('1'); +INSERT INTO t1 VALUES ('1'); +COMMIT; +ERROR HY000: Transactional commit not supported by involved engine(s) +DROP TABLE t; +DROP TABLE t1; diff --git a/mysql-test/suite/galera/t/MDEV-30804.cnf b/mysql-test/suite/galera/t/MDEV-30804.cnf new file mode 100644 index 00000000000..9dbd81f758d --- /dev/null +++ b/mysql-test/suite/galera/t/MDEV-30804.cnf @@ -0,0 +1,7 @@ +!include ../galera_2nodes.cnf + +[mysqld.1] +log-bin + +[mysqld.2] +log-bin diff --git a/mysql-test/suite/galera/t/MDEV-30804.test b/mysql-test/suite/galera/t/MDEV-30804.test new file mode 100644 index 00000000000..561953a0578 --- /dev/null +++ b/mysql-test/suite/galera/t/MDEV-30804.test @@ -0,0 +1,21 @@ +# +# Test that transaction requiring two-phase commit and involving +# storage engines not supporting it rolls back with a message. +# + +--source include/galera_cluster.inc +--source include/have_innodb.inc +--source include/have_aria.inc + +CREATE TABLE t (a INT) ENGINE=Aria; +CREATE TABLE t1 (a INT) ENGINE=InnoDB; + +START TRANSACTION; +INSERT INTO t VALUES ('1'); +INSERT INTO t1 VALUES ('1'); + +--error ER_ERROR_DURING_COMMIT +COMMIT; + +DROP TABLE t; +DROP TABLE t1; diff --git a/mysql-test/suite/innodb/r/innodb_skip_innodb_is_tables.result b/mysql-test/suite/innodb/r/innodb_skip_innodb_is_tables.result index dd253311ddd..385034f7853 100644 --- a/mysql-test/suite/innodb/r/innodb_skip_innodb_is_tables.result +++ b/mysql-test/suite/innodb/r/innodb_skip_innodb_is_tables.result @@ -161,8 +161,8 @@ trx_commits_insert_update transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NUL trx_rollbacks transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of transactions rolled back trx_rollbacks_savepoint transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of transactions rolled back to savepoint trx_rseg_history_len transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 value Length of the TRX_RSEG_HISTORY list -trx_undo_slots_used transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of undo slots used -trx_undo_slots_cached transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of undo slots cached +trx_undo_slots_used transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 value Number of undo slots used +trx_undo_slots_cached transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 value Number of undo slots cached trx_rseg_current_size transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 value Current rollback segment size in pages purge_del_mark_records purge 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of delete-marked rows purged purge_upd_exist_or_extern_records purge 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of purges on updates of existing records and updates on delete marked record with externally stored field diff --git a/mysql-test/suite/innodb_fts/r/concurrent_insert.result b/mysql-test/suite/innodb_fts/r/concurrent_insert.result index 2335982816b..bc47511b046 100644 --- a/mysql-test/suite/innodb_fts/r/concurrent_insert.result +++ b/mysql-test/suite/innodb_fts/r/concurrent_insert.result @@ -19,7 +19,7 @@ INSERT INTO t2 VALUES('mariadb'); connection default; SET @saved_dbug = @@GLOBAL.debug_dbug; SET GLOBAL debug_dbug ='+d,fts_instrument_sync_request,ib_optimize_wq_hang'; -SET DEBUG_SYNC= 'fts_sync_end +SET DEBUG_SYNC= 'fts_instrument_sync_request SIGNAL drop_index_start WAIT_FOR sync_op'; INSERT INTO t1 VALUES('Keyword'); connect con1,localhost,root,,,; diff --git a/mysql-test/suite/innodb_fts/r/sync.result b/mysql-test/suite/innodb_fts/r/sync.result index 74a5d2f13fb..928efffdb21 100644 --- a/mysql-test/suite/innodb_fts/r/sync.result +++ b/mysql-test/suite/innodb_fts/r/sync.result @@ -11,19 +11,19 @@ INSERT INTO t1(title) VALUES('database'); connection con1; SET @old_dbug = @@SESSION.debug_dbug; SET debug_dbug = '+d,fts_instrument_sync_debug'; -SET DEBUG_SYNC= 'fts_sync_end SIGNAL written WAIT_FOR selected'; +SET DEBUG_SYNC= 'fts_write_node SIGNAL written WAIT_FOR selected'; INSERT INTO t1(title) VALUES('mysql database'); connection default; SET DEBUG_SYNC= 'now WAIT_FOR written'; SET GLOBAL innodb_ft_aux_table="test/t1"; SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION -SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE; -WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION database 2 3 2 2 0 database 2 3 2 3 6 mysql 1 3 2 1 0 mysql 1 3 2 3 0 +SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE; +WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION SET GLOBAL innodb_ft_aux_table=default; SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); FTS_DOC_ID title @@ -59,7 +59,7 @@ INSERT INTO t1(title) VALUES('mysql'); INSERT INTO t1(title) VALUES('database'); connection con1; SET debug_dbug = '+d,fts_instrument_sync_debug'; -SET DEBUG_SYNC= 'fts_sync_end SIGNAL written WAIT_FOR inserted'; +SET DEBUG_SYNC= 'fts_write_node SIGNAL written WAIT_FOR inserted'; INSERT INTO t1(title) VALUES('mysql database'); connection default; SET DEBUG_SYNC= 'now WAIT_FOR written'; @@ -70,14 +70,14 @@ SET debug_dbug = @old_dbug; SET GLOBAL innodb_ft_aux_table="test/t1"; SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION -database 4 4 1 4 6 -mysql 4 4 1 4 0 SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE; WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION database 2 3 2 2 0 database 2 3 2 3 6 -mysql 1 3 2 1 0 -mysql 1 3 2 3 0 +database 4 4 1 4 6 +mysql 1 4 3 1 0 +mysql 1 4 3 3 0 +mysql 1 4 3 4 0 SET GLOBAL innodb_ft_aux_table=default; SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); FTS_DOC_ID title diff --git a/mysql-test/suite/innodb_fts/r/sync_block.result b/mysql-test/suite/innodb_fts/r/sync_block.result new file mode 100644 index 00000000000..65bee127e80 --- /dev/null +++ b/mysql-test/suite/innodb_fts/r/sync_block.result @@ -0,0 +1,83 @@ +SET @old_log_output = @@global.log_output; +SET @old_slow_query_log = @@global.slow_query_log; +SET @old_general_log = @@global.general_log; +SET @old_long_query_time = @@global.long_query_time; +SET @old_debug = @@global.debug_dbug; +SET GLOBAL log_output = 'TABLE'; +SET GLOBAL general_log = 1; +SET GLOBAL slow_query_log = 1; +SET GLOBAL long_query_time = 1; +connect con1,localhost,root,,; +connect con2,localhost,root,,; +connection default; +# Case 1: Sync blocks DML(insert) on the same table. +CREATE TABLE t1 ( +FTS_DOC_ID BIGINT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY, +title VARCHAR(200), +FULLTEXT(title) +) ENGINE = InnoDB; +connection con1; +SET GLOBAL debug_dbug='+d,fts_instrument_sync_debug,fts_instrument_sync_sleep'; +SET DEBUG_SYNC= 'fts_sync_begin SIGNAL begin WAIT_FOR continue'; +INSERT INTO t1(title) VALUES('mysql database'); +connection con2; +SET DEBUG_SYNC= 'now WAIT_FOR begin'; +SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); +connection default; +SET DEBUG_SYNC= 'now SIGNAL continue'; +connection con1; +/* connection con1 */ INSERT INTO t1(title) VALUES('mysql database'); +connection con2; +/* conneciton con2 */ SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); +FTS_DOC_ID title +connection default; +# make con1 & con2 show up in mysql.slow_log +SELECT SLEEP(2); +SLEEP(2) +0 +# slow log results should only contain INSERT INTO t1. +SELECT sql_text FROM mysql.slow_log WHERE query_time >= '00:00:02'; +sql_text +INSERT INTO t1(title) VALUES('mysql database') +SET GLOBAL debug_dbug = @old_debug; +TRUNCATE TABLE mysql.slow_log; +DROP TABLE t1; +# Case 2: Sync blocks DML(insert) on other tables. +CREATE TABLE t1 ( +FTS_DOC_ID BIGINT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY, +title VARCHAR(200), +FULLTEXT(title) +) ENGINE = InnoDB; +CREATE TABLE t2(id INT); +connection con1; +SET GLOBAL debug_dbug='+d,fts_instrument_sync_request,fts_instrument_sync_sleep'; +SET DEBUG_SYNC= 'fts_instrument_sync_request SIGNAL begin WAIT_FOR continue'; +INSERT INTO t1(title) VALUES('mysql database'); +connection con2; +SET DEBUG_SYNC= 'now WAIT_FOR begin'; +INSERT INTO t2 VALUES(1); +connection default; +SET DEBUG_SYNC= 'now SIGNAL continue'; +connection con1; +/* connection con1 */ INSERT INTO t1(title) VALUES('mysql database'); +connection con2; +/* conneciton con2 */ INSERT INTO t2 VALUES(1); +connection default; +SET DEBUG_SYNC = 'RESET'; +# make con1 & con2 show up in mysql.slow_log +SELECT SLEEP(2); +SLEEP(2) +0 +# slow log results should be empty here. +SELECT sql_text FROM mysql.slow_log WHERE query_time >= '00:00:02'; +sql_text +SET GLOBAL debug_dbug = @old_debug; +TRUNCATE TABLE mysql.slow_log; +DROP TABLE t1,t2; +disconnect con1; +disconnect con2; +# Restore slow log settings. +SET GLOBAL log_output = @old_log_output; +SET GLOBAL general_log = @old_general_log; +SET GLOBAL slow_query_log = @old_slow_query_log; +SET GLOBAL long_query_time = @old_long_query_time; diff --git a/mysql-test/suite/innodb_fts/t/concurrent_insert.test b/mysql-test/suite/innodb_fts/t/concurrent_insert.test index b6991f6e503..9b4d9517b1a 100644 --- a/mysql-test/suite/innodb_fts/t/concurrent_insert.test +++ b/mysql-test/suite/innodb_fts/t/concurrent_insert.test @@ -31,7 +31,7 @@ INSERT INTO t2 VALUES('mariadb'); connection default; SET @saved_dbug = @@GLOBAL.debug_dbug; SET GLOBAL debug_dbug ='+d,fts_instrument_sync_request,ib_optimize_wq_hang'; -SET DEBUG_SYNC= 'fts_sync_end +SET DEBUG_SYNC= 'fts_instrument_sync_request SIGNAL drop_index_start WAIT_FOR sync_op'; send INSERT INTO t1 VALUES('Keyword'); diff --git a/mysql-test/suite/innodb_fts/t/sync.test b/mysql-test/suite/innodb_fts/t/sync.test index 7c5c835f2ee..168309a5c92 100644 --- a/mysql-test/suite/innodb_fts/t/sync.test +++ b/mysql-test/suite/innodb_fts/t/sync.test @@ -27,7 +27,7 @@ connection con1; SET @old_dbug = @@SESSION.debug_dbug; SET debug_dbug = '+d,fts_instrument_sync_debug'; -SET DEBUG_SYNC= 'fts_sync_end SIGNAL written WAIT_FOR selected'; +SET DEBUG_SYNC= 'fts_write_node SIGNAL written WAIT_FOR selected'; send INSERT INTO t1(title) VALUES('mysql database'); @@ -74,7 +74,7 @@ connection con1; SET debug_dbug = '+d,fts_instrument_sync_debug'; -SET DEBUG_SYNC= 'fts_sync_end SIGNAL written WAIT_FOR inserted'; +SET DEBUG_SYNC= 'fts_write_node SIGNAL written WAIT_FOR inserted'; send INSERT INTO t1(title) VALUES('mysql database'); diff --git a/mysql-test/suite/innodb_fts/t/sync_block.test b/mysql-test/suite/innodb_fts/t/sync_block.test new file mode 100644 index 00000000000..895d2ba8a59 --- /dev/null +++ b/mysql-test/suite/innodb_fts/t/sync_block.test @@ -0,0 +1,124 @@ +# +# BUG#22516559 MYSQL INSTANCE STALLS WHEN SYNCING FTS INDEX +# + +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/have_log_bin.inc +--source include/count_sessions.inc + +SET @old_log_output = @@global.log_output; +SET @old_slow_query_log = @@global.slow_query_log; +SET @old_general_log = @@global.general_log; +SET @old_long_query_time = @@global.long_query_time; +SET @old_debug = @@global.debug_dbug; + +SET GLOBAL log_output = 'TABLE'; +SET GLOBAL general_log = 1; +SET GLOBAL slow_query_log = 1; +SET GLOBAL long_query_time = 1; + +connect (con1,localhost,root,,); +connect (con2,localhost,root,,); +connection default; + +--echo # Case 1: Sync blocks DML(insert) on the same table. +CREATE TABLE t1 ( + FTS_DOC_ID BIGINT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY, + title VARCHAR(200), + FULLTEXT(title) +) ENGINE = InnoDB; + +connection con1; + +SET GLOBAL debug_dbug='+d,fts_instrument_sync_debug,fts_instrument_sync_sleep'; + +SET DEBUG_SYNC= 'fts_sync_begin SIGNAL begin WAIT_FOR continue'; + +send INSERT INTO t1(title) VALUES('mysql database'); + +connection con2; + +SET DEBUG_SYNC= 'now WAIT_FOR begin'; + +send SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); + +connection default; +SET DEBUG_SYNC= 'now SIGNAL continue'; + +connection con1; +--echo /* connection con1 */ INSERT INTO t1(title) VALUES('mysql database'); +--reap + +connection con2; +--echo /* conneciton con2 */ SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); +--reap + +connection default; +-- echo # make con1 & con2 show up in mysql.slow_log +SELECT SLEEP(2); +-- echo # slow log results should only contain INSERT INTO t1. +SELECT sql_text FROM mysql.slow_log WHERE query_time >= '00:00:02'; + +SET GLOBAL debug_dbug = @old_debug; +TRUNCATE TABLE mysql.slow_log; + +DROP TABLE t1; + +--echo # Case 2: Sync blocks DML(insert) on other tables. +CREATE TABLE t1 ( + FTS_DOC_ID BIGINT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY, + title VARCHAR(200), + FULLTEXT(title) +) ENGINE = InnoDB; + +CREATE TABLE t2(id INT); + +connection con1; + +SET GLOBAL debug_dbug='+d,fts_instrument_sync_request,fts_instrument_sync_sleep'; + +SET DEBUG_SYNC= 'fts_instrument_sync_request SIGNAL begin WAIT_FOR continue'; + +send INSERT INTO t1(title) VALUES('mysql database'); + +connection con2; + +SET DEBUG_SYNC= 'now WAIT_FOR begin'; + +send INSERT INTO t2 VALUES(1); + +connection default; +SET DEBUG_SYNC= 'now SIGNAL continue'; + +connection con1; +--echo /* connection con1 */ INSERT INTO t1(title) VALUES('mysql database'); +--reap + +connection con2; +--echo /* conneciton con2 */ INSERT INTO t2 VALUES(1); +--reap + +connection default; +SET DEBUG_SYNC = 'RESET'; +-- echo # make con1 & con2 show up in mysql.slow_log +SELECT SLEEP(2); +-- echo # slow log results should be empty here. +SELECT sql_text FROM mysql.slow_log WHERE query_time >= '00:00:02'; + +SET GLOBAL debug_dbug = @old_debug; +TRUNCATE TABLE mysql.slow_log; + +DROP TABLE t1,t2; + +disconnect con1; +disconnect con2; + +--source include/wait_until_count_sessions.inc + +-- echo # Restore slow log settings. +SET GLOBAL log_output = @old_log_output; +SET GLOBAL general_log = @old_general_log; +SET GLOBAL slow_query_log = @old_slow_query_log; +SET GLOBAL long_query_time = @old_long_query_time; diff --git a/sql/handler.cc b/sql/handler.cc index a2bbeed589b..a8c4cde9898 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -1821,7 +1821,19 @@ int ha_commit_trans(THD *thd, bool all) ordering is normally done. Commit ordering must be done here. */ if (run_wsrep_hooks) - error= wsrep_before_commit(thd, all); + { + // This commit involves more than one storage engine and requires + // two phases, but some engines don't support it. + // Issue a message to the client and roll back the transaction. + if (trans->no_2pc && rw_ha_count > 1) + { + my_message(ER_ERROR_DURING_COMMIT, "Transactional commit not supported " + "by involved engine(s)", MYF(0)); + error= 1; + } + else + error= wsrep_before_commit(thd, all); + } if (error) { ha_rollback_trans(thd, FALSE); diff --git a/sql/opt_range.cc b/sql/opt_range.cc index 86bd5663623..9b69a62044b 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -461,7 +461,7 @@ void print_range_for_non_indexed_field(String *out, Field *field, static void print_min_range_operator(String *out, const ha_rkey_function flag); static void print_max_range_operator(String *out, const ha_rkey_function flag); -static bool is_field_an_unique_index(RANGE_OPT_PARAM *param, Field *field); +static bool is_field_an_unique_index(Field *field); /* SEL_IMERGE is a list of possible ways to do index merge, i.e. it is @@ -7759,8 +7759,13 @@ SEL_TREE *Item_func_ne::get_func_mm_tree(RANGE_OPT_PARAM *param, If this condition is a "col1<>...", where there is a UNIQUE KEY(col1), do not construct a SEL_TREE from it. A condition that excludes just one row in the table is not selective (unless there are only a few rows) + + Note: this logic must be in sync with code in + check_group_min_max_predicates(). That function walks an Item* condition + and checks if the range optimizer would produce an equivalent range for + it. */ - if (is_field_an_unique_index(param, field)) + if (param->using_real_indexes && is_field_an_unique_index(field)) DBUG_RETURN(NULL); DBUG_RETURN(get_ne_mm_tree(param, field, value, value)); } @@ -7872,7 +7877,7 @@ SEL_TREE *Item_func_in::get_func_mm_tree(RANGE_OPT_PARAM *param, - if there are a lot of constants, the overhead of building and processing enormous range list is not worth it. */ - if (is_field_an_unique_index(param, field)) + if (param->using_real_indexes && is_field_an_unique_index(field)) DBUG_RETURN(0); /* Get a SEL_TREE for "(-inf|NULL) < X < c_0" interval. */ @@ -8581,24 +8586,18 @@ SEL_TREE *Item_equal::get_mm_tree(RANGE_OPT_PARAM *param, Item **cond_ptr) In the future we could also add "almost unique" indexes where any value is present only in a few rows (but necessarily exactly one row) */ -static bool is_field_an_unique_index(RANGE_OPT_PARAM *param, Field *field) +static bool is_field_an_unique_index(Field *field) { DBUG_ENTER("is_field_an_unique_index"); - - // The check for using_real_indexes is there because of the heuristics - // this function is used for. - if (param->using_real_indexes) + key_map::Iterator it(field->key_start); + uint key_no; + while ((key_no= it++) != key_map::Iterator::BITMAP_END) { - key_map::Iterator it(field->key_start); - uint key_no; - while ((key_no= it++) != key_map::Iterator::BITMAP_END) + KEY *key_info= &field->table->key_info[key_no]; + if (key_info->user_defined_key_parts == 1 && + (key_info->flags & HA_NOSAME)) { - KEY *key_info= &field->table->key_info[key_no]; - if (key_info->user_defined_key_parts == 1 && - (key_info->flags & HA_NOSAME)) - { - DBUG_RETURN(true); - } + DBUG_RETURN(true); } } DBUG_RETURN(false); @@ -13537,7 +13536,7 @@ cost_group_min_max(TABLE* table, KEY *index_info, uint used_key_parts, - (C between const_i and const_j) - C IS NULL - C IS NOT NULL - - C != const + - C != const (unless C is the primary key) SA4. If Q has a GROUP BY clause, there are no other aggregate functions except MIN and MAX. For queries with DISTINCT, aggregate functions are allowed. @@ -14431,6 +14430,17 @@ check_group_min_max_predicates(Item *cond, Item_field *min_max_arg_item, if (!simple_pred(pred, args, &inv)) DBUG_RETURN(FALSE); + /* + Follow the logic in Item_func_ne::get_func_mm_tree(): condition + in form "tbl.primary_key <> const" is not used to produce intervals. + + If the condition doesn't have an equivalent interval, this means we + fail LooseScan's condition SA3. Return FALSE to indicate this. + */ + if (pred_type == Item_func::NE_FUNC && + is_field_an_unique_index(min_max_arg_item->field)) + DBUG_RETURN(FALSE); + if (args[0] && args[1]) // this is a binary function or BETWEEN { DBUG_ASSERT(pred->fixed_type_handler()); diff --git a/sql/tztime.cc b/sql/tztime.cc index 8e1a8805d96..8ddb9f0e30e 100644 --- a/sql/tztime.cc +++ b/sql/tztime.cc @@ -2433,9 +2433,9 @@ print_tz_as_sql(const char* tz_name, const TIME_ZONE_INFO *sp) #define SAVE_ENGINE(e) \ - "\"select ENGINE into @" e "_engine" \ + "'select ENGINE into @" e "_engine" \ " from information_schema.TABLES" \ - " where TABLE_SCHEMA=DATABASE() and TABLE_NAME='" e "'\"" + " where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''" e "'''" /* Print info about leap seconds in time zone as SQL statements diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 6308ea0b5dc..365ec1340f0 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1311,11 +1311,11 @@ buf_tmp_buffer_t *buf_pool_t::io_buf_t::reserve() for (buf_tmp_buffer_t *s= slots, *e= slots + n_slots; s != e; s++) if (s->acquire()) return s; - os_aio_wait_until_no_pending_writes(); + os_aio_wait_until_no_pending_writes(true); for (buf_tmp_buffer_t *s= slots, *e= slots + n_slots; s != e; s++) if (s->acquire()) return s; - os_aio_wait_until_no_pending_reads(); + os_aio_wait_until_no_pending_reads(true); } } diff --git a/storage/innobase/buf/buf0dump.cc b/storage/innobase/buf/buf0dump.cc index 63a28aa1a37..bb705ec5651 100644 --- a/storage/innobase/buf/buf0dump.cc +++ b/storage/innobase/buf/buf0dump.cc @@ -651,7 +651,7 @@ buf_load() ut_free(dump); if (i == dump_n) { - os_aio_wait_until_no_pending_reads(); + os_aio_wait_until_no_pending_reads(true); } ut_sprintf_timestamp(now); diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 4a8924f3282..ad3c313f631 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -209,7 +209,7 @@ void buf_flush_remove_pages(uint32_t id) if (!deferred) break; - os_aio_wait_until_no_pending_writes(); + os_aio_wait_until_no_pending_writes(true); } } @@ -1216,6 +1216,11 @@ static void buf_flush_LRU_list_batch(ulint max, bool evict, ++n->evicted; /* fall through */ case 1: + if (UNIV_LIKELY(scanned & 31)) + continue; + mysql_mutex_unlock(&buf_pool.mutex); + reacquire_mutex: + mysql_mutex_lock(&buf_pool.mutex); continue; } @@ -1251,32 +1256,37 @@ static void buf_flush_LRU_list_batch(ulint max, bool evict, auto p= buf_flush_space(space_id); space= p.first; last_space_id= space_id; + if (!space) + { + mysql_mutex_lock(&buf_pool.mutex); + goto no_space; + } mysql_mutex_lock(&buf_pool.mutex); - if (p.second) - buf_pool.stat.n_pages_written+= p.second; + buf_pool.stat.n_pages_written+= p.second; } else + { ut_ad(!space); + goto no_space; + } } else if (space->is_stopping()) { space->release(); space= nullptr; - } - - if (!space) - { + no_space: mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_flush_discard_page(bpage); + continue; } - else if (neighbors && space->is_rotational()) + + if (neighbors && space->is_rotational()) { mysql_mutex_unlock(&buf_pool.mutex); n->flushed+= buf_flush_try_neighbors(space, page_id, bpage, neighbors == 1, do_evict, n->flushed, max); -reacquire_mutex: - mysql_mutex_lock(&buf_pool.mutex); + goto reacquire_mutex; } else if (n->flushed >= max && !recv_recovery_is_on()) { @@ -1643,7 +1653,7 @@ done: space->release(); if (space->purpose == FIL_TYPE_IMPORT) - os_aio_wait_until_no_pending_writes(); + os_aio_wait_until_no_pending_writes(true); else buf_dblwr.flush_buffered_writes(); @@ -1855,7 +1865,7 @@ static void buf_flush_wait(lsn_t lsn) break; } mysql_mutex_unlock(&buf_pool.flush_list_mutex); - os_aio_wait_until_no_pending_writes(); + os_aio_wait_until_no_pending_writes(false); mysql_mutex_lock(&buf_pool.flush_list_mutex); } } @@ -1890,16 +1900,20 @@ ATTRIBUTE_COLD void buf_flush_wait_flushed(lsn_t sync_lsn) MONITOR_FLUSH_SYNC_COUNT, MONITOR_FLUSH_SYNC_PAGES, n_pages); } - os_aio_wait_until_no_pending_writes(); + os_aio_wait_until_no_pending_writes(false); mysql_mutex_lock(&buf_pool.flush_list_mutex); } while (buf_pool.get_oldest_modification(sync_lsn) < sync_lsn); } else #endif + { + thd_wait_begin(nullptr, THD_WAIT_DISKIO); + tpool::tpool_wait_begin(); buf_flush_wait(sync_lsn); - - thd_wait_end(nullptr); + tpool::tpool_wait_end(); + thd_wait_end(nullptr); + } } mysql_mutex_unlock(&buf_pool.flush_list_mutex); @@ -2409,7 +2423,7 @@ static void buf_flush_page_cleaner() mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_flush_wait_LRU_batch_end(); mysql_mutex_unlock(&buf_pool.flush_list_mutex); - os_aio_wait_until_no_pending_writes(); + os_aio_wait_until_no_pending_writes(false); } mysql_mutex_lock(&buf_pool.flush_list_mutex); @@ -2459,7 +2473,7 @@ ATTRIBUTE_COLD void buf_flush_buffer_pool() { mysql_mutex_unlock(&buf_pool.flush_list_mutex); buf_flush_list(srv_max_io_capacity); - os_aio_wait_until_no_pending_writes(); + os_aio_wait_until_no_pending_writes(false); mysql_mutex_lock(&buf_pool.flush_list_mutex); service_manager_extend_timeout(INNODB_EXTEND_TIMEOUT_INTERVAL, "Waiting to flush " ULINTPF " pages", @@ -2467,7 +2481,6 @@ ATTRIBUTE_COLD void buf_flush_buffer_pool() } mysql_mutex_unlock(&buf_pool.flush_list_mutex); - ut_ad(!os_aio_pending_writes()); ut_ad(!os_aio_pending_reads()); } diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index d1c7f0ba666..724aa641f12 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -448,15 +448,15 @@ got_block: mysql_mutex_unlock(&buf_pool.mutex); mysql_mutex_lock(&buf_pool.flush_list_mutex); const auto n_flush = buf_pool.n_flush(); + if (!buf_pool.try_LRU_scan) { + buf_pool.page_cleaner_wakeup(true); + } mysql_mutex_unlock(&buf_pool.flush_list_mutex); mysql_mutex_lock(&buf_pool.mutex); if (!n_flush) { goto not_found; } if (!buf_pool.try_LRU_scan) { - mysql_mutex_lock(&buf_pool.flush_list_mutex); - buf_pool.page_cleaner_wakeup(true); - mysql_mutex_unlock(&buf_pool.flush_list_mutex); my_cond_wait(&buf_pool.done_free, &buf_pool.mutex.m_mutex); } diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index 8631a03644f..39ecd5de27f 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -685,7 +685,7 @@ void buf_read_recv_pages(uint32_t space_id, st_::span<uint32_t> page_nos) } if (os_aio_pending_reads() >= limit) { - os_aio_wait_until_no_pending_reads(); + os_aio_wait_until_no_pending_reads(false); } space->reacquire(); diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 24319bcb8a6..d6a4d5c17f4 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -295,6 +295,8 @@ fil_node_t* fil_space_t::add(const char* name, pfs_os_file_t handle, uint32_t size, bool is_raw, bool atomic_write, uint32_t max_pages) { + mysql_mutex_assert_owner(&fil_system.mutex); + fil_node_t* node; ut_ad(name != NULL); @@ -319,7 +321,6 @@ fil_node_t* fil_space_t::add(const char* name, pfs_os_file_t handle, node->atomic_write = atomic_write; - mysql_mutex_lock(&fil_system.mutex); this->size += size; UT_LIST_ADD_LAST(chain, node); if (node->is_open()) { @@ -330,7 +331,6 @@ fil_node_t* fil_space_t::add(const char* name, pfs_os_file_t handle, release(); } } - mysql_mutex_unlock(&fil_system.mutex); return node; } @@ -804,8 +804,17 @@ pfs_os_file_t fil_system_t::detach(fil_space_t *space, bool detach_handle) space_list_t::iterator s= space_list_t::iterator(space); if (space_list_last_opened == space) { - space_list_t::iterator prev= s; - space_list_last_opened= &*--prev; + if (s == space_list.begin()) + { + ut_ad(srv_operation > SRV_OPERATION_EXPORT_RESTORED || + srv_shutdown_state > SRV_SHUTDOWN_NONE); + space_list_last_opened= nullptr; + } + else + { + space_list_t::iterator prev= s; + space_list_last_opened= &*--prev; + } } space_list.erase(s); } @@ -939,6 +948,7 @@ fil_space_t *fil_space_t::create(uint32_t id, uint32_t flags, { fil_space_t* space; + mysql_mutex_assert_owner(&fil_system.mutex); ut_ad(fil_system.is_initialised()); ut_ad(fil_space_t::is_valid_flags(flags & ~FSP_FLAGS_MEM_MASK, id)); ut_ad(srv_page_size == UNIV_PAGE_SIZE_ORIG || flags != 0); @@ -971,8 +981,6 @@ fil_space_t *fil_space_t::create(uint32_t id, uint32_t flags, space->latch.SRW_LOCK_INIT(fil_space_latch_key); - mysql_mutex_lock(&fil_system.mutex); - if (const fil_space_t *old_space = fil_space_get_by_id(id)) { ib::error() << "Trying to add tablespace with id " << id << " to the cache, but tablespace '" @@ -980,7 +988,6 @@ fil_space_t *fil_space_t::create(uint32_t id, uint32_t flags, ? old_space->chain.start->name : "") << "' already exists in the cache!"; - mysql_mutex_unlock(&fil_system.mutex); space->~fil_space_t(); ut_free(space); return(NULL); @@ -1027,12 +1034,12 @@ fil_space_t *fil_space_t::create(uint32_t id, uint32_t flags, if (rotate) { fil_system.default_encrypt_tables.push_back(*space); space->is_in_default_encrypt = true; - } - mysql_mutex_unlock(&fil_system.mutex); - - if (rotate && srv_n_fil_crypt_threads_started) { - fil_crypt_threads_signal(); + if (srv_n_fil_crypt_threads_started) { + mysql_mutex_unlock(&fil_system.mutex); + fil_crypt_threads_signal(); + mysql_mutex_lock(&fil_system.mutex); + } } return(space); @@ -1315,9 +1322,9 @@ void fil_system_t::close() void fil_system_t::add_opened_last_to_space_list(fil_space_t *space) { if (UNIV_LIKELY(space_list_last_opened != nullptr)) - space_list.insert(space_list_t::iterator(space_list_last_opened), *space); + space_list.insert(++space_list_t::iterator(space_list_last_opened), *space); else - space_list.push_back(*space); + space_list.push_front(*space); space_list_last_opened= space; } @@ -1629,9 +1636,7 @@ pfs_os_file_t fil_delete_tablespace(uint32_t id) mtr_t mtr; mtr.start(); mtr.log_file_op(FILE_DELETE, id, space->chain.start->name); - handle= space->chain.start->handle; - mtr.commit_file(*space, nullptr); - + mtr.commit_file(*space, nullptr, &handle); fil_space_free_low(space); } @@ -1945,16 +1950,20 @@ err_exit: DBUG_EXECUTE_IF("checkpoint_after_file_create", log_make_checkpoint();); + mysql_mutex_lock(&fil_system.mutex); if (fil_space_t* space = fil_space_t::create(space_id, flags, FIL_TYPE_TABLESPACE, crypt_data, mode, true)) { fil_node_t* node = space->add(path, file, size, false, true); + mysql_mutex_unlock(&fil_system.mutex); IF_WIN(node->find_metadata(), node->find_metadata(file, true)); mtr.start(); mtr.set_named_space(space); ut_a(fsp_header_init(space, size, &mtr) == DB_SUCCESS); mtr.commit(); return space; + } else { + mysql_mutex_unlock(&fil_system.mutex); } if (space_name.data()) { @@ -2214,8 +2223,10 @@ skip_validate: first_page) : NULL; + mysql_mutex_lock(&fil_system.mutex); space = fil_space_t::create(id, flags, purpose, crypt_data); if (!space) { + mysql_mutex_unlock(&fil_system.mutex); goto error; } @@ -2225,6 +2236,7 @@ skip_validate: space->add( df_remote.is_open() ? df_remote.filepath() : df_default.filepath(), OS_FILE_CLOSED, 0, false, true); + mysql_mutex_unlock(&fil_system.mutex); if (must_validate && !srv_read_only_mode) { df_remote.close(); @@ -2510,10 +2522,13 @@ tablespace_check: return FIL_LOAD_INVALID; } + mysql_mutex_lock(&fil_system.mutex); + space = fil_space_t::create( space_id, flags, FIL_TYPE_TABLESPACE, crypt_data); if (space == NULL) { + mysql_mutex_unlock(&fil_system.mutex); return(FIL_LOAD_INVALID); } @@ -2525,6 +2540,7 @@ tablespace_check: let fil_node_open() do that task. */ space->add(file.filepath(), OS_FILE_CLOSED, 0, false, false); + mysql_mutex_unlock(&fil_system.mutex); return(FIL_LOAD_OK); } diff --git a/storage/innobase/fsp/fsp0space.cc b/storage/innobase/fsp/fsp0space.cc index 6bdf9fcc4d8..c2152b08590 100644 --- a/storage/innobase/fsp/fsp0space.cc +++ b/storage/innobase/fsp/fsp0space.cc @@ -88,25 +88,25 @@ Tablespace::open_or_create(bool is_temp) ut_ad(!m_files.empty()); for (iterator it = begin(); it != end(); ++it) { - if (it->m_exists) { err = it->open_or_create( m_ignore_read_only ? false : srv_read_only_mode); + if (err != DB_SUCCESS) { + return err; + } } else { err = it->open_or_create( m_ignore_read_only ? false : srv_read_only_mode); - /* Set the correct open flags now that we have - successfully created the file. */ - if (err == DB_SUCCESS) { - file_found(*it); + if (err != DB_SUCCESS) { + return err; } - } - if (err != DB_SUCCESS) { - break; + /* Set the correct open flags now that we have + successfully created the file. */ + file_found(*it); } /* We can close the handle now and open the tablespace @@ -130,20 +130,22 @@ Tablespace::open_or_create(bool is_temp) fsp_flags = FSP_FLAGS_PAGE_SSIZE(); } + mysql_mutex_lock(&fil_system.mutex); space = fil_space_t::create( m_space_id, fsp_flags, is_temp ? FIL_TYPE_TEMPORARY : FIL_TYPE_TABLESPACE, NULL); if (!space) { + mysql_mutex_unlock(&fil_system.mutex); return DB_ERROR; } + } else { + mysql_mutex_lock(&fil_system.mutex); } - - ut_a(fil_validate()); - space->add(it->m_filepath, OS_FILE_CLOSED, it->m_size, false, true); + mysql_mutex_unlock(&fil_system.mutex); } return(err); diff --git a/storage/innobase/fsp/fsp0sysspace.cc b/storage/innobase/fsp/fsp0sysspace.cc index 09eea04b655..9aef6f389ef 100644 --- a/storage/innobase/fsp/fsp0sysspace.cc +++ b/storage/innobase/fsp/fsp0sysspace.cc @@ -920,6 +920,7 @@ SysTablespace::open_or_create( /* Close the curent handles, add space and file info to the fil_system cache and the Data Dictionary, and re-open them in file_system cache so that they stay open until shutdown. */ + mysql_mutex_lock(&fil_system.mutex); ulint node_counter = 0; for (files_t::iterator it = begin; it != end; ++it) { it->close(); @@ -933,7 +934,8 @@ SysTablespace::open_or_create( FIL_TYPE_TEMPORARY, NULL); ut_ad(space == fil_system.temp_space); if (!space) { - return DB_ERROR; + err = DB_ERROR; + break; } ut_ad(!space->is_compressed()); ut_ad(space->full_crc32()); @@ -944,12 +946,11 @@ SysTablespace::open_or_create( FIL_TYPE_TABLESPACE, NULL); ut_ad(space == fil_system.sys_space); if (!space) { - return DB_ERROR; + err = DB_ERROR; + break; } } - ut_a(fil_validate()); - uint32_t max_size = (++node_counter == m_files.size() ? (m_last_file_size_max == 0 ? UINT32_MAX @@ -960,6 +961,7 @@ SysTablespace::open_or_create( it->m_type != SRV_NOT_RAW, true, max_size); } + mysql_mutex_unlock(&fil_system.mutex); return(err); } diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 6fdb0e11182..8886cfa6d9f 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -38,22 +38,6 @@ Full Text Search interface #include "dict0stats.h" #include "btr0pcur.h" -/** The SYNC state of the cache. There is one instance of this struct -associated with each ADD thread. */ -struct fts_sync_t { - /** Transaction used for SYNCing the cache to disk */ - trx_t *trx; - /** Table with FTS index(es) */ - dict_table_t *table; - /** Max size in bytes of the cache */ - ulint max_cache_size; - /** The doc id at which the cache was noted as being - full, we use this to set the upper_limit field */ - doc_id_t max_doc_id; - /** SYNC start time; only used if fts_enable_diag_print */ - time_t start_time; -}; - static const ulint FTS_MAX_ID_LEN = 32; /** Column name from the FTS config table */ @@ -201,8 +185,15 @@ struct fts_tokenize_param_t { /** Run SYNC on the table, i.e., write out data from the cache to the FTS auxiliary INDEX table and clear the cache at the end. @param[in,out] sync sync state +@param[in] unlock_cache whether unlock cache lock when write node +@param[in] wait whether wait when a sync is in progress @return DB_SUCCESS if all OK */ -static dberr_t fts_sync(fts_sync_t *sync); +static +dberr_t +fts_sync( + fts_sync_t* sync, + bool unlock_cache, + bool wait); /****************************************************************//** Release all resources help by the words rb tree e.g., the node ilist. */ @@ -275,6 +266,7 @@ fts_cache_destroy(fts_cache_t* cache) mysql_mutex_destroy(&cache->init_lock); mysql_mutex_destroy(&cache->deleted_lock); mysql_mutex_destroy(&cache->doc_id_lock); + pthread_cond_destroy(&cache->sync->cond); if (cache->stopword_info.cached_stopword) { rbt_free(cache->stopword_info.cached_stopword); @@ -574,6 +566,7 @@ fts_index_cache_init( for (i = 0; i < FTS_NUM_AUX_INDEX; ++i) { ut_a(index_cache->ins_graph[i] == NULL); + ut_a(index_cache->sel_graph[i] == NULL); } } @@ -643,6 +636,7 @@ fts_cache_create( mem_heap_zalloc(heap, sizeof(fts_sync_t))); cache->sync->table = table; + pthread_cond_init(&cache->sync->cond, nullptr); /* Create the index cache vector that will hold the inverted indexes. */ cache->indexes = ib_vector_create( @@ -968,6 +962,10 @@ fts_cache_index_cache_create( mem_heap_zalloc(static_cast<mem_heap_t*>( cache->self_heap->arg), n_bytes)); + index_cache->sel_graph = static_cast<que_t**>( + mem_heap_zalloc(static_cast<mem_heap_t*>( + cache->self_heap->arg), n_bytes)); + fts_index_cache_init(cache->sync_heap, index_cache); if (cache->get_docs) { @@ -1041,6 +1039,13 @@ fts_cache_clear( index_cache->ins_graph[j] = NULL; } + + if (index_cache->sel_graph[j] != NULL) { + + que_graph_free(index_cache->sel_graph[j]); + + index_cache->sel_graph[j] = NULL; + } } index_cache->doc_stats = NULL; @@ -1333,7 +1338,8 @@ fts_cache_add_doc( ib_vector_last(word->nodes)); } - if (!fts_node || fts_node->ilist_size > FTS_ILIST_MAX_SIZE + if (fts_node == NULL || fts_node->synced + || fts_node->ilist_size > FTS_ILIST_MAX_SIZE || doc_id < fts_node->last_doc_id) { fts_node = static_cast<fts_node_t*>( @@ -3320,7 +3326,7 @@ fts_add_doc_from_tuple( if (cache->total_size > fts_max_cache_size / 5 || fts_need_sync) { - fts_sync(cache->sync); + fts_sync(cache->sync, true, false); } mtr_start(&mtr); @@ -3356,7 +3362,7 @@ fts_add_doc_by_id( dict_index_t* fts_id_index; ibool is_id_cluster; fts_cache_t* cache = ftt->table->fts->cache; - bool need_sync= false; + ut_ad(cache->get_docs); /* If Doc ID has been supplied by the user, then the table @@ -3496,32 +3502,44 @@ fts_add_doc_by_id( get_doc->index_cache, doc_id, doc.tokens); - /** FTS cache sync should happen - frequently. Because user thread - shouldn't hold the cache lock for - longer time. So cache should sync - whenever cache size exceeds 512 KB */ - need_sync = - cache->total_size > 512*1024; + bool need_sync = !cache->sync->in_progress + && (fts_need_sync + || (cache->total_size + - cache->total_size_at_sync) + > fts_max_cache_size / 10); + if (need_sync) { + cache->total_size_at_sync = + cache->total_size; + } mysql_mutex_unlock(&table->fts->cache->lock); DBUG_EXECUTE_IF( "fts_instrument_sync", - fts_sync_table(table); + fts_optimize_request_sync_table(table); + mysql_mutex_lock(&cache->lock); + if (cache->sync->in_progress) + my_cond_wait( + &cache->sync->cond, + &cache->lock.m_mutex); + mysql_mutex_unlock(&cache->lock); ); DBUG_EXECUTE_IF( "fts_instrument_sync_debug", - fts_sync(cache->sync); + fts_sync(cache->sync, true, true); ); DEBUG_SYNC_C("fts_instrument_sync_request"); DBUG_EXECUTE_IF( "fts_instrument_sync_request", - need_sync= true; + fts_optimize_request_sync_table(table); ); + if (need_sync) { + fts_optimize_request_sync_table(table); + } + mtr_start(&mtr); if (i < num_idx - 1) { @@ -3547,10 +3565,6 @@ func_exit: ut_free(pcur.old_rec_buf); mem_heap_free(heap); - - if (need_sync) { - fts_sync_table(table); - } } @@ -3910,13 +3924,15 @@ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_sync_write_words( trx_t* trx, - fts_index_cache_t* index_cache) + fts_index_cache_t* index_cache, + bool unlock_cache) { fts_table_t fts_table; ulint n_nodes = 0; ulint n_words = 0; const ib_rbt_node_t* rbt_node; dberr_t error = DB_SUCCESS; + ibool print_error = FALSE; dict_table_t* table = index_cache->index->table; FTS_INIT_INDEX_TABLE( @@ -3947,36 +3963,53 @@ fts_sync_write_words( fts_table.suffix = fts_get_suffix(selected); + /* We iterate over all the nodes even if there was an error */ for (i = 0; i < ib_vector_size(word->nodes); ++i) { fts_node_t* fts_node = static_cast<fts_node_t*>( ib_vector_get(word->nodes, i)); - error = fts_write_node( - trx, &index_cache->ins_graph[selected], - &fts_table, &word->text, fts_node); + if (fts_node->synced) { + continue; + } else { + fts_node->synced = true; + } + + /*FIXME: we need to handle the error properly. */ + if (error == DB_SUCCESS) { + if (unlock_cache) { + mysql_mutex_unlock( + &table->fts->cache->lock); + } + + error = fts_write_node( + trx, + &index_cache->ins_graph[selected], + &fts_table, &word->text, fts_node); - DEBUG_SYNC_C("fts_write_node"); - DBUG_EXECUTE_IF("fts_write_node_crash", + DEBUG_SYNC_C("fts_write_node"); + DBUG_EXECUTE_IF("fts_write_node_crash", DBUG_SUICIDE();); - DBUG_EXECUTE_IF("fts_instrument_sync_sleep", + DBUG_EXECUTE_IF( + "fts_instrument_sync_sleep", std::this_thread::sleep_for( std::chrono::seconds(1));); - if (error != DB_SUCCESS) { - goto err_exit; + if (unlock_cache) { + mysql_mutex_lock( + &table->fts->cache->lock); + } } } n_nodes += ib_vector_size(word->nodes); - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { -err_exit: + if (UNIV_UNLIKELY(error != DB_SUCCESS) && !print_error) { ib::error() << "(" << error << ") writing" " word node to FTS auxiliary index table " << table->name; - break; + print_error = TRUE; } } @@ -4035,44 +4068,58 @@ fts_sync_index( ut_ad(rbt_validate(index_cache->words)); - return(fts_sync_write_words(trx, index_cache)); + return(fts_sync_write_words(trx, index_cache, sync->unlock_cache)); } -/** Rollback a sync operation -@param[in,out] sync sync state */ +/** Check if index cache has been synced completely +@param[in,out] index_cache index cache +@return true if index is synced, otherwise false. */ static -void -fts_sync_rollback( - fts_sync_t* sync) +bool +fts_sync_index_check( + fts_index_cache_t* index_cache) { - trx_t* trx = sync->trx; - fts_cache_t* cache = sync->table->fts->cache; - - for (ulint i = 0; i < ib_vector_size(cache->indexes); ++i) { - ulint j; - fts_index_cache_t* index_cache; - - index_cache = static_cast<fts_index_cache_t*>( - ib_vector_get(cache->indexes, i)); + const ib_rbt_node_t* rbt_node; - for (j = 0; fts_index_selector[j].value; ++j) { + for (rbt_node = rbt_first(index_cache->words); + rbt_node != NULL; + rbt_node = rbt_next(index_cache->words, rbt_node)) { - if (index_cache->ins_graph[j] != NULL) { + fts_tokenizer_word_t* word; + word = rbt_value(fts_tokenizer_word_t, rbt_node); - que_graph_free(index_cache->ins_graph[j]); + fts_node_t* fts_node; + fts_node = static_cast<fts_node_t*>(ib_vector_last(word->nodes)); - index_cache->ins_graph[j] = NULL; - } + if (!fts_node->synced) { + return(false); } } - mysql_mutex_unlock(&cache->lock); + return(true); +} - fts_sql_rollback(trx); +/** Reset synced flag in index cache when rollback +@param[in,out] index_cache index cache */ +static +void +fts_sync_index_reset( + fts_index_cache_t* index_cache) +{ + const ib_rbt_node_t* rbt_node; - /* Avoid assertion in trx_t::free(). */ - trx->dict_operation_lock_mode = false; - trx->free(); + for (rbt_node = rbt_first(index_cache->words); + rbt_node != NULL; + rbt_node = rbt_next(index_cache->words, rbt_node)) { + + fts_tokenizer_word_t* word; + word = rbt_value(fts_tokenizer_word_t, rbt_node); + + fts_node_t* fts_node; + fts_node = static_cast<fts_node_t*>(ib_vector_last(word->nodes)); + + fts_node->synced = false; + } } /** Commit the SYNC, change state of processed doc ids etc. @@ -4105,20 +4152,19 @@ fts_sync_commit( sync, cache->deleted_doc_ids); } + /* We need to do this within the deleted lock since fts_delete() can + attempt to add a deleted doc id to the cache deleted id array. */ + fts_cache_clear(cache); + DEBUG_SYNC_C("fts_deleted_doc_ids_clear"); + fts_cache_init(cache); + mysql_mutex_unlock(&cache->lock); + if (UNIV_LIKELY(error == DB_SUCCESS)) { - /* We need to do this within the deleted lock - since fts_delete() can attempt to add a deleted - doc id to the cache deleted id array. */ - fts_cache_clear(cache); - DEBUG_SYNC_C("fts_deleted_doc_ids_clear"); - fts_cache_init(cache); - mysql_mutex_unlock(&cache->lock); fts_sql_commit(trx); } else { + fts_sql_rollback(trx); ib::error() << "(" << error << ") during SYNC of " "table " << sync->table->name; - fts_sync_rollback(sync); - return error; } if (UNIV_UNLIKELY(fts_enable_diag_print) && elapsed_time) { @@ -4138,13 +4184,66 @@ fts_sync_commit( return(error); } +/** Rollback a sync operation +@param[in,out] sync sync state */ +static +void +fts_sync_rollback( + fts_sync_t* sync) +{ + trx_t* trx = sync->trx; + fts_cache_t* cache = sync->table->fts->cache; + + for (ulint i = 0; i < ib_vector_size(cache->indexes); ++i) { + ulint j; + fts_index_cache_t* index_cache; + + index_cache = static_cast<fts_index_cache_t*>( + ib_vector_get(cache->indexes, i)); + + /* Reset synced flag so nodes will not be skipped + in the next sync, see fts_sync_write_words(). */ + fts_sync_index_reset(index_cache); + + for (j = 0; fts_index_selector[j].value; ++j) { + + if (index_cache->ins_graph[j] != NULL) { + + que_graph_free(index_cache->ins_graph[j]); + + index_cache->ins_graph[j] = NULL; + } + + if (index_cache->sel_graph[j] != NULL) { + + que_graph_free(index_cache->sel_graph[j]); + + index_cache->sel_graph[j] = NULL; + } + } + } + + mysql_mutex_unlock(&cache->lock); + + fts_sql_rollback(trx); + + /* Avoid assertion in trx_t::free(). */ + trx->dict_operation_lock_mode = false; + trx->free(); +} + /** Run SYNC on the table, i.e., write out data from the cache to the FTS auxiliary INDEX table and clear the cache at the end. @param[in,out] sync sync state @param[in] unlock_cache whether unlock cache lock when write node @param[in] wait whether wait when a sync is in progress @return DB_SUCCESS if all OK */ -static dberr_t fts_sync(fts_sync_t *sync) +static +dberr_t +fts_sync( + fts_sync_t* sync, + bool unlock_cache, + bool wait) { if (srv_read_only_mode) { return DB_READ_ONLY; @@ -4155,13 +4254,33 @@ static dberr_t fts_sync(fts_sync_t *sync) fts_cache_t* cache = sync->table->fts->cache; mysql_mutex_lock(&cache->lock); + + /* Check if cache is being synced. + Note: we release cache lock in fts_sync_write_words() to + avoid long wait for the lock by other threads. */ + if (sync->in_progress) { + if (!wait) { + mysql_mutex_unlock(&cache->lock); + return(DB_SUCCESS); + } + do { + my_cond_wait(&sync->cond, &cache->lock.m_mutex); + } while (sync->in_progress); + } + + sync->unlock_cache = unlock_cache; + sync->in_progress = true; + DEBUG_SYNC_C("fts_sync_begin"); fts_sync_begin(sync); +begin_sync: const size_t fts_cache_size= fts_max_cache_size; if (cache->total_size > fts_cache_size) { /* Avoid the case: sync never finish when insert/update keeps comming. */ + ut_ad(sync->unlock_cache); + sync->unlock_cache = false; ib::warn() << "Total InnoDB FTS size " << cache->total_size << " for the table " << cache->sync->table->name @@ -4185,23 +4304,52 @@ static dberr_t fts_sync(fts_sync_t *sync) error = fts_sync_index(sync, index_cache); if (error != DB_SUCCESS) { - goto err_exit; + goto end_sync; + } + + if (!sync->unlock_cache + && cache->total_size < fts_max_cache_size) { + /* Reset the unlock cache if the value + is less than innodb_ft_cache_size */ + sync->unlock_cache = true; } } DBUG_EXECUTE_IF("fts_instrument_sync_interrupted", + sync->interrupted = true; error = DB_INTERRUPTED; - goto err_exit; + goto end_sync; ); - if (error == DB_SUCCESS) { + /* Make sure all the caches are synced. */ + for (i = 0; i < ib_vector_size(cache->indexes); ++i) { + fts_index_cache_t* index_cache; + + index_cache = static_cast<fts_index_cache_t*>( + ib_vector_get(cache->indexes, i)); + + if (index_cache->index->to_be_dropped + || fts_sync_index_check(index_cache)) { + continue; + } + + goto begin_sync; + } + +end_sync: + if (error == DB_SUCCESS && !sync->interrupted) { error = fts_sync_commit(sync); } else { -err_exit: fts_sync_rollback(sync); - return error; } + mysql_mutex_lock(&cache->lock); + ut_ad(sync->in_progress); + sync->interrupted = false; + sync->in_progress = false; + pthread_cond_broadcast(&sync->cond); + mysql_mutex_unlock(&cache->lock); + /* We need to check whether an optimize is required, for that we make copies of the two variables that control the trigger. These variables can change behind our back and we don't want to hold the @@ -4213,7 +4361,6 @@ err_exit: mysql_mutex_unlock(&cache->deleted_lock); - DEBUG_SYNC_C("fts_sync_end"); return(error); } @@ -4222,12 +4369,12 @@ FTS auxiliary INDEX table and clear the cache at the end. @param[in,out] table fts table @param[in] wait whether wait for existing sync to finish @return DB_SUCCESS on success, error code on failure. */ -dberr_t fts_sync_table(dict_table_t* table) +dberr_t fts_sync_table(dict_table_t* table, bool wait) { ut_ad(table->fts); return table->space && !table->corrupted && table->fts->cache - ? fts_sync(table->fts->cache->sync) + ? fts_sync(table->fts->cache->sync, !wait, wait) : DB_SUCCESS; } diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index 7c40a25e6e7..fe31767d901 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -83,8 +83,9 @@ enum fts_msg_type_t { FTS_MSG_ADD_TABLE, /*!< Add table to the optimize thread's work queue */ - FTS_MSG_DEL_TABLE /*!< Remove a table from the optimize + FTS_MSG_DEL_TABLE, /*!< Remove a table from the optimize threads work queue */ + FTS_MSG_SYNC_TABLE /*!< Sync fts cache of a table */ }; /** Compressed list of words that have been read from FTS INDEX @@ -2624,6 +2625,36 @@ fts_optimize_remove_table( mysql_mutex_unlock(&fts_optimize_wq->mutex); } +/** Send sync fts cache for the table. +@param[in] table table to sync */ +void +fts_optimize_request_sync_table( + dict_table_t* table) +{ + /* if the optimize system not yet initialized, return */ + if (!fts_optimize_wq) { + return; + } + + mysql_mutex_lock(&fts_optimize_wq->mutex); + + /* FTS optimizer thread is already exited */ + if (fts_opt_start_shutdown) { + ib::info() << "Try to sync table " << table->name + << " after FTS optimize thread exiting."; + } else if (table->fts->sync_message) { + /* If the table already has SYNC message in + fts_optimize_wq queue then ignore it */ + } else { + add_msg(fts_optimize_create_msg(FTS_MSG_SYNC_TABLE, table)); + table->fts->sync_message = true; + DBUG_EXECUTE_IF("fts_optimize_wq_count_check", + DBUG_ASSERT(fts_optimize_wq->length <= 1000);); + } + + mysql_mutex_unlock(&fts_optimize_wq->mutex); +} + /** Add a table to fts_slots if it doesn't already exist. */ static bool fts_optimize_new_table(dict_table_t* table) { @@ -2765,8 +2796,7 @@ static void fts_optimize_sync_table(dict_table_t *table, if (sync_table->fts && sync_table->fts->cache && sync_table->is_accessible()) { - fts_sync_table(sync_table); - + fts_sync_table(sync_table, false); if (process_message) { mysql_mutex_lock(&fts_optimize_wq->mutex); @@ -2866,6 +2896,24 @@ retry_later: --n_tables; } break; + + case FTS_MSG_SYNC_TABLE: + if (UNIV_UNLIKELY(wsrep_sst_disable_writes)) { + add_msg(msg); + goto retry_later; + } + + DBUG_EXECUTE_IF( + "fts_instrument_msg_sync_sleep", + std::this_thread::sleep_for( + std::chrono::milliseconds( + 300));); + + fts_optimize_sync_table( + static_cast<dict_table_t*>(msg->ptr), + true); + break; + default: ut_error; } @@ -2998,7 +3046,7 @@ void fts_sync_during_ddl(dict_table_t* table) if (!sync_message) return; - fts_sync_table(table); + fts_sync_table(table, false); mysql_mutex_lock(&fts_optimize_wq->mutex); table->fts->sync_message = false; diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 5825b4ae78a..20decb897ba 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -11604,8 +11604,12 @@ foreign_fail: ut_d(dict_table_check_for_dup_indexes( ctx->new_table, CHECK_ABORTED_OK)); - ut_ad(!ctx->new_table->fts - || fts_check_cached_index(ctx->new_table)); +#ifdef UNIV_DEBUG + if (!(ctx->new_table->fts != NULL + && ctx->new_table->fts->cache->sync->in_progress)) { + ut_a(fts_check_cached_index(ctx->new_table)); + } +#endif } unlock_and_close_files(deleted, trx); diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 008cb3d1261..73e80d77b56 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -1482,6 +1482,7 @@ public: if (space_list_last_opened == space) { + ut_ad(s != space_list.begin()); space_list_t::iterator prev= s; space_list_last_opened= &*--prev; } diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 720fe7f25b9..c0151b44063 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -648,6 +648,12 @@ fts_optimize_remove_table( void fts_optimize_shutdown(); +/** Send sync fts cache for the table. +@param[in] table table to sync */ +void +fts_optimize_request_sync_table( + dict_table_t* table); + /**********************************************************************//** Take a FTS savepoint. */ void @@ -702,8 +708,9 @@ fts_savepoint_rollback_last_stmt( /** Run SYNC on the table, i.e., write out data from the cache to the FTS auxiliary INDEX table and clear the cache at the end. @param[in,out] table fts table +@param[in] wait whether to wait for existing sync to finish @return DB_SUCCESS on success, error code on failure. */ -dberr_t fts_sync_table(dict_table_t* table); +dberr_t fts_sync_table(dict_table_t* table, bool wait = true); /****************************************************************//** Create an FTS index cache. */ diff --git a/storage/innobase/include/fts0types.h b/storage/innobase/include/fts0types.h index 04e99d595c5..fb278d543c4 100644 --- a/storage/innobase/include/fts0types.h +++ b/storage/innobase/include/fts0types.h @@ -75,6 +75,7 @@ struct fts_index_cache_t { que_t** ins_graph; /*!< Insert query graphs */ + que_t** sel_graph; /*!< Select query graphs */ CHARSET_INFO* charset; /*!< charset */ }; @@ -86,7 +87,35 @@ struct fts_stopword_t { CHARSET_INFO* charset; /*!< charset for stopword */ }; -struct fts_sync_t; +/** The SYNC state of the cache. There is one instance of this struct +associated with each ADD thread. */ +struct fts_sync_t { + trx_t* trx; /*!< The transaction used for SYNCing + the cache to disk */ + dict_table_t* table; /*!< Table with FTS index(es) */ + ulint max_cache_size; /*!< Max size in bytes of the cache */ + ibool cache_full; /*!< flag, when true it indicates that + we need to sync the cache to disk */ + ulint lower_index; /*!< the start index of the doc id + vector from where to start adding + documents to the FTS cache */ + ulint upper_index; /*!< max index of the doc id vector to + add to the FTS cache */ + ibool interrupted; /*!< TRUE if SYNC was interrupted */ + doc_id_t min_doc_id; /*!< The smallest doc id added to the + cache. It should equal to + doc_ids[lower_index] */ + doc_id_t max_doc_id; /*!< The doc id at which the cache was + noted as being full, we use this to + set the upper_limit field */ + time_t start_time; /*!< SYNC start time; only used if + fts_enable_diag_print */ + bool in_progress; /*!< flag whether sync is in progress.*/ + bool unlock_cache; /*!< flag whether unlock cache when + write fts node */ + /** condition variable for in_progress; used with table->fts->cache->lock */ + pthread_cond_t cond; +}; /** The cache for the FTS system. It is a memory-based inverted index that new entries are added to, until it grows over the configured maximum @@ -175,6 +204,7 @@ struct fts_node_t { ulint ilist_size_alloc; /*!< Allocated size of ilist in bytes */ + bool synced; /*!< flag whether the node is synced */ }; /** A tokenizer word. Contains information about one word. */ diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h index b8df6d9f63e..4d7b77b4638 100644 --- a/storage/innobase/include/mtr0mtr.h +++ b/storage/innobase/include/mtr0mtr.h @@ -93,10 +93,16 @@ struct mtr_t { ATTRIBUTE_COLD void commit_shrink(fil_space_t &space); /** Commit a mini-transaction that is deleting or renaming a file. - @param space tablespace that is being renamed or deleted - @param name new file name (nullptr=the file will be deleted) + @param space tablespace that is being renamed or deleted + @param name new file name (nullptr=the file will be deleted) + @param detached_handle if detached_handle != nullptr and if space is detached + during the function execution the file handle if its + node will be set to OS_FILE_CLOSED, and the previous + value of the file handle will be assigned to the + address, pointed by detached_handle. @return whether the operation succeeded */ - ATTRIBUTE_COLD bool commit_file(fil_space_t &space, const char *name); + ATTRIBUTE_COLD bool commit_file(fil_space_t &space, const char *name, + pfs_os_file_t *detached_handle= nullptr); /** Commit a mini-transaction that did not modify any pages, but generated some redo log on a higher level, such as @@ -335,7 +341,7 @@ public: { mtr_memo_slot_t &slot= m_memo[savepoint]; ut_ad(slot.type <= MTR_MEMO_BUF_FIX); - ut_ad(type <= MTR_MEMO_BUF_FIX); + ut_ad(type < MTR_MEMO_S_LOCK); slot.type= type; } diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h index 764071cfab9..13f9d3de3f8 100644 --- a/storage/innobase/include/os0file.h +++ b/storage/innobase/include/os0file.h @@ -1056,11 +1056,13 @@ size_t os_aio_pending_reads_approx(); /** @return number of pending writes */ size_t os_aio_pending_writes(); -/** Wait until there are no pending asynchronous writes. */ -void os_aio_wait_until_no_pending_writes(); +/** Wait until there are no pending asynchronous writes. +@param declare whether the wait will be declared in tpool */ +void os_aio_wait_until_no_pending_writes(bool declare); -/** Wait until all pending asynchronous reads have completed. */ -void os_aio_wait_until_no_pending_reads(); +/** Wait until all pending asynchronous reads have completed. +@param declare whether the wait will be declared in tpool */ +void os_aio_wait_until_no_pending_reads(bool declare); /** Prints info of the aio arrays. @param[in/out] file file where to print */ diff --git a/storage/innobase/include/trx0rseg.h b/storage/innobase/include/trx0rseg.h index 8b7eacbbc18..1d95b7d2e7a 100644 --- a/storage/innobase/include/trx0rseg.h +++ b/storage/innobase/include/trx0rseg.h @@ -129,7 +129,8 @@ public: #endif } /** @return whether the segment is marked for undo truncation */ - bool skip_allocation() const { return ref_load() & SKIP; } + bool skip_allocation() const + { return ref.load(std::memory_order_acquire) & SKIP; } /** Increment the reference count */ void acquire() { ut_d(auto r=) ref.fetch_add(REF); ut_ad(!(r & SKIP)); } diff --git a/storage/innobase/include/trx0undo.h b/storage/innobase/include/trx0undo.h index 3474a903f6c..670fe00c25b 100644 --- a/storage/innobase/include/trx0undo.h +++ b/storage/innobase/include/trx0undo.h @@ -203,16 +203,18 @@ trx_undo_assign(trx_t* trx, dberr_t* err, mtr_t* mtr) MY_ATTRIBUTE((nonnull)); /** Assign an undo log for a transaction. A new undo log is created or a cached undo log reused. +@tparam is_temp whether this is temporary undo log @param[in,out] trx transaction @param[in] rseg rollback segment @param[out] undo the undo log -@param[out] err error code @param[in,out] mtr mini-transaction +@param[out] err error code @return the undo log block -@retval NULL on error */ +@retval nullptr on error */ +template<bool is_temp> buf_block_t* -trx_undo_assign_low(trx_t* trx, trx_rseg_t* rseg, trx_undo_t** undo, - dberr_t* err, mtr_t* mtr) +trx_undo_assign_low(trx_t *trx, trx_rseg_t *rseg, trx_undo_t **undo, + mtr_t *mtr, dberr_t *err) MY_ATTRIBUTE((nonnull, warn_unused_result)); /******************************************************************//** Sets the state of the undo log segment at a transaction finish. diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 88b6bae963f..37a496725fc 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -775,9 +775,10 @@ retry: if (!os_file_status(name->c_str(), &exists, &ftype) || !exists) goto processed; } - create(it, *name, static_cast<uint32_t> - (1U << FSP_FLAGS_FCRC32_POS_MARKER | - FSP_FLAGS_FCRC32_PAGE_SSIZE()), nullptr, 0); + if (create(it, *name, static_cast<uint32_t> + (1U << FSP_FLAGS_FCRC32_POS_MARKER | + FSP_FLAGS_FCRC32_PAGE_SSIZE()), nullptr, 0)) + mysql_mutex_unlock(&fil_system.mutex); } } else @@ -806,7 +807,7 @@ processed: @param flags FSP_SPACE_FLAGS @param crypt_data encryption metadata @param size tablespace size in pages - @return tablespace + @return tablespace; the caller must release fil_system.mutex @retval nullptr if crypt_data is invalid */ static fil_space_t *create(const recv_spaces_t::const_iterator &it, const std::string &name, uint32_t flags, @@ -818,6 +819,7 @@ processed: ut_free(crypt_data); return nullptr; } + mysql_mutex_lock(&fil_system.mutex); fil_space_t *space= fil_space_t::create(it->first, flags, FIL_TYPE_TABLESPACE, crypt_data); ut_ad(space); @@ -878,12 +880,13 @@ processed: space->free_limit= fsp_header_get_field(page, FSP_FREE_LIMIT); space->free_len= flst_get_len(FSP_HEADER_OFFSET + FSP_FREE + page); fil_node_t *node= UT_LIST_GET_FIRST(space->chain); + mysql_mutex_unlock(&fil_system.mutex); if (!space->acquire()) - { + { free_space: fil_space_free(it->first, false); goto next_item; - } + } if (os_file_write(IORequestWrite, node->name, node->handle, page, 0, fil_space_t::physical_size(flags)) != DB_SUCCESS) @@ -953,6 +956,7 @@ bool recv_sys_t::recover_deferred(recv_sys_t::map::iterator &p, space->free_len= flst_get_len(FSP_HEADER_OFFSET + FSP_FREE + page); fil_node_t *node= UT_LIST_GET_FIRST(space->chain); node->deferred= true; + mysql_mutex_unlock(&fil_system.mutex); if (!space->acquire()) goto release_and_fail; fil_names_dirty(space); @@ -976,8 +980,10 @@ bool recv_sys_t::recover_deferred(recv_sys_t::map::iterator &p, uint32_t(file_size / fil_space_t::physical_size(flags)); if (n_pages > size) { + mysql_mutex_lock(&fil_system.mutex); space->size= node->size= n_pages; space->set_committed_size(); + mysql_mutex_unlock(&fil_system.mutex); goto size_set; } } @@ -3582,7 +3588,7 @@ next_free_block: else { mysql_mutex_unlock(&mutex); - os_aio_wait_until_no_pending_reads(); + os_aio_wait_until_no_pending_reads(false); mysql_mutex_lock(&mutex); ut_ad(pages.empty()); } diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index 48ab5c6835e..3be297e9666 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -597,8 +597,14 @@ void mtr_t::commit_shrink(fil_space_t &space) /** Commit a mini-transaction that is deleting or renaming a file. @param space tablespace that is being renamed or deleted @param name new file name (nullptr=the file will be deleted) +@param detached_handle if detached_handle != nullptr and if space is detached + during the function execution the file handle if its + node will be set to OS_FILE_CLOSED, and the previous + value of the file handle will be assigned to the + address, pointed by detached_handle. @return whether the operation succeeded */ -bool mtr_t::commit_file(fil_space_t &space, const char *name) +bool mtr_t::commit_file(fil_space_t &space, const char *name, + pfs_os_file_t *detached_handle) { ut_ad(is_active()); ut_ad(!is_inside_ibuf()); @@ -687,7 +693,9 @@ bool mtr_t::commit_file(fil_space_t &space, const char *name) ut_ad(!space.referenced()); ut_ad(space.is_stopping()); - fil_system.detach(&space, true); + pfs_os_file_t handle = fil_system.detach(&space, true); + if (detached_handle) + *detached_handle = handle; mysql_mutex_unlock(&fil_system.mutex); success= true; diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index d28d7b02c43..aafa4361b0b 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -3437,9 +3437,8 @@ static void io_callback(tpool::aiocb *cb) else { ut_ad(write_slots->contains(cb)); - const IORequest req{request}; + fil_aio_callback(request); write_slots->release(cb); - fil_aio_callback(req); } } @@ -3625,9 +3624,9 @@ void os_aio_free() } /** Wait until there are no pending asynchronous writes. */ -static void os_aio_wait_until_no_pending_writes_low() +static void os_aio_wait_until_no_pending_writes_low(bool declare) { - bool notify_wait = write_slots->pending_io_count() > 0; + const bool notify_wait= declare && write_slots->pending_io_count(); if (notify_wait) tpool::tpool_wait_begin(); @@ -3638,10 +3637,11 @@ static void os_aio_wait_until_no_pending_writes_low() tpool::tpool_wait_end(); } -/** Wait until there are no pending asynchronous writes. */ -void os_aio_wait_until_no_pending_writes() +/** Wait until there are no pending asynchronous writes. +@param declare whether the wait will be declared in tpool */ +void os_aio_wait_until_no_pending_writes(bool declare) { - os_aio_wait_until_no_pending_writes_low(); + os_aio_wait_until_no_pending_writes_low(declare); buf_dblwr.wait_flush_buffered_writes(); } @@ -3669,10 +3669,11 @@ size_t os_aio_pending_writes() return pending; } -/** Wait until all pending asynchronous reads have completed. */ -void os_aio_wait_until_no_pending_reads() +/** Wait until all pending asynchronous reads have completed. +@param declare whether the wait will be declared in tpool */ +void os_aio_wait_until_no_pending_reads(bool declare) { - const auto notify_wait= read_slots->pending_io_count(); + const bool notify_wait= declare && read_slots->pending_io_count(); if (notify_wait) tpool::tpool_wait_begin(); diff --git a/storage/innobase/rem/rem0rec.cc b/storage/innobase/rem/rem0rec.cc index d743ef9aa5c..766f5000a0d 100644 --- a/storage/innobase/rem/rem0rec.cc +++ b/storage/innobase/rem/rem0rec.cc @@ -242,9 +242,9 @@ enum rec_leaf_format { REC_LEAF_INSTANT }; -#if defined __GNUC__ && !defined __clang__ && __GNUC__ < 11 +#if defined __GNUC__ && !defined __clang__ && __GNUC__ < 12 # pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wconversion" /* GCC 5 to 10 need this */ +# pragma GCC diagnostic ignored "-Wconversion" /* GCC 5 to 11 need this */ #endif /** Determine the offset to each field in a leaf-page record in ROW_FORMAT=COMPACT,DYNAMIC,COMPRESSED. diff --git a/storage/innobase/row/row0quiesce.cc b/storage/innobase/row/row0quiesce.cc index a4d634f2d14..eadb30bfcfa 100644 --- a/storage/innobase/row/row0quiesce.cc +++ b/storage/innobase/row/row0quiesce.cc @@ -553,7 +553,7 @@ row_quiesce_table_start( if (!trx_is_interrupted(trx)) { /* Ensure that all asynchronous IO is completed. */ - os_aio_wait_until_no_pending_writes(); + os_aio_wait_until_no_pending_writes(true); table->space->flush<false>(); if (row_quiesce_write_cfg(table, trx->mysql_thd) diff --git a/storage/innobase/srv/srv0mon.cc b/storage/innobase/srv/srv0mon.cc index eca7a45563c..4427168e29d 100644 --- a/storage/innobase/srv/srv0mon.cc +++ b/storage/innobase/srv/srv0mon.cc @@ -686,16 +686,18 @@ static monitor_info_t innodb_counter_info[] = {"trx_rseg_history_len", "transaction", "Length of the TRX_RSEG_HISTORY list", static_cast<monitor_type_t>( - MONITOR_EXISTING | MONITOR_DISPLAY_CURRENT | MONITOR_DEFAULT_ON), + MONITOR_EXISTING | MONITOR_DISPLAY_CURRENT), MONITOR_DEFAULT_START, MONITOR_RSEG_HISTORY_LEN}, {"trx_undo_slots_used", "transaction", "Number of undo slots used", - MONITOR_NONE, + static_cast<monitor_type_t>( + MONITOR_EXISTING | MONITOR_DISPLAY_CURRENT), MONITOR_DEFAULT_START, MONITOR_NUM_UNDO_SLOT_USED}, {"trx_undo_slots_cached", "transaction", "Number of undo slots cached", - MONITOR_NONE, + static_cast<monitor_type_t>( + MONITOR_EXISTING | MONITOR_DISPLAY_CURRENT | MONITOR_DEFAULT_ON), MONITOR_DEFAULT_START, MONITOR_NUM_UNDO_SLOT_CACHED}, {"trx_rseg_current_size", "transaction", @@ -1345,6 +1347,24 @@ TPOOL_SUPPRESS_TSAN static ulint srv_mon_get_rseg_size() return size; } +/** @return number of used undo log slots */ +TPOOL_SUPPRESS_TSAN static ulint srv_mon_get_rseg_used() +{ + ulint size= 0; + for (const auto &rseg : trx_sys.rseg_array) + size+= UT_LIST_GET_LEN(rseg.undo_list); + return size; +} + +/** @return number of cached undo log slots */ +TPOOL_SUPPRESS_TSAN static ulint srv_mon_get_rseg_cached() +{ + ulint size= 0; + for (const auto &rseg : trx_sys.rseg_array) + size+= UT_LIST_GET_LEN(rseg.undo_cached); + return size; +} + /****************************************************************//** This function consolidates some existing server counters used by "system status variables". These existing system variables do not have @@ -1632,7 +1652,12 @@ srv_mon_process_existing_counter( case MONITOR_RSEG_CUR_SIZE: value = srv_mon_get_rseg_size(); break; - + case MONITOR_NUM_UNDO_SLOT_USED: + value = srv_mon_get_rseg_used(); + break; + case MONITOR_NUM_UNDO_SLOT_CACHED: + value = srv_mon_get_rseg_cached(); + break; case MONITOR_OVLD_N_FILE_OPENED: value = fil_system.n_open; break; @@ -1743,7 +1768,6 @@ srv_mon_process_existing_counter( case MONITOR_TIMEOUT: value = lock_sys.timeouts; break; - default: ut_error; } diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index dd5a4d9b3cb..5f45e7cae3b 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -457,14 +457,12 @@ err_exit: fil_set_max_space_id_if_bigger(space_id); + mysql_mutex_lock(&fil_system.mutex); fil_space_t *space= fil_space_t::create(space_id, fsp_flags, FIL_TYPE_TABLESPACE, nullptr, FIL_ENCRYPTION_DEFAULT, true); - ut_a(fil_validate()); - ut_a(space); - + ut_ad(space); fil_node_t *file= space->add(name, fh, 0, false, true); - mysql_mutex_lock(&fil_system.mutex); if (create) { @@ -864,10 +862,10 @@ same_size: ut_ad(flushed_lsn == log_sys.get_lsn()); ut_ad(!os_aio_pending_reads()); - ut_ad(!os_aio_pending_writes()); ut_d(mysql_mutex_lock(&buf_pool.flush_list_mutex)); ut_ad(!buf_pool.get_oldest_modification(0)); ut_d(mysql_mutex_unlock(&buf_pool.flush_list_mutex)); + ut_d(os_aio_wait_until_no_pending_writes(false)); DBUG_RETURN(flushed_lsn); } diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 4c8c5842e13..0e4bfbd15d9 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -275,13 +275,11 @@ trx_purge_add_undo_to_history(const trx_t* trx, trx_undo_t*& undo, mtr_t* mtr) if (undo->state != TRX_UNDO_CACHED) { /* The undo log segment will not be reused */ ut_a(undo->id < TRX_RSEG_N_SLOTS); - compile_time_assert(FIL_NULL == 0xffffffff); + static_assert(FIL_NULL == 0xffffffff, ""); mtr->memset(rseg_header, TRX_RSEG + TRX_RSEG_UNDO_SLOTS + undo->id * TRX_RSEG_SLOT_SIZE, 4, 0xff); - MONITOR_DEC(MONITOR_NUM_UNDO_SLOT_USED); - uint32_t hist_size = mach_read_from_4( TRX_RSEG_HISTORY_SIZE + TRX_RSEG + rseg_header->page.frame); @@ -363,7 +361,6 @@ trx_purge_add_undo_to_history(const trx_t* trx, trx_undo_t*& undo, mtr_t* mtr) if (undo->state == TRX_UNDO_CACHED) { UT_LIST_ADD_FIRST(rseg->undo_cached, undo); - MONITOR_INC(MONITOR_NUM_UNDO_SLOT_CACHED); } else { ut_ad(undo->state == TRX_UNDO_TO_PURGE); ut_free(undo); @@ -385,161 +382,168 @@ static dberr_t trx_purge_remove_log_hdr(buf_block_t *rseg, buf_block_t* log, uint16_t(offset + TRX_UNDO_HISTORY_NODE), mtr); } -MY_ATTRIBUTE((nonnull, warn_unused_result)) -/** Free an undo log segment, and remove the header from the history list. -@param[in,out] mtr mini-transaction -@param[in,out] rseg rollback segment -@param[in] hdr_addr file address of log_hdr -@return error code */ -static dberr_t -trx_purge_free_segment(mtr_t &mtr, trx_rseg_t* rseg, fil_addr_t hdr_addr) +/** Free an undo log segment. +@param block rollback segment header page +@param mtr mini-transaction */ +static void trx_purge_free_segment(buf_block_t *block, mtr_t &mtr) { - mtr.commit(); - log_free_check(); - mtr.start(); - - const page_id_t hdr_page_id{rseg->space->id, hdr_addr.page}; - dberr_t err; - buf_block_t *rseg_hdr= rseg->get(&mtr, &err); - if (!rseg_hdr) - return err; - buf_block_t *block= buf_page_get_gen(hdr_page_id, 0, RW_X_LATCH, - nullptr, BUF_GET_POSSIBLY_FREED, - &mtr, &err); - if (!block) - return err; - - const uint32_t seg_size= - flst_get_len(TRX_UNDO_SEG_HDR + TRX_UNDO_PAGE_LIST + block->page.frame); - - err= trx_purge_remove_log_hdr(rseg_hdr, block, hdr_addr.boffset, &mtr); - if (UNIV_UNLIKELY(err != DB_SUCCESS)) - return err; - - ut_ad(rseg->curr_size >= seg_size); - rseg->curr_size-= seg_size; - rseg->history_size--; - - byte *hist= TRX_RSEG + TRX_RSEG_HISTORY_SIZE + rseg_hdr->page.frame; - ut_ad(mach_read_from_4(hist) >= seg_size); - mtr.write<4>(*rseg_hdr, hist, mach_read_from_4(hist) - seg_size); - while (!fseg_free_step_not_header(TRX_UNDO_SEG_HDR + TRX_UNDO_FSEG_HEADER + block->page.frame, &mtr)) { block->fix(); + const page_id_t id{block->page.id()}; mtr.commit(); /* NOTE: If the server is killed after the log that was produced up to this point was written, and before the log from the mtr.commit() in our caller is written, then the pages belonging to the undo log will become unaccessible garbage. - This does not matters when using multiple innodb_undo_tablespaces; + This does not matter when using multiple innodb_undo_tablespaces; innodb_undo_log_truncate=ON will be able to reclaim the space. */ log_free_check(); mtr.start(); block->page.lock.x_lock(); - mtr.memo_push(block, MTR_MEMO_PAGE_X_MODIFY); + if (UNIV_UNLIKELY(block->page.id() != id)) + { + block->unfix(); + block->page.lock.x_unlock(); + block= buf_page_get_gen(id, 0, RW_X_LATCH, nullptr, BUF_GET, &mtr); + if (!block) + return; + } + else + mtr.memo_push(block, MTR_MEMO_PAGE_X_MODIFY); } while (!fseg_free_step(TRX_UNDO_SEG_HDR + TRX_UNDO_FSEG_HEADER + block->page.frame, &mtr)); - return DB_SUCCESS; } /** Remove unnecessary history data from a rollback segment. @param[in,out] rseg rollback segment @param[in] limit truncate anything before this @return error code */ -static -dberr_t -trx_purge_truncate_rseg_history( - trx_rseg_t& rseg, - const purge_sys_t::iterator& limit) +static dberr_t +trx_purge_truncate_rseg_history(trx_rseg_t& rseg, + const purge_sys_t::iterator& limit) { - fil_addr_t hdr_addr; - mtr_t mtr; + fil_addr_t hdr_addr; + mtr_t mtr; - mtr.start(); + log_free_check(); + mtr.start(); - dberr_t err; - buf_block_t* rseg_hdr = rseg.get(&mtr, &err); - if (!rseg_hdr) { - goto func_exit; - } + dberr_t err; +reget: + buf_block_t *rseg_hdr= rseg.get(&mtr, &err); + if (!rseg_hdr) + { +func_exit: + mtr.commit(); + return err; + } - hdr_addr = flst_get_last(TRX_RSEG + TRX_RSEG_HISTORY - + rseg_hdr->page.frame); - hdr_addr.boffset = static_cast<uint16_t>(hdr_addr.boffset - - TRX_UNDO_HISTORY_NODE); + hdr_addr= flst_get_last(TRX_RSEG + TRX_RSEG_HISTORY + rseg_hdr->page.frame); + hdr_addr.boffset= static_cast<uint16_t>(hdr_addr.boffset - + TRX_UNDO_HISTORY_NODE); loop: - if (hdr_addr.page == FIL_NULL) { -func_exit: - mtr.commit(); - return err; - } + if (hdr_addr.page == FIL_NULL) + goto func_exit; - buf_block_t* block = buf_page_get_gen(page_id_t(rseg.space->id, - hdr_addr.page), - 0, RW_X_LATCH, nullptr, - BUF_GET_POSSIBLY_FREED, - &mtr, &err); - if (!block) { - goto func_exit; - } + buf_block_t *b= + buf_page_get_gen(page_id_t(rseg.space->id, hdr_addr.page), + 0, RW_X_LATCH, nullptr, BUF_GET_POSSIBLY_FREED, + &mtr, &err); + if (!b) + goto func_exit; - const trx_id_t undo_trx_no = mach_read_from_8( - block->page.frame + hdr_addr.boffset + TRX_UNDO_TRX_NO); + const trx_id_t undo_trx_no= + mach_read_from_8(b->page.frame + hdr_addr.boffset + TRX_UNDO_TRX_NO); - if (undo_trx_no >= limit.trx_no) { - if (undo_trx_no == limit.trx_no) { - err = trx_undo_truncate_start( - &rseg, hdr_addr.page, - hdr_addr.boffset, limit.undo_no); - } + if (undo_trx_no >= limit.trx_no) + { + if (undo_trx_no == limit.trx_no) + err = trx_undo_truncate_start(&rseg, hdr_addr.page, + hdr_addr.boffset, limit.undo_no); + goto func_exit; + } - goto func_exit; - } + fil_addr_t prev_hdr_addr= + flst_get_prev_addr(b->page.frame + hdr_addr.boffset + + TRX_UNDO_HISTORY_NODE); + prev_hdr_addr.boffset= static_cast<uint16_t>(prev_hdr_addr.boffset - + TRX_UNDO_HISTORY_NODE); + err= trx_purge_remove_log_hdr(rseg_hdr, b, hdr_addr.boffset, &mtr); + if (UNIV_UNLIKELY(err != DB_SUCCESS)) + goto func_exit; - fil_addr_t prev_hdr_addr = flst_get_prev_addr( - block->page.frame + hdr_addr.boffset + TRX_UNDO_HISTORY_NODE); - prev_hdr_addr.boffset = static_cast<uint16_t>(prev_hdr_addr.boffset - - TRX_UNDO_HISTORY_NODE); - - if (!rseg.is_referenced() - && rseg.needs_purge <= (purge_sys.head.trx_no - ? purge_sys.head.trx_no - : purge_sys.tail.trx_no) - && mach_read_from_2(TRX_UNDO_SEG_HDR + TRX_UNDO_STATE - + block->page.frame) - == TRX_UNDO_TO_PURGE - && !mach_read_from_2(block->page.frame + hdr_addr.boffset - + TRX_UNDO_NEXT_LOG)) { - /* We can free the whole log segment. - This will call trx_purge_remove_log_hdr(). */ - err = trx_purge_free_segment(mtr, &rseg, hdr_addr); - } else { - /* Remove the log hdr from the rseg history. */ - rseg.history_size--; - err = trx_purge_remove_log_hdr(rseg_hdr, block, - hdr_addr.boffset, &mtr); - } + rseg_hdr->fix(); - mtr.commit(); - if (err != DB_SUCCESS) { - return err; - } - mtr.start(); + if (mach_read_from_2(b->page.frame + hdr_addr.boffset + TRX_UNDO_NEXT_LOG) || + rseg.is_referenced() || + rseg.needs_purge > (purge_sys.head.trx_no + ? purge_sys.head.trx_no + : purge_sys.tail.trx_no)) + /* We cannot free the entire undo page. */; + else + { + const uint32_t seg_size= + flst_get_len(TRX_UNDO_SEG_HDR + TRX_UNDO_PAGE_LIST + b->page.frame); + switch (mach_read_from_2(TRX_UNDO_SEG_HDR + TRX_UNDO_STATE + + b->page.frame)) { + case TRX_UNDO_TO_PURGE: + { + byte *hist= TRX_RSEG + TRX_RSEG_HISTORY_SIZE + rseg_hdr->page.frame; + ut_ad(mach_read_from_4(hist) >= seg_size); + mtr.write<4>(*rseg_hdr, hist, mach_read_from_4(hist) - seg_size); + } + free_segment: + ut_ad(rseg.curr_size >= seg_size); + rseg.curr_size-= seg_size; + trx_purge_free_segment(b, mtr); + break; + case TRX_UNDO_CACHED: + /* rseg.undo_cached must point to this page */ + trx_undo_t *undo= UT_LIST_GET_FIRST(rseg.undo_cached); + for (; undo; undo= UT_LIST_GET_NEXT(undo_list, undo)) + if (undo->hdr_page_no == hdr_addr.page) + goto found_cached; + ut_ad("inconsistent undo logs" == 0); + break; + found_cached: + UT_LIST_REMOVE(rseg.undo_cached, undo); + static_assert(FIL_NULL == 0xffffffff, ""); + if (UNIV_UNLIKELY(mach_read_from_4(TRX_RSEG + TRX_RSEG_FORMAT + + rseg_hdr->page.frame))) + trx_rseg_format_upgrade(rseg_hdr, &mtr); + mtr.memset(rseg_hdr, TRX_RSEG + TRX_RSEG_UNDO_SLOTS + + undo->id * TRX_RSEG_SLOT_SIZE, 4, 0xff); + ut_free(undo); + mtr.write<8,mtr_t::MAYBE_NOP>(*rseg_hdr, TRX_RSEG + TRX_RSEG_MAX_TRX_ID + + rseg_hdr->page.frame, + trx_sys.get_max_trx_id() - 1); + goto free_segment; + } + } - hdr_addr = prev_hdr_addr; + hdr_addr= prev_hdr_addr; - rseg_hdr = rseg.get(&mtr, &err); - if (!rseg_hdr) { - goto func_exit; - } + mtr.commit(); + ut_ad(rseg.history_size > 0); + rseg.history_size--; + log_free_check(); + mtr.start(); + rseg_hdr->page.lock.x_lock(); + if (UNIV_UNLIKELY(rseg_hdr->page.id() != rseg.page_id())) + { + rseg_hdr->unfix(); + rseg_hdr->page.lock.x_unlock(); + goto reget; + } + mtr.memo_push(rseg_hdr, MTR_MEMO_PAGE_X_MODIFY); - goto loop; + goto loop; } /** Cleanse purge queue to remove the rseg that reside in undo-tablespace diff --git a/storage/innobase/trx/trx0rec.cc b/storage/innobase/trx/trx0rec.cc index e70516a2d2d..c1a7b08b717 100644 --- a/storage/innobase/trx/trx0rec.cc +++ b/storage/innobase/trx/trx0rec.cc @@ -1875,26 +1875,28 @@ trx_undo_report_row_operation( } mtr_t mtr; + dberr_t err; mtr.start(); trx_undo_t** pundo; trx_rseg_t* rseg; const bool is_temp = index->table->is_temporary(); + buf_block_t* undo_block; if (is_temp) { mtr.set_log_mode(MTR_LOG_NO_REDO); - rseg = trx->get_temp_rseg(); pundo = &trx->rsegs.m_noredo.undo; + undo_block = trx_undo_assign_low<true>(trx, rseg, pundo, + &mtr, &err); } else { ut_ad(!trx->read_only); ut_ad(trx->id); pundo = &trx->rsegs.m_redo.undo; rseg = trx->rsegs.m_redo.rseg; + undo_block = trx_undo_assign_low<false>(trx, rseg, pundo, + &mtr, &err); } - dberr_t err; - buf_block_t* undo_block = trx_undo_assign_low(trx, rseg, pundo, - &err, &mtr); trx_undo_t* undo = *pundo; ut_ad((err == DB_SUCCESS) == (undo_block != NULL)); if (UNIV_UNLIKELY(undo_block == NULL)) { diff --git a/storage/innobase/trx/trx0rseg.cc b/storage/innobase/trx/trx0rseg.cc index d30300d70a7..0d7b96e9280 100644 --- a/storage/innobase/trx/trx0rseg.cc +++ b/storage/innobase/trx/trx0rseg.cc @@ -394,7 +394,6 @@ void trx_rseg_t::reinit(uint32_t page) { next= UT_LIST_GET_NEXT(undo_list, undo); UT_LIST_REMOVE(undo_cached, undo); - MONITOR_DEC(MONITOR_NUM_UNDO_SLOT_CACHED); ut_free(undo); } @@ -403,6 +402,7 @@ void trx_rseg_t::reinit(uint32_t page) last_commit_and_offset= 0; last_page_no= FIL_NULL; curr_size= 1; + ref.store(0, std::memory_order_release); } /** Read the undo log lists. @@ -424,7 +424,6 @@ static dberr_t trx_undo_lists_init(trx_rseg_t *rseg, if (!undo) return DB_CORRUPTION; rseg->curr_size+= undo->size; - MONITOR_INC(MONITOR_NUM_UNDO_SLOT_USED); } } diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index ae3a799622a..43574625ffb 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -811,28 +811,17 @@ static void trx_assign_rseg_low(trx_t *trx) static Atomic_counter<unsigned> rseg_slot; unsigned slot = rseg_slot++ % TRX_SYS_N_RSEGS; ut_d(if (trx_rseg_n_slots_debug) slot = 0); + ut_d(const auto start_scan_slot = slot); + ut_d(bool look_for_rollover = false); trx_rseg_t* rseg; -#ifdef UNIV_DEBUG - ulint start_scan_slot = slot; - bool look_for_rollover = false; -#endif /* UNIV_DEBUG */ - bool allocated; do { for (;;) { rseg = &trx_sys.rseg_array[slot]; - -#ifdef UNIV_DEBUG - /* Ensure that we are not revisiting the same - slot that we have already inspected. */ - if (look_for_rollover) { - ut_ad(start_scan_slot != slot); - } - look_for_rollover = true; -#endif /* UNIV_DEBUG */ - + ut_ad(!look_for_rollover || start_scan_slot != slot); + ut_d(look_for_rollover = true); ut_d(if (!trx_rseg_n_slots_debug)) slot = (slot + 1) % TRX_SYS_N_RSEGS; @@ -1034,7 +1023,13 @@ trx_write_serialisation_history( mtr_t temp_mtr; temp_mtr.start(); temp_mtr.set_log_mode(MTR_LOG_NO_REDO); - trx_undo_set_state_at_finish(undo, &temp_mtr); + buf_block_t* block= buf_page_get(page_id_t(SRV_TMP_SPACE_ID, + undo->hdr_page_no), + 0, RW_X_LATCH, mtr); + ut_a(block); + temp_mtr.write<2>(*block, TRX_UNDO_SEG_HDR + TRX_UNDO_STATE + + block->page.frame, TRX_UNDO_TO_PURGE); + undo->state = TRX_UNDO_TO_PURGE; temp_mtr.commit(); } diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc index 33b1f93ff65..20434d9fb9c 100644 --- a/storage/innobase/trx/trx0undo.cc +++ b/storage/innobase/trx/trx0undo.cc @@ -535,8 +535,6 @@ trx_undo_seg_create(fil_space_t *space, buf_block_t *rseg_hdr, ulint *id, + slot_no * TRX_RSEG_SLOT_SIZE + rseg_hdr->page.frame, block->page.id().page_no()); - MONITOR_INC(MONITOR_NUM_UNDO_SLOT_USED); - *err = DB_SUCCESS; return block; } @@ -996,7 +994,6 @@ static void trx_undo_seg_free(const trx_undo_t *undo) static_assert(FIL_NULL == 0xffffffff, "compatibility"); mtr.memset(rseg_header, TRX_RSEG + TRX_RSEG_UNDO_SLOTS + undo->id * TRX_RSEG_SLOT_SIZE, 4, 0xff); - MONITOR_DEC(MONITOR_NUM_UNDO_SLOT_USED); } } @@ -1155,7 +1152,6 @@ corrupted_type: UT_LIST_ADD_LAST(rseg->undo_list, undo); } else { UT_LIST_ADD_LAST(rseg->undo_cached, undo); - MONITOR_INC(MONITOR_NUM_UNDO_SLOT_CACHED); } mtr.commit(); @@ -1294,27 +1290,25 @@ trx_undo_create(trx_t* trx, trx_rseg_t* rseg, trx_undo_t** undo, @param[in,out] rseg rollback segment @param[out] pundo the undo log memory object @param[in,out] mtr mini-transaction +@param[out] err error code @return the undo log block @retval NULL if none cached */ static buf_block_t* trx_undo_reuse_cached(trx_t* trx, trx_rseg_t* rseg, trx_undo_t** pundo, - mtr_t* mtr) + mtr_t* mtr, dberr_t *err) { - if (rseg->is_persistent()) { - ut_ad(rseg->is_referenced()); - if (rseg->needs_purge <= trx->id) { - /* trx_purge_truncate_history() compares - rseg->needs_purge <= head.trx_no - so we need to compensate for that. - The rseg->needs_purge after crash - recovery would be at least trx->id + 1, - because that is the minimum possible value - assigned by trx_serialise() on commit. */ - rseg->needs_purge = trx->id + 1; - } - } else { - ut_ad(!rseg->is_referenced()); + ut_ad(rseg->is_persistent()); + ut_ad(rseg->is_referenced()); + if (rseg->needs_purge <= trx->id) { + /* trx_purge_truncate_history() compares + rseg->needs_purge <= head.trx_no + so we need to compensate for that. + The rseg->needs_purge after crash + recovery would be at least trx->id + 1, + because that is the minimum possible value + assigned by trx_serialise() on commit. */ + rseg->needs_purge = trx->id + 1; } trx_undo_t* undo = UT_LIST_GET_FIRST(rseg->undo_cached); @@ -1325,15 +1319,15 @@ trx_undo_reuse_cached(trx_t* trx, trx_rseg_t* rseg, trx_undo_t** pundo, ut_ad(undo->size == 1); ut_ad(undo->id < TRX_RSEG_N_SLOTS); - buf_block_t* block = buf_page_get(page_id_t(undo->rseg->space->id, - undo->hdr_page_no), - 0, RW_X_LATCH, mtr); + buf_block_t* block = buf_page_get_gen(page_id_t(undo->rseg->space->id, + undo->hdr_page_no), + 0, RW_X_LATCH, nullptr, BUF_GET, + mtr, err); if (!block) { return NULL; } UT_LIST_REMOVE(rseg->undo_cached, undo); - MONITOR_DEC(MONITOR_NUM_UNDO_SLOT_CACHED); *pundo = undo; @@ -1379,11 +1373,12 @@ trx_undo_assign(trx_t* trx, dberr_t* err, mtr_t* mtr) BUF_GET, mtr, err); } + *err = DB_SUCCESS; trx_rseg_t* rseg = trx->rsegs.m_redo.rseg; rseg->latch.wr_lock(SRW_LOCK_CALL); buf_block_t* block = trx_undo_reuse_cached( - trx, rseg, &trx->rsegs.m_redo.undo, mtr); + trx, rseg, &trx->rsegs.m_redo.undo, mtr, err); if (!block) { block = trx_undo_create(trx, rseg, &trx->rsegs.m_redo.undo, @@ -1392,8 +1387,6 @@ trx_undo_assign(trx_t* trx, dberr_t* err, mtr_t* mtr) if (!block) { goto func_exit; } - } else { - *err = DB_SUCCESS; } UT_LIST_ADD_FIRST(rseg->undo_list, trx->rsegs.m_redo.undo); @@ -1405,18 +1398,20 @@ func_exit: /** Assign an undo log for a transaction. A new undo log is created or a cached undo log reused. +@tparam is_temp whether this is temporary undo log @param[in,out] trx transaction @param[in] rseg rollback segment @param[out] undo the undo log -@param[out] err error code @param[in,out] mtr mini-transaction +@param[out] err error code @return the undo log block -@retval NULL on error */ +@retval nullptr on error */ +template<bool is_temp> buf_block_t* -trx_undo_assign_low(trx_t* trx, trx_rseg_t* rseg, trx_undo_t** undo, - dberr_t* err, mtr_t* mtr) +trx_undo_assign_low(trx_t *trx, trx_rseg_t *rseg, trx_undo_t **undo, + mtr_t *mtr, dberr_t *err) { - ut_d(const bool is_temp = rseg == trx->rsegs.m_noredo.rseg); + ut_ad(is_temp == (rseg == trx->rsegs.m_noredo.rseg)); ut_ad(is_temp || rseg == trx->rsegs.m_redo.rseg); ut_ad(undo == (is_temp ? &trx->rsegs.m_noredo.undo @@ -1436,19 +1431,24 @@ trx_undo_assign_low(trx_t* trx, trx_rseg_t* rseg, trx_undo_t** undo, *err = DB_TOO_MANY_CONCURRENT_TRXS; return NULL; ); + *err = DB_SUCCESS; rseg->latch.wr_lock(SRW_LOCK_CALL); - buf_block_t* block = trx_undo_reuse_cached(trx, rseg, undo, mtr); - - if (!block) { - block = trx_undo_create(trx, rseg, undo, err, mtr); - ut_ad(!block == (*err != DB_SUCCESS)); - if (!block) { - goto func_exit; - } + buf_block_t* block; + if (is_temp) { + ut_ad(!UT_LIST_GET_LEN(rseg->undo_cached)); } else { - *err = DB_SUCCESS; + block = trx_undo_reuse_cached(trx, rseg, undo, mtr, err); + if (block) { + goto got_block; + } + } + block = trx_undo_create(trx, rseg, undo, err, mtr); + ut_ad(!block == (*err != DB_SUCCESS)); + if (!block) { + goto func_exit; } +got_block: UT_LIST_ADD_FIRST(rseg->undo_list, *undo); func_exit: @@ -1456,6 +1456,13 @@ func_exit: return block; } +template buf_block_t* +trx_undo_assign_low<false>(trx_t *trx, trx_rseg_t *rseg, trx_undo_t **undo, + mtr_t *mtr, dberr_t *err); +template buf_block_t* +trx_undo_assign_low<true>(trx_t *trx, trx_rseg_t *rseg, trx_undo_t **undo, + mtr_t *mtr, dberr_t *err); + /******************************************************************//** Sets the state of the undo log segment at a transaction finish. @return undo log segment header page, x-latched */ @@ -1466,6 +1473,7 @@ trx_undo_set_state_at_finish( mtr_t* mtr) /*!< in: mtr */ { ut_ad(undo->id < TRX_RSEG_N_SLOTS); + ut_ad(undo->rseg->is_persistent()); buf_block_t *block= buf_page_get(page_id_t(undo->rseg->space->id, undo->hdr_page_no), 0, @@ -1537,29 +1545,19 @@ the data can be discarded. @param undo temporary undo log */ void trx_undo_commit_cleanup(trx_undo_t *undo) { - trx_rseg_t* rseg = undo->rseg; - ut_ad(rseg->space == fil_system.temp_space); - - rseg->latch.wr_lock(SRW_LOCK_CALL); - - UT_LIST_REMOVE(rseg->undo_list, undo); - - if (undo->state == TRX_UNDO_CACHED) { - UT_LIST_ADD_FIRST(rseg->undo_cached, undo); - MONITOR_INC(MONITOR_NUM_UNDO_SLOT_CACHED); - undo = nullptr; - } else { - ut_ad(undo->state == TRX_UNDO_TO_PURGE); - - /* Delete first the undo log segment in the file */ - trx_undo_seg_free(undo); + trx_rseg_t *rseg= undo->rseg; + ut_ad(rseg->space == fil_system.temp_space); + rseg->latch.wr_lock(SRW_LOCK_CALL); - ut_ad(rseg->curr_size > undo->size); - rseg->curr_size -= undo->size; - } + UT_LIST_REMOVE(rseg->undo_list, undo); + ut_ad(undo->state == TRX_UNDO_TO_PURGE); + /* Delete first the undo log segment in the file */ + trx_undo_seg_free(undo); + ut_ad(rseg->curr_size > undo->size); + rseg->curr_size-= undo->size; - rseg->latch.wr_unlock(); - ut_free(undo); + rseg->latch.wr_unlock(); + ut_free(undo); } /** At shutdown, frees the undo logs of a transaction. */ diff --git a/storage/rocksdb/mysql-test/rocksdb/r/innodb_i_s_tables_disabled.result b/storage/rocksdb/mysql-test/rocksdb/r/innodb_i_s_tables_disabled.result index 65255699551..04b07292b60 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/innodb_i_s_tables_disabled.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/innodb_i_s_tables_disabled.result @@ -143,8 +143,8 @@ trx_commits_insert_update transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NUL trx_rollbacks transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of transactions rolled back trx_rollbacks_savepoint transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of transactions rolled back to savepoint trx_rseg_history_len transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 value Length of the TRX_RSEG_HISTORY list -trx_undo_slots_used transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of undo slots used -trx_undo_slots_cached transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of undo slots cached +trx_undo_slots_used transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 value Number of undo slots used +trx_undo_slots_cached transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 value Number of undo slots cached trx_rseg_current_size transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 value Current rollback segment size in pages purge_del_mark_records purge 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of delete-marked rows purged purge_upd_exist_or_extern_records purge 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of purges on updates of existing records and updates on delete marked record with externally stored field diff --git a/storage/spider/mysql-test/spider/bugfix/r/self_reference_multi.result b/storage/spider/mysql-test/spider/bugfix/r/self_reference_multi.result new file mode 100644 index 00000000000..c4399ddf9d2 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/self_reference_multi.result @@ -0,0 +1,21 @@ +for master_1 +for child2 +for child3 + +MDEV-6268 SPIDER table with no COMMENT clause causes queries to wait forever + +CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t2 (c int); +create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; +create table t0 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t1"'; +alter table t2 ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t0"'; +select * from t0; +ERROR HY000: An infinite loop is detected when opening table test.t0 +select * from t1; +ERROR HY000: An infinite loop is detected when opening table test.t0 +select * from t2; +ERROR HY000: An infinite loop is detected when opening table test.t0 +drop table t0, t1, t2; +for master_1 +for child2 +for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/t/self_reference_multi.test b/storage/spider/mysql-test/spider/bugfix/t/self_reference_multi.test new file mode 100644 index 00000000000..8b6f070d167 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/self_reference_multi.test @@ -0,0 +1,29 @@ +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log + +--echo +--echo MDEV-6268 SPIDER table with no COMMENT clause causes queries to wait forever +--echo + +--replace_regex /SOCKET ".*"/SOCKET "$MASTER_1_MYSOCK"/ +eval CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t2 (c int); +create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; +create table t0 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t1"'; +alter table t2 ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t0"'; +--error 12719 +select * from t0; +--error 12719 +select * from t1; +--error 12719 +select * from t2; +drop table t0, t1, t2; + +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log |