diff options
186 files changed, 3108 insertions, 1831 deletions
diff --git a/BUILD-CMAKE b/BUILD-CMAKE index c95482cf619..aedce4400c6 100644 --- a/BUILD-CMAKE +++ b/BUILD-CMAKE @@ -97,7 +97,7 @@ shell>cd ../build shell>cmake ../src Note: if a directory was used for in-source build, out-of-source will -not work. To reenable out-of-source build, remove <source-root>/CMakeCache.txt +not work. To re-enable out-of-source build, remove <source-root>/CMakeCache.txt file. diff --git a/BUILD/FINISH.sh b/BUILD/FINISH.sh index cae34885145..447eae0a65f 100644 --- a/BUILD/FINISH.sh +++ b/BUILD/FINISH.sh @@ -32,15 +32,19 @@ then configure="$configure --verbose" fi +commands="" # git clean -fdX removes all ignored (build) files -commands="\ +if test -d .git +then + commands="\ git clean -fdX cd ./libmariadb git submodule update cd ../storage/rocksdb/rocksdb git submodule update -cd ../../.. - +cd ../../.." +fi +commands="$commands path=`dirname $0` . \"$path/autorun.sh\"" diff --git a/BUILD/check-cpu b/BUILD/check-cpu index ad8816dc421..c1c85cfd908 100755 --- a/BUILD/check-cpu +++ b/BUILD/check-cpu @@ -40,6 +40,12 @@ check_compiler_cpu_flags () { cc_major=$1 cc_minor=$2 cc_patch=$3 + if test -z "$cc_minor"; then + cc_minor="0"; + fi + if test -z "$cc_patch"; then + cc_minor="0"; + fi cc_comp=`expr $cc_major '*' 100 '+' $cc_minor` fi diff --git a/BUILD/compile-pentium64-asan-max b/BUILD/compile-pentium64-asan-max new file mode 100755 index 00000000000..cd5a292d17c --- /dev/null +++ b/BUILD/compile-pentium64-asan-max @@ -0,0 +1,24 @@ +#! /bin/sh +# Copyright (c) 2018, MariaDB Corporation. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +path=`dirname $0` +. "$path/SETUP.sh" + +extra_flags="$pentium64_cflags $debug_cflags -lasan -O -g -fsanitize=address" +extra_configs="$pentium_configs $debug_configs $valgrind_configs $max_configs" +export LDFLAGS="-ldl" + +. "$path/FINISH.sh" diff --git a/Docs/sp-imp-spec.txt b/Docs/sp-imp-spec.txt index f2794e14793..0526cc43f81 100644 --- a/Docs/sp-imp-spec.txt +++ b/Docs/sp-imp-spec.txt @@ -323,7 +323,7 @@ One "obvious" solution would be to simply push "mysql.proc" to the list of tables used by the query, but this implies a "join" with this table if the query is a select, so it doesn't work (and we can't exclude this - table easily; since a priviledged used might in fact want to search + table easily; since a privileged used might in fact want to search the proc table). Another solution would of course be to allow the opening and closing of the mysql.proc table during a query execution, but this it not @@ -400,7 +400,7 @@ instruction. Calling and returning from a CONTINUE handler poses some special - problems. Since we need to return to the point after its invokation, + problems. Since we need to return to the point after its invocation, we push the return location on a stack in the sp_rcontext (this is done by the exectution loop). The handler then ends with a special instruction, sp_instr_hreturn, which returns to this location. diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake index 5b8aa36ff0d..513f3671b5d 100644 --- a/cmake/cpack_rpm.cmake +++ b/cmake/cpack_rpm.cmake @@ -201,7 +201,7 @@ ALTERNATIVE_NAME("server" "mysql-server") ALTERNATIVE_NAME("test" "mysql-test") # Argh! Different distributions call packages differently, to be a drop-in -# replacement we have to fake distribution-speficic dependencies +# replacement we have to fake distribution-specificic dependencies IF(RPM MATCHES "(rhel|centos)6") ALTERNATIVE_NAME("client" "mysql") diff --git a/cmake/install_layout.cmake b/cmake/install_layout.cmake index 5484691e94a..63e2419951b 100644 --- a/cmake/install_layout.cmake +++ b/cmake/install_layout.cmake @@ -176,7 +176,7 @@ SET(INSTALL_SYSCONF2DIR_DEB "/etc/mysql/conf.d") SET(INSTALL_LIBDIR_DEB "lib") SET(INSTALL_PLUGINDIR_DEB "lib/mysql/plugin") # -SET(INSTALL_INCLUDEDIR_DEB "include/mysql") +SET(INSTALL_INCLUDEDIR_DEB "include/mariadb") # SET(INSTALL_DOCDIR_DEB "share/doc") SET(INSTALL_DOCREADMEDIR_DEB "share/doc") diff --git a/cmake/os/Windows.cmake b/cmake/os/Windows.cmake index 021f23a14f2..4fd01e5914e 100644 --- a/cmake/os/Windows.cmake +++ b/cmake/os/Windows.cmake @@ -142,6 +142,11 @@ IF(MSVC) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4800 /wd4805 /wd4996 /we4700 /we4311 /we4477 /we4302 /we4090 /wd4267 ") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4800 /wd4805 /wd4996 /wd4291 /wd4577 /we4099 /we4700 /we4311 /we4477 /we4302 /we4090 /wd4267") + IF(MYSQL_MAINTAINER_MODE MATCHES "ERR") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /WX") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /WX") + ENDIF() + ENDIF() # Always link with socket library diff --git a/configure.cmake b/configure.cmake index 7258077d68b..b62ac12e634 100644 --- a/configure.cmake +++ b/configure.cmake @@ -488,7 +488,7 @@ int main() { # -# Test for endianess +# Test for endianness # INCLUDE(TestBigEndian) IF(APPLE) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index cc5b04188f4..fca3482e64b 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -33,7 +33,7 @@ fi # Look up distro-version specific stuff # # Always keep the actual packaging as up-to-date as possible following the latest -# Debian policy and targetting Debian Sid. Then case-by-case run in autobake-deb.sh +# Debian policy and targeting Debian Sid. Then case-by-case run in autobake-deb.sh # tests for backwards compatibility and strip away parts on older builders. # If iproute2 is not available (before Debian Jessie and Ubuntu Trusty) @@ -82,11 +82,20 @@ if [[ $GCCVERSION -lt 40800 ]] || [[ $(arch) =~ i[346]86 ]] || [[ $TRAVIS ]] then sed '/Package: mariadb-plugin-rocksdb/,+13d' -i debian/control fi + +# AWS SDK requires c++11 -capable compiler +# Minimal supported versions are g++ 4.8 and clang 3.3. if [[ $GCCVERSION -lt 40800 ]] || [[ $TRAVIS ]] then - sed '/Package: mariadb-plugin-aws-key-management-10.3/,+15d' -i debian/control + sed '/Package: mariadb-plugin-aws-key-management/,+14d' -i debian/control fi +# Mroonga, TokuDB never built on Travis CI anyway, see build flags above +if [[ $TRAVIS ]] +then + sed -i -e "/Package: mariadb-plugin-tokudb/,+17d" debian/control + sed -i -e "/Package: mariadb-plugin-mroonga/,+16d" debian/control +fi # Adjust changelog, add new version echo "Incrementing changelog and starting build scripts" @@ -102,12 +111,20 @@ dch -b -D ${CODENAME} -v "${UPSTREAM}${PATCHLEVEL}~${CODENAME}" "Automatic build echo "Creating package version ${UPSTREAM}${PATCHLEVEL}~${CODENAME} ... " +# On Travis CI, use -b to build binary only packages as there is no need to +# waste time on generating the source package. +if [[ $TRAVIS ]] +then + BUILDPACKAGE_FLAGS="-b" +fi + # Build the package # Pass -I so that .git and other unnecessary temporary and source control files # will be ignored by dpkg-source when creating the tar.gz source package. -# Use -b to build binary only packages as there is no need to waste time on -# generating the source package. -fakeroot dpkg-buildpackage -us -uc -I -b +fakeroot dpkg-buildpackage -us -uc -I $BUILDPACKAGE_FLAGS + +# If the step above fails due to missing dependencies, you can manually run +# sudo mk-build-deps debian/control -r -i # Don't log package contents on Travis-CI to save time and log size if [[ ! $TRAVIS ]] diff --git a/debian/control b/debian/control index fe9a10a2cb8..9ace4dc8841 100644 --- a/debian/control +++ b/debian/control @@ -13,17 +13,17 @@ Build-Depends: bison, libaio-dev [linux-any], libboost-dev, libcrack2-dev (>= 2.9.0), + libcurl3-dev, libjemalloc-dev (>= 3.0.0~) [linux-any], libjudy-dev, libkrb5-dev, - libcurl3-dev, libncurses5-dev (>= 5.0-6~), libnuma-dev, libpam0g-dev, libpcre3-dev (>= 2:8.35-3.2~), libreadline-gplv2-dev, - libssl-dev | libssl1.0-dev, libsnappy-dev, + libssl-dev | libssl1.0-dev, libsystemd-dev, libxml2-dev, lsb-release, @@ -38,86 +38,157 @@ Homepage: http://mariadb.org/ Vcs-Git: https://github.com/MariaDB/server.git Vcs-Browser: https://github.com/MariaDB/server/ +Package: libmariadb-dev +Architecture: any +Section: libdevel +Depends: libmariadb3 (= ${binary:Version}), + zlib1g-dev, + ${misc:Depends}, + ${shlibs:Depends} +Breaks: libmariadbclient-dev +Replaces: libmariadbclient-dev +Conflicts: libmariadbclient16-dev +Provides: libmariadbclient-dev +Description: MariaDB database development files + MariaDB is a fast, stable and true multi-user, multi-threaded SQL database + server. SQL (Structured Query Language) is the most popular database query + language in the world. The main goals of MariaDB are speed, robustness and + ease of use. + . + This package includes development libraries and header files. To allow sources + expecting the MariaDB Connector/C to build. Sources that expect the MySQL + Client libraries should use files from the libmariadb-dev-compat package. + +Package: libmariadb-dev-compat +Architecture: any +Multi-Arch: same +Section: libdevel +Priority: extra +Depends: libmariadb-dev (= ${binary:Version}), + ${misc:Depends} +Conflicts: libmariadb-client-lgpl-dev-compat, + libmariadbclient-dev-compat, + libmysqlclient-dev, + libmysqlclient10-dev, + libmysqlclient12-dev, + libmysqlclient14-dev, + libmysqlclient15-dev, + libmysqlclient16-dev +Provides: libmariadb-client-lgpl-dev-compat, + libmariadbclient-dev-compat, + libmysqlclient-dev +Replaces: libmariadb-client-lgpl-dev-compat, + libmariadbclient-dev-compat, + libmysqlclient-dev +Description: MariaDB Connector/C, compatibility symlinks + MariaDB is a fast, stable and true multi-user, multi-threaded SQL database + server. SQL (Structured Query Language) is the most popular database query + language in the world. The main goals of MariaDB are speed, robustness and + ease of use. + . + This package includes compatibility symlinks to allow sources expecting the + MySQL client libraries to be built against MariaDB Connector/C. + Package: libmariadb3 Architecture: any Section: libs -Depends: mariadb-common, ${misc:Depends}, ${shlibs:Depends} -Conflicts: mariadb-galera-server-10.0 (<< 10.0.5), +Depends: mariadb-common, + ${misc:Depends}, + ${shlibs:Depends} +Conflicts: libmariadbclient18 (<< 10.2.0), + mariadb-galera-server-10.0 (<< 10.0.5), mariadb-galera-server-5.5 (<< 5.5.33), mariadb-server-10.0 (<< 10.0.5), mariadb-server-5.1, mariadb-server-5.2, mariadb-server-5.3, - mariadb-server-5.5 (<< 5.5.33), - libmariadbclient18 (<< 10.2.0) + mariadb-server-5.5 (<< 5.5.33) +Description: MariaDB database client library + MariaDB is a fast, stable and true multi-user, multi-threaded SQL database + server. SQL (Structured Query Language) is the most popular database query + language in the world. The main goals of MariaDB are speed, robustness and + ease of use. + . + This package includes the client library. + +Package: libmariadb3-compat +Architecture: any +Section: libs +Depends: libmariadb3, + mariadb-common, + ${misc:Depends}, + ${shlibs:Depends} +Breaks: libmysqlclient19, + libmysqlclient20 Replaces: libmysqlclient19, libmysqlclient20 Provides: libmysqlclient19, libmysqlclient20 -Description: MariaDB database client library +Description: MariaDB database client library MySQL compat package MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query language in the world. The main goals of MariaDB are speed, robustness and ease of use. . - This package includes the client library. + This package includes the client runtime libraries that simulate and replace + the equivalents found in MySQL 5.6 and 5.7 (mysqlclient19 and 20). Package: libmariadbclient18 Section: libs Architecture: any -Depends: libmariadb3 (= ${binary:Version}), ${misc:Depends} +Depends: libmariadb3 (= ${binary:Version}), + ${misc:Depends} Replaces: libmariadbclient18 Provides: libmariadbclient18 -Description: Virtual package to satisfy external depends +Description: Virtual package to satisfy external libmariadbclient18 depends MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query language in the world. The main goals of MariaDB are speed, robustness and ease of use. . - This package provides compatibility symlinks for libmariadb3 + This package provides compatibility symlinks for binaries that expect to find + libmariadbclient.so.18 will automatically use libmariadb.so.3 instead. Package: libmysqlclient18 Section: libs Architecture: any -Depends: libmariadb3 (= ${binary:Version}), ${misc:Depends} +Depends: libmariadb3 (= ${binary:Version}), + ${misc:Depends} Replaces: libmysqlclient18 Provides: libmysqlclient18 -Description: Virtual package to satisfy external depends +Description: Virtual package to satisfy external libmysqlclient18 depends MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query language in the world. The main goals of MariaDB are speed, robustness and ease of use. . - This package provides compatibility symlinks for libmariadb3 + This package provides compatibility symlinks for binaries that expect to find + libmysqlclient.so.18 will automatically use libmariadb.so.3 instead. -Package: libmariadb-dev +Package: libmariadbd-dev Architecture: any Section: libdevel -Depends: libmariadb3 (= ${binary:Version}), - zlib1g-dev, +Provides: libmysqld-dev +Pre-Depends: ${misc:Pre-Depends} +Depends: libmariadb-dev (= ${binary:Version}), + libmariadbd19 (= ${binary:Version}), ${misc:Depends}, ${shlibs:Depends} -Breaks: libmariadbclient-dev, libmysqlclient-dev -Replaces: libmariadbclient-dev, libmysqlclient-dev -Conflicts: libmariadbclient16-dev, - libmysqlclient10-dev, - libmysqlclient12-dev, - libmysqlclient14-dev, - libmysqlclient15-dev, - libmysqlclient16-dev -Provides: libmariadbclient-dev, libmariadbclient-dev-compat, libmysqlclient-dev -Description: MariaDB database development files +Breaks: libmysqld-dev +Replaces: libmysqld-dev +Description: MariaDB embedded database, development files MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query language in the world. The main goals of MariaDB are speed, robustness and ease of use. . - This package includes development libraries and header files. + This package includes the embedded server library development and header files. Package: libmariadbd19 Architecture: any Section: libs -Depends: ${misc:Depends}, ${shlibs:Depends} +Depends: ${misc:Depends}, + ${shlibs:Depends} Multi-Arch: same Breaks: libmariadbd-dev (<< ${source:Version}) Replaces: libmariadbd-dev (<< ${source:Version}) @@ -129,28 +200,10 @@ Description: MariaDB embedded database, shared library . This package includes a shared library for embedded MariaDB applications -Package: libmariadbd-dev -Architecture: any -Section: libdevel -Provides: libmysqld-dev -Pre-Depends: ${misc:Pre-Depends} -Depends: libmariadb-dev (= ${binary:Version}), - libmariadbd19 (= ${binary:Version}), - ${misc:Depends}, - ${shlibs:Depends} -Breaks: libmysqld-dev -Replaces: libmysqld-dev -Description: MariaDB embedded database, development files - MariaDB is a fast, stable and true multi-user, multi-threaded SQL database - server. SQL (Structured Query Language) is the most popular database query - language in the world. The main goals of MariaDB are speed, robustness and - ease of use. - . - This package includes the embedded server library development and header files. - Package: mysql-common Architecture: all -Depends: ${misc:Depends}, ${shlibs:Depends} +Depends: ${misc:Depends}, + ${shlibs:Depends} Description: MariaDB database common files (e.g. /etc/mysql/my.cnf) MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query @@ -162,7 +215,9 @@ Description: MariaDB database common files (e.g. /etc/mysql/my.cnf) Package: mariadb-common Architecture: all -Depends: mysql-common, ${misc:Depends}, ${shlibs:Depends} +Depends: mysql-common, + ${misc:Depends}, + ${shlibs:Depends} Description: MariaDB database common files (e.g. /etc/mysql/conf.d/mariadb.cnf) MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query @@ -284,7 +339,9 @@ Provides: default-mysql-client, mysql-client-5.6, mysql-client-5.7, virtual-mysql-client -Recommends: libdbd-mysql-perl (>= 1.2202), libdbi-perl, libterm-readkey-perl +Recommends: libdbd-mysql-perl (>= 1.2202), + libdbi-perl, + libterm-readkey-perl Description: MariaDB database client binaries MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query @@ -353,9 +410,14 @@ Description: MariaDB database core server files Package: mariadb-server-10.3 Architecture: any -Suggests: mailx, mariadb-test, netcat-openbsd, tinyca +Suggests: mailx, + mariadb-test, + netcat-openbsd, + tinyca Recommends: libhtml-template-perl -Pre-Depends: adduser (>= 3.40), debconf, mariadb-common (>= ${source:Version}) +Pre-Depends: adduser (>= 3.40), + debconf, + mariadb-common (>= ${source:Version}) Depends: bsdutils, coreutils, findutils, @@ -416,7 +478,8 @@ Replaces: libmariadbclient-dev (<< 5.5.0), mysql-server-5.6, mysql-server-5.7, virtual-mysql-server -Provides: default-mysql-server, virtual-mysql-server +Provides: default-mysql-server, + virtual-mysql-server Description: MariaDB database server binaries MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query @@ -427,7 +490,8 @@ Description: MariaDB database server binaries Package: mariadb-server Architecture: all -Depends: mariadb-server-10.3 (>= ${source:Version}), ${misc:Depends} +Depends: mariadb-server-10.3 (>= ${source:Version}), + ${misc:Depends} Description: MariaDB database server (metapackage depending on the latest version) This is an empty package that depends on the current "best" version of mariadb-server (currently mariadb-server-10.3), as determined by the MariaDB @@ -442,7 +506,8 @@ Description: MariaDB database server (metapackage depending on the latest versio Package: mariadb-client Architecture: all -Depends: mariadb-client-10.3 (>= ${source:Version}), ${misc:Depends} +Depends: mariadb-client-10.3 (>= ${source:Version}), + ${misc:Depends} Description: MariaDB database client (metapackage depending on the latest version) This is an empty package that depends on the current "best" version of mariadb-client (currently mariadb-client-10.3), as determined by the MariaDB @@ -500,7 +565,7 @@ Description: OQGraph storage engine for MariaDB This package contains the OQGraph plugin for MariaDB. Package: mariadb-plugin-tokudb -Architecture: any +Architecture: amd64 Depends: mariadb-server-10.3 (= ${binary:Version}), ${misc:Depends}, ${shlibs:Depends} @@ -519,7 +584,7 @@ Description: TokuDB storage engine for MariaDB This package contains the TokuDB plugin for MariaDB. Package: mariadb-plugin-mroonga -Architecture: any +Architecture: any-alpha any-amd64 any-arm any-arm64 any-i386 any-ia64 any-mips64el any-mips64r6el any-mipsel any-mipsr6el any-nios2 any-powerpcel any-ppc64el any-sh3 any-sh4 any-tilegx Depends: mariadb-server-10.3 (= ${binary:Version}), ${misc:Depends}, ${shlibs:Depends} @@ -589,6 +654,7 @@ Replaces: mariadb-gssapi-server-10.1, mariadb-gssapi-server-10.2, mariadb-gssapi-server-10.3 Description: GSSAPI authentication plugin for MariaDB server + This package contains the server components. Package: mariadb-plugin-gssapi-client Architecture: any @@ -603,18 +669,19 @@ Replaces: mariadb-gssapi-client-10.1, mariadb-gssapi-client-10.2, mariadb-gssapi-client-10.3 Description: GSSAPI authentication plugin for MariaDB client + This package contains the client components. Package: mariadb-backup-10.3 -Section: database Architecture: any Breaks: mariadb-backup-10.1, mariadb-backup-10.2 Replaces: mariadb-backup-10.1, - mariadb-backup-10.2 + mariadb-backup-10.2 Depends: mariadb-server-10.3, ${misc:Depends}, ${shlibs:Depends} Description: Backup tool for MariaDB server + This backup tool is guaranteed to be compatible with MariaDB. Package: mariadb-plugin-cracklib-password-check Architecture: any @@ -626,15 +693,14 @@ Description: CrackLib Password Validation Plugin for MariaDB This password validation plugin uses cracklib to allow only sufficiently secure (as defined by cracklib) user passwords in MariaDB. -Package: mariadb-plugin-aws-key-management-10.3 -Section: database +Package: mariadb-plugin-aws-key-management Architecture: any Breaks: mariadb-aws-key-management-10.1, mariadb-aws-key-management-10.2 Replaces: mariadb-aws-key-management-10.1, mariadb-aws-key-management-10.2 -Depends: mariadb-server-10.3, - libcurl3, +Depends: libcurl3, + mariadb-server-10.3, ${misc:Depends}, ${shlibs:Depends} Description: Amazon Web Service Key Management Service Plugin for MariaDB diff --git a/debian/libmariadb-dev-compat.install b/debian/libmariadb-dev-compat.install new file mode 100644 index 00000000000..7fe10c50f69 --- /dev/null +++ b/debian/libmariadb-dev-compat.install @@ -0,0 +1,4 @@ +usr/lib/*/libmysqlclient.a +usr/lib/*/libmysqlclient.so +usr/lib/*/libmysqlclient_r.a +usr/lib/*/libmysqlclient_r.so diff --git a/debian/libmariadb-dev-compat.links b/debian/libmariadb-dev-compat.links new file mode 100644 index 00000000000..11e0dbce225 --- /dev/null +++ b/debian/libmariadb-dev-compat.links @@ -0,0 +1,3 @@ +usr/bin/mariadb_config usr/bin/mysql_config +usr/include/mariadb usr/include/mysql +usr/share/pkgconfig/mariadb.pc usr/share/pkgconfig/mysqlclient.pc diff --git a/debian/libmariadb-dev.install b/debian/libmariadb-dev.install index e62aad1b43f..72798dec379 100644 --- a/debian/libmariadb-dev.install +++ b/debian/libmariadb-dev.install @@ -1,9 +1,9 @@ -usr/bin/mysql_config -usr/include/mysql +usr/bin/mariadb_config +usr/include/mariadb usr/lib/*/libmariadb.so -usr/lib/*/libmysqlclient.so -usr/lib/*/libmariadbclient.so usr/lib/*/libmariadbclient.a +usr/lib/*/libmariadbclient.so usr/lib/*/libmysqlservices.a usr/share/aclocal/mysql.m4 +usr/share/man/man1/mysql_config.1 usr/share/pkgconfig/mariadb.pc diff --git a/debian/libmariadb-dev.links b/debian/libmariadb-dev.links new file mode 100644 index 00000000000..6a7bf707ca7 --- /dev/null +++ b/debian/libmariadb-dev.links @@ -0,0 +1 @@ +usr/share/man/man1/mysql_config.1.gz usr/share/man/man1/mariadb_config.1.gz diff --git a/debian/libmariadb-dev.manpages b/debian/libmariadb-dev.manpages deleted file mode 100644 index 3aac7f4b4a9..00000000000 --- a/debian/libmariadb-dev.manpages +++ /dev/null @@ -1 +0,0 @@ -debian/tmp/usr/share/man/man1/mysql_config.1 diff --git a/debian/libmariadb3-compat.install b/debian/libmariadb3-compat.install new file mode 100644 index 00000000000..4e0d631d6c8 --- /dev/null +++ b/debian/libmariadb3-compat.install @@ -0,0 +1,2 @@ +usr/lib/*/libmysqlclient.so.19 +usr/lib/*/libmysqlclient.so.20 diff --git a/debian/libmariadb3.install b/debian/libmariadb3.install index 58e2bd1dcb2..8636166a493 100644 --- a/debian/libmariadb3.install +++ b/debian/libmariadb3.install @@ -1,5 +1,5 @@ -usr/lib/*/libmysqlclient.so.19 -usr/lib/*/libmysqlclient.so.20 usr/lib/*/libmariadb.so.* +usr/lib/mysql/plugin/client_ed25519.so usr/lib/mysql/plugin/dialog.so usr/lib/mysql/plugin/mysql_clear_password.so +usr/lib/mysql/plugin/sha256_password.so diff --git a/debian/libmariadbd-dev.install b/debian/libmariadbd-dev.install index 6fe225b15ba..4608bf6e838 100644 --- a/debian/libmariadbd-dev.install +++ b/debian/libmariadbd-dev.install @@ -1,3 +1,2 @@ -usr/bin/mariadb_config usr/lib/*/libmysqld.a usr/lib/*/libmysqld.so diff --git a/debian/mariadb-backup-10.3.files b/debian/mariadb-backup-10.3.install index 734117c92e2..734117c92e2 100644 --- a/debian/mariadb-backup-10.3.files +++ b/debian/mariadb-backup-10.3.install diff --git a/debian/mariadb-client-10.3.README.Debian b/debian/mariadb-client-10.3.README.Debian index b245638f9c9..64f0f509951 100644 --- a/debian/mariadb-client-10.3.README.Debian +++ b/debian/mariadb-client-10.3.README.Debian @@ -1,4 +1,4 @@ FAQ: -Q: My <tab> completition is gone, why? +Q: My <tab> completion is gone, why? A: You have "no-auto-rehash" in the "[mysql]" section of /etc/mysql/my.cnf! diff --git a/debian/mariadb-client-10.3.docs b/debian/mariadb-client-10.3.docs index 8117d689011..c09092629c3 100644 --- a/debian/mariadb-client-10.3.docs +++ b/debian/mariadb-client-10.3.docs @@ -1,2 +1,2 @@ -debian/additions/innotop/changelog.innotop README.md +debian/additions/innotop/changelog.innotop diff --git a/debian/mariadb-client-10.3.install b/debian/mariadb-client-10.3.install index 271f2a2f417..945bf77c689 100644 --- a/debian/mariadb-client-10.3.install +++ b/debian/mariadb-client-10.3.install @@ -10,3 +10,13 @@ usr/bin/mysqldumpslow usr/bin/mysqlimport usr/bin/mysqlshow usr/bin/mysqlslap +usr/share/man/man1/mysql_find_rows.1 +usr/share/man/man1/mysql_fix_extensions.1 +usr/share/man/man1/mysql_waitpid.1 +usr/share/man/man1/mysqlaccess.1 +usr/share/man/man1/mysqladmin.1 +usr/share/man/man1/mysqldump.1 +usr/share/man/man1/mysqldumpslow.1 +usr/share/man/man1/mysqlimport.1 +usr/share/man/man1/mysqlshow.1 +usr/share/man/man1/mysqlslap.1 diff --git a/debian/mariadb-client-10.3.links b/debian/mariadb-client-10.3.links index 0b86e87f2e9..4a504969246 100644 --- a/debian/mariadb-client-10.3.links +++ b/debian/mariadb-client-10.3.links @@ -1,6 +1,6 @@ -usr/bin/mysqlcheck usr/bin/mysqlrepair usr/bin/mysqlcheck usr/bin/mysqlanalyze usr/bin/mysqlcheck usr/bin/mysqloptimize -usr/share/man/man1/mysqlcheck.1.gz usr/share/man/man1/mysqlrepair.1.gz +usr/bin/mysqlcheck usr/bin/mysqlrepair usr/share/man/man1/mysqlcheck.1.gz usr/share/man/man1/mysqlanalyze.1.gz usr/share/man/man1/mysqlcheck.1.gz usr/share/man/man1/mysqloptimize.1.gz +usr/share/man/man1/mysqlcheck.1.gz usr/share/man/man1/mysqlrepair.1.gz diff --git a/debian/mariadb-client-10.3.manpages b/debian/mariadb-client-10.3.manpages index 504619785bc..6f3e2bc188c 100644 --- a/debian/mariadb-client-10.3.manpages +++ b/debian/mariadb-client-10.3.manpages @@ -1,12 +1,2 @@ debian/additions/innotop/innotop.1 -debian/tmp/usr/share/man/man1/mysqlaccess.1 -debian/tmp/usr/share/man/man1/mysqladmin.1 -debian/tmp/usr/share/man/man1/mysqldump.1 -debian/tmp/usr/share/man/man1/mysqldumpslow.1 -debian/tmp/usr/share/man/man1/mysql_find_rows.1 -debian/tmp/usr/share/man/man1/mysql_fix_extensions.1 -debian/tmp/usr/share/man/man1/mysqlimport.1 debian/additions/mysqlreport.1 -debian/tmp/usr/share/man/man1/mysqlshow.1 -debian/tmp/usr/share/man/man1/mysqlslap.1 -debian/tmp/usr/share/man/man1/mysql_waitpid.1 diff --git a/debian/mariadb-client-core-10.3.install b/debian/mariadb-client-core-10.3.install index 6308d769930..a2781309439 100644 --- a/debian/mariadb-client-core-10.3.install +++ b/debian/mariadb-client-core-10.3.install @@ -1,2 +1,4 @@ usr/bin/mysql usr/bin/mysqlcheck +usr/share/man/man1/mysql.1 +usr/share/man/man1/mysqlcheck.1 diff --git a/debian/mariadb-client-core-10.3.manpages b/debian/mariadb-client-core-10.3.manpages deleted file mode 100644 index 2be91d81f9a..00000000000 --- a/debian/mariadb-client-core-10.3.manpages +++ /dev/null @@ -1,2 +0,0 @@ -debian/tmp/usr/share/man/man1/mysql.1 -debian/tmp/usr/share/man/man1/mysqlcheck.1 diff --git a/debian/mariadb-common.install b/debian/mariadb-common.install index 78dbc22b9f6..611c7d4d36e 100644 --- a/debian/mariadb-common.install +++ b/debian/mariadb-common.install @@ -1 +1 @@ -debian/additions/mariadb.cnf etc/mysql/conf.d +debian/additions/mariadb.cnf etc/mysql/ diff --git a/debian/mariadb-plugin-aws-key-management-10.2.install b/debian/mariadb-plugin-aws-key-management-10.2.install deleted file mode 100644 index ed966b4115d..00000000000 --- a/debian/mariadb-plugin-aws-key-management-10.2.install +++ /dev/null @@ -1,2 +0,0 @@ -usr/lib/mysql/plugin/aws_key_management.so -debian/additions/enable_encryption.preset etc/mysql/conf.d/ diff --git a/debian/mariadb-plugin-aws-key-management.install b/debian/mariadb-plugin-aws-key-management.install new file mode 100644 index 00000000000..199d4c87950 --- /dev/null +++ b/debian/mariadb-plugin-aws-key-management.install @@ -0,0 +1,3 @@ +debian/additions/enable_encryption.preset etc/mysql/mariadb.conf.d +etc/mysql/conf.d/aws_key_management.cnf etc/mysql/mariadb.conf.d +usr/lib/mysql/plugin/aws_key_management.so diff --git a/debian/mariadb-plugin-connect.install b/debian/mariadb-plugin-connect.install index 8a7aee412df..22d73c7df05 100644 --- a/debian/mariadb-plugin-connect.install +++ b/debian/mariadb-plugin-connect.install @@ -1,2 +1,2 @@ -etc/mysql/conf.d/connect.cnf +etc/mysql/conf.d/connect.cnf etc/mysql/mariadb.conf.d usr/lib/mysql/plugin/ha_connect.so diff --git a/debian/mariadb-plugin-mroonga.install b/debian/mariadb-plugin-mroonga.install index c28fde2fd18..fedcf62eef0 100644 --- a/debian/mariadb-plugin-mroonga.install +++ b/debian/mariadb-plugin-mroonga.install @@ -1,3 +1,5 @@ usr/lib/mysql/plugin/ha_mroonga.so +usr/share/mysql/mroonga/AUTHORS +usr/share/mysql/mroonga/COPYING usr/share/mysql/mroonga/install.sql usr/share/mysql/mroonga/uninstall.sql diff --git a/debian/mariadb-plugin-rocksdb.install b/debian/mariadb-plugin-rocksdb.install index ee45a822e0c..224ef5501c5 100644 --- a/debian/mariadb-plugin-rocksdb.install +++ b/debian/mariadb-plugin-rocksdb.install @@ -1,4 +1,4 @@ etc/mysql/conf.d/rocksdb.cnf etc/mysql/mariadb.conf.d -usr/lib/mysql/plugin/ha_rocksdb.so usr/bin/mysql_ldb usr/bin/sst_dump +usr/lib/mysql/plugin/ha_rocksdb.so diff --git a/debian/mariadb-plugin-tokudb.install b/debian/mariadb-plugin-tokudb.install index 3c366ee0c44..d15db03b3e5 100644 --- a/debian/mariadb-plugin-tokudb.install +++ b/debian/mariadb-plugin-tokudb.install @@ -1,4 +1,7 @@ etc/mysql/conf.d/tokudb.cnf etc/mysql/mariadb.conf.d +usr/bin/tokuft_logprint usr/bin/tokuftdump usr/lib/mysql/plugin/ha_tokudb.so usr/share/doc/mariadb-server-10.3/README.md usr/share/doc/mariadb-plugin-tokudb/README.md +usr/share/man/man1/tokuft_logdump.1 +usr/share/man/man1/tokuftdump.1 diff --git a/debian/mariadb-server-10.3.README.Debian b/debian/mariadb-server-10.3.README.Debian index be2e33d705d..ca81cc357c4 100644 --- a/debian/mariadb-server-10.3.README.Debian +++ b/debian/mariadb-server-10.3.README.Debian @@ -42,7 +42,7 @@ https://mariadb.com/kb It is strongly recommended you create an admin users for your database adminstration needs. -If your your local unix account is the one you want to have local super user +If your local unix account is the one you want to have local super user access on your database with you can create the following account that will only work for the local unix user connecting to the database locally. @@ -57,14 +57,14 @@ the DB server over the network: sudo /usr/bin/mysql -e "GRANT ALL ON *.* TO 'USERNAME'@'%' IDENTIFIED BY 'password' WITH GRANT OPTION" -Scripts should run as a user have have the required grants and be identified via unix_socket. +Scripts should run as a user have the required grants and be identified via unix_socket. If you are too tired to type the password in every time and unix_socket auth doesn't suit your needs, you can store it in the file $HOME/.my.cnf. It should -be chmod 0600 (-rw------- username username .my.cnf) to ensure that nobody else +be chmod 0600 (-rw------- username usergroup .my.cnf) to ensure that nobody else can read it. Every other configuration parameter can be stored there, too. -For more information in the MariaDB manual in/usr/share/doc/mariadb-doc or +For more information in the MariaDB manual in/usr/share/doc/mariadb-doc or https://mariadb.com/kb/en/configuring-mariadb-with-mycnf/. ATTENTION: It is necessary, that a ~/.my.cnf from root always contains a "user" diff --git a/debian/mariadb-server-10.3.install b/debian/mariadb-server-10.3.install index 604179c7dca..d8de5512b05 100644 --- a/debian/mariadb-server-10.3.install +++ b/debian/mariadb-server-10.3.install @@ -19,7 +19,6 @@ usr/bin/myisamchk usr/bin/myisamlog usr/bin/myisampack usr/bin/mysql_convert_table_format -usr/bin/mysql_install_db usr/bin/mysql_plugin usr/bin/mysql_secure_installation usr/bin/mysql_setpermission @@ -34,11 +33,12 @@ usr/bin/replace usr/bin/resolve_stack_dump usr/bin/resolveip usr/bin/wsrep_sst_common +usr/bin/wsrep_sst_mariabackup usr/bin/wsrep_sst_mysqldump usr/bin/wsrep_sst_rsync usr/bin/wsrep_sst_xtrabackup usr/bin/wsrep_sst_xtrabackup-v2 -usr/bin/wsrep_sst_mariabackup +usr/lib/mysql/plugin/auth_ed25519.so usr/lib/mysql/plugin/auth_pam.so usr/lib/mysql/plugin/auth_socket.so usr/lib/mysql/plugin/file_key_management.so @@ -58,6 +58,39 @@ usr/lib/mysql/plugin/sql_errlog.so usr/lib/mysql/plugin/wsrep_info.so usr/share/apport/package-hooks/source_mariadb-10.3.py usr/share/doc/mariadb-server-10.3/mysqld.sym.gz +usr/share/man/man1/aria_chk.1 +usr/share/man/man1/aria_dump_log.1 +usr/share/man/man1/aria_ftdump.1 +usr/share/man/man1/aria_pack.1 +usr/share/man/man1/aria_read_log.1 +usr/share/man/man1/galera_new_cluster.1 +usr/share/man/man1/galera_recovery.1 +usr/share/man/man1/mariadb-service-convert.1 +usr/share/man/man1/msql2mysql.1 +usr/share/man/man1/my_print_defaults.1 +usr/share/man/man1/myisam_ftdump.1 +usr/share/man/man1/myisamchk.1 +usr/share/man/man1/myisamlog.1 +usr/share/man/man1/myisampack.1 +usr/share/man/man1/mysql_convert_table_format.1 +usr/share/man/man1/mysql_plugin.1 +usr/share/man/man1/mysql_secure_installation.1 +usr/share/man/man1/mysql_setpermission.1 +usr/share/man/man1/mysql_tzinfo_to_sql.1 +usr/share/man/man1/mysqlbinlog.1 +usr/share/man/man1/mysqld_multi.1 +usr/share/man/man1/mysqld_safe.1 +usr/share/man/man1/mysqld_safe_helper.1 +usr/share/man/man1/mysqlhotcopy.1 +usr/share/man/man1/perror.1 +usr/share/man/man1/replace.1 +usr/share/man/man1/resolve_stack_dump.1 +usr/share/man/man1/resolveip.1 +usr/share/man/man1/wsrep_sst_common.1 +usr/share/man/man1/wsrep_sst_mysqldump.1 +usr/share/man/man1/wsrep_sst_rsync.1 +usr/share/man/man1/wsrep_sst_xtrabackup-v2.1 +usr/share/man/man1/wsrep_sst_xtrabackup.1 usr/share/mysql/errmsg-utf8.txt usr/share/mysql/fill_help_tables.sql usr/share/mysql/maria_add_gis_sp_bootstrap.sql diff --git a/debian/mariadb-server-10.3.manpages b/debian/mariadb-server-10.3.manpages deleted file mode 100644 index 1825b7655da..00000000000 --- a/debian/mariadb-server-10.3.manpages +++ /dev/null @@ -1,25 +0,0 @@ -debian/tmp/usr/share/man/man1/aria_chk.1 -debian/tmp/usr/share/man/man1/aria_dump_log.1 -debian/tmp/usr/share/man/man1/aria_ftdump.1 -debian/tmp/usr/share/man/man1/aria_pack.1 -debian/tmp/usr/share/man/man1/aria_read_log.1 -debian/tmp/usr/share/man/man1/msql2mysql.1 -debian/tmp/usr/share/man/man1/myisamchk.1 -debian/tmp/usr/share/man/man1/myisam_ftdump.1 -debian/tmp/usr/share/man/man1/myisamlog.1 -debian/tmp/usr/share/man/man1/myisampack.1 -debian/tmp/usr/share/man/man1/my_print_defaults.1 -debian/tmp/usr/share/man/man1/mysqlbinlog.1 -debian/tmp/usr/share/man/man1/mysql_convert_table_format.1 -debian/tmp/usr/share/man/man1/mysqld_multi.1 -debian/tmp/usr/share/man/man1/mysqld_safe.1 -debian/tmp/usr/share/man/man1/mysqlhotcopy.1 -debian/tmp/usr/share/man/man1/mysql_install_db.1 -debian/tmp/usr/share/man/man1/mysql_plugin.1 -debian/tmp/usr/share/man/man1/mysql_secure_installation.1 -debian/tmp/usr/share/man/man1/mysql_setpermission.1 -debian/tmp/usr/share/man/man1/mysql_tzinfo_to_sql.1 -debian/tmp/usr/share/man/man1/perror.1 -debian/tmp/usr/share/man/man1/replace.1 -debian/tmp/usr/share/man/man1/resolveip.1 -debian/tmp/usr/share/man/man1/resolve_stack_dump.1 diff --git a/debian/mariadb-server-10.3.mysql.init b/debian/mariadb-server-10.3.mysql.init index d7f6d58e8ca..35a52d5d8db 100644 --- a/debian/mariadb-server-10.3.mysql.init +++ b/debian/mariadb-server-10.3.mysql.init @@ -25,7 +25,7 @@ SELF=$(cd $(dirname $0); pwd -P)/$(basename $0) MYADMIN="/usr/bin/mysqladmin --defaults-file=/etc/mysql/debian.cnf" -# priority can be overriden and "-s" adds output to stderr +# priority can be overridden and "-s" adds output to stderr ERR_LOGGER="logger -p daemon.err -t /etc/init.d/mysql -i" if [ -f /etc/default/mysql ]; then diff --git a/debian/mariadb-server-core-10.3.install b/debian/mariadb-server-core-10.3.install index d882bd53f52..46c116b618d 100644 --- a/debian/mariadb-server-core-10.3.install +++ b/debian/mariadb-server-core-10.3.install @@ -1,6 +1,11 @@ usr/bin/innochecksum +usr/bin/mysql_install_db usr/bin/mysql_upgrade usr/sbin/mysqld +usr/share/man/man1/innochecksum.1 +usr/share/man/man1/mysql_install_db.1 +usr/share/man/man1/mysql_upgrade.1 +usr/share/man/man8/mysqld.8 usr/share/mysql/charsets usr/share/mysql/czech usr/share/mysql/danish diff --git a/debian/mariadb-server-core-10.3.manpages b/debian/mariadb-server-core-10.3.manpages deleted file mode 100644 index 234108cde50..00000000000 --- a/debian/mariadb-server-core-10.3.manpages +++ /dev/null @@ -1,3 +0,0 @@ -debian/tmp/usr/share/man/man8/mysqld.8 -debian/tmp/usr/share/man/man1/innochecksum.1 -debian/tmp/usr/share/man/man1/mysql_upgrade.1 diff --git a/debian/mariadb-test.install b/debian/mariadb-test.install index 75f52b3f526..e1de4074310 100644 --- a/debian/mariadb-test.install +++ b/debian/mariadb-test.install @@ -17,9 +17,16 @@ usr/lib/mysql/plugin/qa_auth_client.so usr/lib/mysql/plugin/qa_auth_interface.so usr/lib/mysql/plugin/qa_auth_server.so usr/lib/mysql/plugin/test_versioning.so +usr/share/man/man1/mysql-stress-test.pl.1 +usr/share/man/man1/mysql-test-run.pl.1 +usr/share/man/man1/mysql_client_test.1 +usr/share/man/man1/mysql_client_test_embedded.1 +usr/share/man/man1/mysqltest.1 +usr/share/man/man1/mysqltest_embedded.1 usr/share/mysql/mysql-test/README usr/share/mysql/mysql-test/README-gcov usr/share/mysql/mysql-test/README.stress +usr/share/mysql/mysql-test/dgcov.pl usr/share/mysql/mysql-test/disabled.def usr/share/mysql/mysql-test/lib usr/share/mysql/mysql-test/mysql-stress-test.pl diff --git a/debian/mariadb-test.links b/debian/mariadb-test.links index 082680fe5ed..884b25a81da 100644 --- a/debian/mariadb-test.links +++ b/debian/mariadb-test.links @@ -1,2 +1,2 @@ -usr/share/mysql/mysql-test/mysql-test-run.pl usr/share/mysql/mysql-test/mysql-test-run usr/share/mysql/mysql-test/mysql-test-run.pl usr/share/mysql/mysql-test/mtr +usr/share/mysql/mysql-test/mysql-test-run.pl usr/share/mysql/mysql-test/mysql-test-run diff --git a/debian/mariadb-test.manpages b/debian/mariadb-test.manpages deleted file mode 100644 index 33ce8710ea7..00000000000 --- a/debian/mariadb-test.manpages +++ /dev/null @@ -1,4 +0,0 @@ -debian/tmp/usr/share/man/man1/mysql_client_test.1 -debian/tmp/usr/share/man/man1/mysql_client_test_embedded.1 -debian/tmp/usr/share/man/man1/mysqltest.1 -debian/tmp/usr/share/man/man1/mysqltest_embedded.1 diff --git a/debian/mysql-common.install b/debian/mysql-common.install index 5ff685f42ff..264df611822 100644 --- a/debian/mysql-common.install +++ b/debian/mysql-common.install @@ -1,2 +1,2 @@ debian/additions/my.cnf etc/mysql -usr/share/mysql-common/internal-use-only
\ No newline at end of file +usr/share/mysql-common/internal-use-only diff --git a/debian/not-installed b/debian/not-installed new file mode 100644 index 00000000000..c151cf753e6 --- /dev/null +++ b/debian/not-installed @@ -0,0 +1,127 @@ +lib/systemd/system/mariadb.service # Installed by rules file +lib/systemd/system/mariadb@.service # Installed by rules file +usr/bin/mysql_config # We already have the MariaDB variant +usr/bin/mysql_embedded # Huge 500 MB file. Not intended for distribution via any disto package. +usr/bin/mytop # Mytop is distributed from a separate source package +usr/lib/sysusers.d/sysusers.conf +usr/lib/tmpfiles.d/tmpfiles.conf +usr/lib/mysql/plugin/JavaWrappers.jar # These are only built if JNI/libjawt.so is installed from e.g. openjdk-8-jre-headless +usr/lib/mysql/plugin/JdbcInterface.jar # These are only built if JNI/libjawt.so is installed from e.g. openjdk-8-jre-headless +usr/share/doc/mariadb-server-10.3/COPYING +usr/share/doc/mariadb-server-10.3/COPYING.AGPLv3 +usr/share/doc/mariadb-server-10.3/COPYING.GPLv2 +usr/share/doc/mariadb-server-10.3/COPYING.thirdparty +usr/share/doc/mariadb-server-10.3/CREDITS +usr/share/doc/mariadb-server-10.3/EXCEPTIONS-CLIENT +usr/share/doc/mariadb-server-10.3/INSTALL-BINARY +usr/share/doc/mariadb-server-10.3/PATENTS +usr/share/doc/mariadb-server-10.3/README-wsrep +usr/share/groonga/COPYING +usr/share/groonga-normalizer-mysql/lgpl-2.0.txt +usr/share/groonga-normalizer-mysql/README.md +usr/share/groonga/README.md +usr/share/man/man1/my_safe_process.1 +usr/share/man/man1/mysql.server.1 +usr/share/mysql/binary-configure +usr/share/mysql/magic +usr/share/mysql/maria_add_gis_sp.sql +usr/share/mysql/mysqld_multi.server +usr/share/mysql/mysql-log-rotate +usr/share/mysql/mysql.server +usr/share/mysql/mysql-test/mtr # Already created by mariadb-test.links +usr/share/mysql/mysql-test/mysql-test-run # Already created by mariadb-test.links +usr/share/mysql/mysql_to_mariadb.sql +usr/share/mysql/policy/apparmor/README # In MariaDB we don't want to use AppArmor at the moment +usr/share/mysql/policy/apparmor/usr.sbin.mysqld # In MariaDB we don't want to use AppArmor at the moment +usr/share/mysql/policy/apparmor/usr.sbin.mysqld.local # In MariaDB we don't want to use AppArmor at the moment +usr/share/mysql/policy/selinux/mariadb-server.fc # In MariaDB we don't want to use SELinux at the moment +usr/share/mysql/policy/selinux/mariadb-server.te # In MariaDB we don't want to use SELinux at the moment +usr/share/mysql/policy/selinux/mariadb.te # In MariaDB we don't want to use SELinux at the moment +usr/share/mysql/policy/selinux/README # In MariaDB we don't want to use SELinux at the moment +usr/share/mysql/systemd/mariadb.service # Installed by rules file +usr/share/mysql/systemd/mariadb@.service # Installed by rules file +usr/share/mysql/systemd/use_galera_new_cluster.conf +usr/share/mysql/wsrep.cnf +usr/sql-bench/bench-count-distinct +usr/sql-bench/bench-init.pl # SQL-bench is distributed from a separate source package +usr/sql-bench/compare-results +usr/sql-bench/copy-db +usr/sql-bench/crash-me +usr/sql-bench/Data/ATIS/aircraft.txt +usr/sql-bench/Data/ATIS/airline.txt +usr/sql-bench/Data/ATIS/airport_service.txt +usr/sql-bench/Data/ATIS/airport.txt +usr/sql-bench/Data/ATIS/city.txt +usr/sql-bench/Data/ATIS/class_of_service.txt +usr/sql-bench/Data/ATIS/code_description.txt +usr/sql-bench/Data/ATIS/compound_class.txt +usr/sql-bench/Data/ATIS/connect_leg.txt +usr/sql-bench/Data/ATIS/date_day.txt +usr/sql-bench/Data/ATIS/day_name.txt +usr/sql-bench/Data/ATIS/dual_carrier.txt +usr/sql-bench/Data/ATIS/fare.txt +usr/sql-bench/Data/ATIS/fconnection.txt +usr/sql-bench/Data/ATIS/flight_class.txt +usr/sql-bench/Data/ATIS/flight_day.txt +usr/sql-bench/Data/ATIS/flight_fare.txt +usr/sql-bench/Data/ATIS/flight.txt +usr/sql-bench/Data/ATIS/food_service.txt +usr/sql-bench/Data/ATIS/ground_service.txt +usr/sql-bench/Data/ATIS/month_name.txt +usr/sql-bench/Data/ATIS/restrict_carrier.txt +usr/sql-bench/Data/ATIS/restrict_class.txt +usr/sql-bench/Data/ATIS/restriction.txt +usr/sql-bench/Data/ATIS/state.txt +usr/sql-bench/Data/ATIS/stop1.txt +usr/sql-bench/Data/ATIS/stop.txt +usr/sql-bench/Data/ATIS/time_interval.txt +usr/sql-bench/Data/ATIS/time_zone.txt +usr/sql-bench/Data/ATIS/transport.txt +usr/sql-bench/Data/Wisconsin/onek.data +usr/sql-bench/Data/Wisconsin/tenk.data +usr/sql-bench/graph-compare-results +usr/sql-bench/innotest1 +usr/sql-bench/innotest1a +usr/sql-bench/innotest1b +usr/sql-bench/innotest2 +usr/sql-bench/innotest2a +usr/sql-bench/innotest2b +usr/sql-bench/limits/access.cfg +usr/sql-bench/limits/access_odbc.cfg +usr/sql-bench/limits/Adabas.cfg +usr/sql-bench/limits/db2.cfg +usr/sql-bench/limits/empress.cfg +usr/sql-bench/limits/frontbase.cfg +usr/sql-bench/limits/Informix.cfg +usr/sql-bench/limits/interbase.cfg +usr/sql-bench/limits/interbase-dialect1.cfg +usr/sql-bench/limits/interbase-dialect3.cfg +usr/sql-bench/limits/interbase-superserver.cfg +usr/sql-bench/limits/mimer.cfg +usr/sql-bench/limits/msql.cfg +usr/sql-bench/limits/ms-sql65.cfg +usr/sql-bench/limits/ms-sql.cfg +usr/sql-bench/limits/mysql-3.22.cfg +usr/sql-bench/limits/mysql-3.23.cfg +usr/sql-bench/limits/mysql-4.0.cfg +usr/sql-bench/limits/mysql-4.1.cfg +usr/sql-bench/limits/mysql.cfg +usr/sql-bench/limits/oracle.cfg +usr/sql-bench/limits/pg.cfg +usr/sql-bench/limits/solid.cfg +usr/sql-bench/limits/solid-nt4.cfg +usr/sql-bench/limits/sybase.cfg +usr/sql-bench/myisam.cnf +usr/sql-bench/README +usr/sql-bench/run-all-tests +usr/sql-bench/server-cfg +usr/sql-bench/test-alter-table +usr/sql-bench/test-ATIS +usr/sql-bench/test-big-tables +usr/sql-bench/test-connect +usr/sql-bench/test-create +usr/sql-bench/test-insert +usr/sql-bench/test-select +usr/sql-bench/test-table-elimination +usr/sql-bench/test-transactions +usr/sql-bench/test-wisconsin diff --git a/debian/rules b/debian/rules index 76433d47576..b4ef2e5da6d 100755 --- a/debian/rules +++ b/debian/rules @@ -39,7 +39,7 @@ else endif # Ignore test suite exit code on unstable platforms -ifneq (,$(filter $(ARCH), mips mipsel)) +ifneq (,$(filter $(ARCH),mips mipsel mips64el alpha powerpc sh4 hurd-i386 sparc64 kfreebsd-i386 kfreebsd-amd64)) TESTSUITE_FAIL_CMD:=true else TESTSUITE_FAIL_CMD:=exit 1 @@ -114,11 +114,6 @@ override_dh_auto_install: dh_testdir dh_testroot -# Skip TokuDB if arch is not amd64 -ifneq ($(ARCH), amd64) - sed -i -e "/Package: mariadb-plugin-tokudb/,+18d" debian/control -endif - # If libthrift-dev was available (manually installed, as it is # not in Debian) and ha_cassandra.so was thus built, create package, # otherwise skip it. @@ -153,13 +148,13 @@ endif install -D -m 644 debian/mariadb-server-10.3.py $(TMP)/usr/share/apport/package-hooks/source_mariadb-10.3.py # Install libmariadbclient18 compatibility links - ln -s /usr/lib/$(DEB_HOST_MULTIARCH)/libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmariadbclient.so - ln -s /usr/lib/$(DEB_HOST_MULTIARCH)/libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmariadbclient.so.18 + ln -s libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmariadbclient.so + ln -s libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmariadbclient.so.18 # Install libmysqlclientclientXX compatibility links - ln -s /usr/lib/$(DEB_HOST_MULTIARCH)/libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmysqlclient.so.18 - ln -s /usr/lib/$(DEB_HOST_MULTIARCH)/libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmysqlclient.so.19 - ln -s /usr/lib/$(DEB_HOST_MULTIARCH)/libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmysqlclient.so.20 + ln -s libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmysqlclient.so.18 + ln -s libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmysqlclient.so.19 + ln -s libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmysqlclient.so.20 touch $@ @@ -182,7 +177,11 @@ override_dh_installcron-arch: get-orig-source: uscan --force-download --verbose +# If a file is not supposed to be included anywhere, add it to the not-installed +# file and document the reason. Note that dh_install supports the above mentioned +# white list file only starting from Debian Stretch and Ubuntu Xenial. +# To find more, grep build logs for 'but is not installed to anywhere'. %: - dh $@ --parallel --with dpatch --with systemd + dh $@ --parallel --with dpatch --with systemd --list-missing # vim: ts=8 diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 20919cf3a81..d9149321702 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -1461,6 +1461,9 @@ innodb_init_param(void) srv_page_size = 0; srv_page_size_shift = 0; +#ifdef BTR_CUR_HASH_ADAPT + btr_ahi_parts = 1; +#endif /* BTR_CUR_HASH_ADAPT */ if (innobase_page_size != (1LL << 14)) { int n_shift = (int)get_bit_shift((ulint) innobase_page_size); diff --git a/include/m_ctype.h b/include/m_ctype.h index b78443f88f8..da52b4c15a0 100644 --- a/include/m_ctype.h +++ b/include/m_ctype.h @@ -376,7 +376,7 @@ typedef size_t (*my_charset_conv_case)(CHARSET_INFO *, A structure to return the statistics of a native string copying, when no Unicode conversion is involved. - The stucture is OK to be unitialized before calling a copying routine. + The stucture is OK to be uninitialized before calling a copying routine. A copying routine must populate the structure as follows: - m_source_end_pos must be set by to a non-NULL value in the range of the input string. @@ -425,7 +425,7 @@ struct my_charset_handler_st my_charset_conv_case caseup; my_charset_conv_case casedn; - /* Charset dependant snprintf() */ + /* Charset dependent snprintf() */ size_t (*snprintf)(CHARSET_INFO *, char *to, size_t n, const char *fmt, ...) ATTRIBUTE_FORMAT_FPTR(printf, 4, 5); diff --git a/include/my_atomic.h b/include/my_atomic.h index 896dc2b5c33..1f5d85fcf12 100644 --- a/include/my_atomic.h +++ b/include/my_atomic.h @@ -101,7 +101,7 @@ sequentially-consistent operation ordering. We choose implementation as follows: on Windows using Visual C++ the native - implementation should be preferrable. When using gcc we prefer the Solaris + implementation should be preferable. When using gcc we prefer the Solaris implementation before the gcc because of stability preference, we choose gcc builtins if available. */ diff --git a/include/my_dbug.h b/include/my_dbug.h index d93c86141d6..f0d4acc358b 100644 --- a/include/my_dbug.h +++ b/include/my_dbug.h @@ -209,10 +209,14 @@ void debug_sync_point(const char* lock_name, uint lock_timeout); #ifdef __cplusplus } +/* + DBUG_LOG() was initially intended for InnoDB. To be able to use it elsewhere + one should #include <sstream>. We intentially avoid including it here to save + compilation time. +*/ # ifdef DBUG_OFF # define DBUG_LOG(keyword, v) do {} while (0) # else -# include <sstream> # define DBUG_LOG(keyword, v) do { \ if (_db_pargs_(__LINE__, keyword)) { \ std::ostringstream _db_s; _db_s << v; \ diff --git a/include/my_global.h b/include/my_global.h index 7986a0e21dd..a96c947cf44 100644 --- a/include/my_global.h +++ b/include/my_global.h @@ -178,7 +178,7 @@ /* The macros below are borrowed from include/linux/compiler.h in the - Linux kernel. Use them to indicate the likelyhood of the truthfulness + Linux kernel. Use them to indicate the likelihood of the truthfulness of a condition. This serves two purposes - newer versions of gcc will be able to optimize for branch predication, which could yield siginficant performance gains in frequently executed sections of the code, and the @@ -527,7 +527,7 @@ typedef SOCKET my_socket; typedef int my_socket; /* File descriptor for sockets */ #define INVALID_SOCKET -1 #endif -/* Type for fuctions that handles signals */ +/* Type for functions that handles signals */ #define sig_handler RETSIGTYPE C_MODE_START #ifdef HAVE_SIGHANDLER_T @@ -857,7 +857,7 @@ static inline double log2(double x) /* Max size that must be added to a so that we know Size to make - adressable obj. + addressable obj. */ #if SIZEOF_CHARP == 4 typedef long my_ptrdiff_t; @@ -869,7 +869,7 @@ typedef long long my_ptrdiff_t; #define MY_ALIGN_DOWN(A,L) ((A) & ~((L) - 1)) #define ALIGN_SIZE(A) MY_ALIGN((A),sizeof(double)) #define ALIGN_MAX_UNIT (sizeof(double)) -/* Size to make adressable obj. */ +/* Size to make addressable obj. */ #define ALIGN_PTR(A, t) ((t*) MY_ALIGN((A), sizeof(double))) #define ADD_TO_PTR(ptr,size,type) (type) ((uchar*) (ptr)+size) #define PTR_BYTE_DIFF(A,B) (my_ptrdiff_t) ((uchar*) (A) - (uchar*) (B)) diff --git a/include/my_pthread.h b/include/my_pthread.h index 037ac0a062f..ecb201a60da 100644 --- a/include/my_pthread.h +++ b/include/my_pthread.h @@ -196,7 +196,7 @@ int sigwait(sigset_t *set, int *sig); #endif #if !defined(HAVE_SIGWAIT) && !defined(HAVE_rts_threads) && !defined(sigwait) && !defined(alpha_linux_port) && !defined(_AIX) -int sigwait(sigset_t *setp, int *sigp); /* Use our implemention */ +int sigwait(sigset_t *setp, int *sigp); /* Use our implementation */ #endif @@ -393,7 +393,7 @@ typedef struct st_safe_mutex_deadlock_t #ifdef SAFE_MUTEX_DETECT_DESTROY /* - Used to track the destroying of mutexes. This needs to be a seperate + Used to track the destroying of mutexes. This needs to be a separate structure because the safe_mutex_t structure could be freed before the mutexes are destroyed. */ diff --git a/include/mysql/service_thd_rnd.h b/include/mysql/service_thd_rnd.h index 21133c7889f..13fc1c16839 100644 --- a/include/mysql/service_thd_rnd.h +++ b/include/mysql/service_thd_rnd.h @@ -18,7 +18,7 @@ @file This service provides access to the thd-local random number generator. - It's preferrable over the global one, because concurrent threads + It's preferable over the global one, because concurrent threads can generate random numbers without fighting each other over the access to the shared rnd state. */ diff --git a/include/mysql_com.h b/include/mysql_com.h index 6db7ef58c6f..7d0190b1ca7 100644 --- a/include/mysql_com.h +++ b/include/mysql_com.h @@ -152,7 +152,7 @@ enum enum_indicator_type */ #define SCRAMBLE_LENGTH 20 #define SCRAMBLE_LENGTH_323 8 -/* length of password stored in the db: new passwords are preceeded with '*' */ +/* length of password stored in the db: new passwords are preceded with '*' */ #define SCRAMBLED_PASSWORD_CHAR_LENGTH (SCRAMBLE_LENGTH*2+1) #define SCRAMBLED_PASSWORD_CHAR_LENGTH_323 (SCRAMBLE_LENGTH_323*2) @@ -299,7 +299,7 @@ enum enum_indicator_type #endif /* - Gather all possible capabilites (flags) supported by the server + Gather all possible capabilities (flags) supported by the server MARIADB_* flags supported only by MariaDB connector(s). */ diff --git a/libmariadb b/libmariadb -Subproject fe129ed39f33ba2b430aac91473baee84de88b1 +Subproject f3944bbd36af729b2f13313587018679c57de67 diff --git a/man/mysql-test-run.pl.1 b/man/mysql-test-run.pl.1 index 93fb42e251a..b928ec66587 100644 --- a/man/mysql-test-run.pl.1 +++ b/man/mysql-test-run.pl.1 @@ -276,7 +276,7 @@ T}:T{ Setting of a timeout in minutes or seconds, corresponding to command line option \fB\-\-\fR\fB\fIname\fR\fR\fB\-timeout\fR\&. - Avaliable timeout names are TESTCASE, + Available timeout names are TESTCASE, SUITE (both in minutes) and START, SHUTDOWN (both in seconds)\&. These variables are supported from @@ -1350,7 +1350,7 @@ for a description\&. .\" noreorder option: mysql-test-run.pl \fB\-\-noreorder\fR .sp -Do not reorder tests to reduce number of restarts, but run them in exactly the order given\&. If a whole suite is to be run, the tests are run in alphabetical order, though similiar combinations will be grouped together\&. If more than one suite is listed, the tests are run one suite at a time, in the order listed\&. +Do not reorder tests to reduce number of restarts, but run them in exactly the order given\&. If a whole suite is to be run, the tests are run in alphabetical order, though similar combinations will be grouped together\&. If more than one suite is listed, the tests are run one suite at a time, in the order listed\&. .RE .sp .RS 4 diff --git a/man/mysql.1 b/man/mysql.1 index b7f7a63bff4..f925fee454b 100644 --- a/man/mysql.1 +++ b/man/mysql.1 @@ -698,7 +698,7 @@ the section called \(lqMYSQL COMMANDS\(rq\&. Disabled by default\&. .\} .\" mysql: net-buffer-length option .\" net-buffer-length option: mysql -\fB\-\-net\-buffer\-lenght=\fR\fB\fIsize\fR\fR +\fB\-\-net\-buffer\-length=\fR\fB\fIsize\fR\fR .sp Set the buffer size for TCP/IP and socket communication\&. (Default value is 16KB\&.) .RE diff --git a/mysql-test/lib/My/SafeProcess/safe_kill_win.cc b/mysql-test/lib/My/SafeProcess/safe_kill_win.cc index bb884cba11e..6ca38ceee81 100644 --- a/mysql-test/lib/My/SafeProcess/safe_kill_win.cc +++ b/mysql-test/lib/My/SafeProcess/safe_kill_win.cc @@ -26,7 +26,20 @@ #include <signal.h> #include <stdlib.h> #include <psapi.h> + +#ifdef _MSC_VER +/* Silence warning in OS header dbghelp.h */ +#pragma warning(push) +#pragma warning(disable : 4091) +#endif + #include <dbghelp.h> + +#ifdef _MSC_VER +/* Silence warning in OS header dbghelp.h */ +#pragma warning(pop) +#endif + #include <tlhelp32.h> #include <vector> diff --git a/mysql-test/r/cte_nonrecursive.result b/mysql-test/r/cte_nonrecursive.result index a473f96bb57..d944843f2d0 100644 --- a/mysql-test/r/cte_nonrecursive.result +++ b/mysql-test/r/cte_nonrecursive.result @@ -1265,3 +1265,33 @@ a 4 deallocate prepare stmt; drop table t1; +# +# MDEV-14852: CTE using temporary table in query +# with two references to the CTE +# +create temporary table t1 (i int); +insert into t1 values (5),(4),(1),(2),(3); +with +c1 as (select i from t1), +c2 as (select i from c1 where c1.i=2) +select i from c1 where i > 3 union select i from c2; +i +5 +4 +2 +drop table t1; +create table t1 (term char(10)); +create temporary table t2 (term char(10)); +insert into t1 values ('TERM01'),('TERM02'),('TERM03'); +insert into t2 values ('TERM02'),('TERM03'),('TERM04'); +with c1 as (select * from t1), c2 as (select * from t2) +(select * from c1 left outer join c2 on c1.term = c2.term) +union all +(select * from c1 right outer join c2 on c1.term = c2.term +where c1.term is null); +term term +TERM02 TERM02 +TERM03 TERM03 +TERM01 NULL +NULL TERM04 +drop table t1,t2; diff --git a/mysql-test/r/cte_recursive.result b/mysql-test/r/cte_recursive.result index bf84c93ebe0..d87802f57f3 100644 --- a/mysql-test/r/cte_recursive.result +++ b/mysql-test/r/cte_recursive.result @@ -2951,6 +2951,112 @@ limit 1 ERROR HY000: Unacceptable mutual recursion with anchored table 'cte_1' drop table t1; # +# mdev-14777: crash caused by the same as in mdev-14755 +# +CREATE TABLE t1 (i1 int NOT NULL, i2 int); +CREATE TABLE t2 (d1 int NOT NULL PRIMARY KEY); +CREATE TABLE t3 (i int ); +insert into t1 select seq,seq from seq_1_to_100000; +insert into t2 select seq from seq_1000_to_100000; +insert into t3 select seq from seq_1_to_1000; +SELECT * +FROM +( +SELECT * +FROM +( +WITH RECURSIVE rt AS +( +SELECT i2 P, i1 C FROM t1 WHERE i1 IN (SELECT d1 FROM t2) +UNION +SELECT t1.i2 P, rt.C C FROM t1, rt +) +SELECT C,P +FROM ( SELECT P,C FROM rt WHERE NOT EXISTS (SELECT 1 FROM t1) ) Y +) X +WHERE 1 = 1 +) K, t3; +C P i +drop table t1,t2,t3; +# +# mdev-14879: subquery with recursive reference in WHERE of CTE +# +create table flights +(departure varchar(32), +arrival varchar(32), +carrier varchar(20), +flight_number char(7)); +insert into flights values +('Seattle', 'Frankfurt', 'Lufthansa', 'LH 491'), +('Seattle', 'Chicago', 'American', 'AA 2573'), +('Seattle', 'Los Angeles', 'Alaska Air', 'AS 410'), +('Chicago', 'New York', 'American', 'AA 375'), +('Chicago', 'Montreal', 'Air Canada', 'AC 3053'), +('Los Angeles', 'New York', 'Delta', 'DL 1197'), +('Moscow', 'Tokyo', 'Aeroflot', 'SU 264'), +('New York', 'Paris', 'Air France', 'AF 23'), +('Frankfurt', 'Moscow', 'Lufthansa', 'LH 1444'), +('Tokyo', 'Seattle', 'ANA', 'NH 178'), +('Los Angeles', 'Tokyo', 'ANA', 'NH 175'), +('Moscow', 'Los Angeles', 'Aeroflot', 'SU 106'), +('Montreal', 'Paris', 'Air Canada', 'AC 870'), +('Cairo', 'Paris', 'Air France', 'AF 503'), +('New York', 'Seattle', 'American', 'AA 45'), +('Paris', 'Chicago', 'Air France', 'AF 6734'); +with recursive destinations (city) as +( select a.arrival from flights a where a.departure='Cairo' + union +select b.arrival from destinations r, flights b where r.city=b.departure) +select * from destinations; +city +Paris +Chicago +New York +Montreal +Seattle +Frankfurt +Los Angeles +Moscow +Tokyo +set standard_compliant_cte=0; +with recursive destinations (city, legs) as +( +select a.arrival, 1 from flights a where a.departure='Cairo' + union +select b.arrival, r.legs + 1 from destinations r, flights b +where r.city=b.departure and b.arrival not in (select city from destinations) +) +select * from destinations; +city legs +Paris 1 +Chicago 2 +New York 3 +Montreal 3 +Seattle 4 +Frankfurt 5 +Los Angeles 5 +Moscow 6 +Tokyo 6 +explain extended with recursive destinations (city, legs) as +( +select a.arrival, 1 from flights a where a.departure='Cairo' + union +select b.arrival, r.legs + 1 from destinations r, flights b +where r.city=b.departure and b.arrival not in (select city from destinations) +) +select * from destinations; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY <derived2> ALL NULL NULL NULL NULL 16 100.00 +2 DERIVED a ALL NULL NULL NULL NULL 16 100.00 Using where +3 RECURSIVE UNION b ALL NULL NULL NULL NULL 16 100.00 Using where +3 RECURSIVE UNION <derived2> ref key0 key0 35 test.b.departure 2 100.00 +4 DEPENDENT SUBQUERY <derived2> ALL NULL NULL NULL NULL 16 100.00 Using where +NULL UNION RESULT <union2,3> ALL NULL NULL NULL NULL NULL NULL +Warnings: +Note 1003 with recursive destinations as (/* select#2 */ select `test`.`a`.`arrival` AS `city`,1 AS `legs` from `test`.`flights` `a` where `test`.`a`.`departure` = 'Cairo' union /* select#3 */ select `test`.`b`.`arrival` AS `arrival`,`r`.`legs` + 1 AS `r.legs + 1` from `destinations` `r` join `test`.`flights` `b` where `r`.`city` = `test`.`b`.`departure` and !<in_optimizer>(`test`.`b`.`arrival`,<exists>(/* select#4 */ select `destinations`.`city` from `destinations` where trigcond(`test`.`b`.`arrival` = `destinations`.`city` or `destinations`.`city` is null) having trigcond(`destinations`.`city` is null))))/* select#1 */ select `destinations`.`city` AS `city`,`destinations`.`legs` AS `legs` from `destinations` +set standard_compliant_cte=default; +drop table flights; +# # MDEV-14217 [db crash] Recursive CTE when SELECT includes new field # CREATE TEMPORARY TABLE a_tbl ( diff --git a/mysql-test/r/derived_cond_pushdown.result b/mysql-test/r/derived_cond_pushdown.result index 0eb12033dd3..2d4638fe4f6 100644 --- a/mysql-test/r/derived_cond_pushdown.result +++ b/mysql-test/r/derived_cond_pushdown.result @@ -12540,6 +12540,70 @@ EXPLAIN } drop table t1; # +# MDEV-13454: consequence of mdev-14368 fixed for 5.5 +# +SET sql_mode = 'ONLY_FULL_GROUP_BY'; +create table t1 (id int, id2 int); +insert into t1 values (1,1),(2,3),(3,4),(7,2); +create table t2(id2 int); +insert t2 values (1),(2),(3); +SELECT * FROM t1 +LEFT OUTER JOIN +(SELECT id2, COUNT(*) as ct FROM t2 GROUP BY id2) vc USING (id2) +WHERE (vc.ct>0); +id2 id ct +1 1 1 +3 2 1 +2 7 1 +EXPLAIN FORMAT=JSON SELECT * FROM t1 +LEFT OUTER JOIN +(SELECT id2, COUNT(*) as ct FROM t2 GROUP BY id2) vc USING (id2) +WHERE (vc.ct>0); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "<derived2>", + "access_type": "ALL", + "rows": 3, + "filtered": 100, + "attached_condition": "vc.ct > 0", + "materialized": { + "query_block": { + "select_id": 2, + "having_condition": "ct > 0", + "filesort": { + "sort_key": "t2.id2", + "temporary_table": { + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 3, + "filtered": 100 + } + } + } + } + } + }, + "block-nl-join": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 4, + "filtered": 100 + }, + "buffer_type": "flat", + "buffer_size": "256Kb", + "join_type": "BNL", + "attached_condition": "t1.id2 = vc.id2" + } + } +} +DROP TABLE t1,t2; +SET sql_mode = DEFAULT; +# # MDEV-10855: Pushdown into derived with window functions # set @save_optimizer_switch= @@optimizer_switch; diff --git a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result index f9b53940df4..1c087a3199c 100644 --- a/mysql-test/r/subselect.result +++ b/mysql-test/r/subselect.result @@ -7214,6 +7214,32 @@ NULL # SELECT (SELECT MAX(sq.f2) FROM t1) FROM (SELECT * FROM t2) AS sq WHERE f2 = 2; # drop table t1, t2; +# +# MDEV-13933: Wrong results in COUNT() query with EXISTS and exists_to_in +# (5.5 test) +# +SET @optimiser_switch_save= @@optimizer_switch; +CREATE TABLE t1 (a INT NOT NULL); +INSERT INTO t1 VALUES (1),(1),(1),(5),(5); +CREATE TABLE t2 (b INT); +INSERT INTO t2 VALUES (5),(1); +CREATE TABLE t3 (c INT, KEY(c)); +INSERT INTO t3 VALUES (5),(5); +SET optimizer_switch='semijoin=on'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET optimizer_switch='semijoin=off'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET @@optimizer_switch= @optimiser_switch_save; +DROP TABLE t1, t2, t3; +End of 5.5 tests # End of 10.0 tests # # MDEV-9487: Server crashes in Time_and_counter_tracker::incr_loops diff --git a/mysql-test/r/subselect_no_exists_to_in.result b/mysql-test/r/subselect_no_exists_to_in.result index 5c8230e7493..eb912d9e331 100644 --- a/mysql-test/r/subselect_no_exists_to_in.result +++ b/mysql-test/r/subselect_no_exists_to_in.result @@ -7214,6 +7214,32 @@ NULL # SELECT (SELECT MAX(sq.f2) FROM t1) FROM (SELECT * FROM t2) AS sq WHERE f2 = 2; # drop table t1, t2; +# +# MDEV-13933: Wrong results in COUNT() query with EXISTS and exists_to_in +# (5.5 test) +# +SET @optimiser_switch_save= @@optimizer_switch; +CREATE TABLE t1 (a INT NOT NULL); +INSERT INTO t1 VALUES (1),(1),(1),(5),(5); +CREATE TABLE t2 (b INT); +INSERT INTO t2 VALUES (5),(1); +CREATE TABLE t3 (c INT, KEY(c)); +INSERT INTO t3 VALUES (5),(5); +SET optimizer_switch='semijoin=on'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET optimizer_switch='semijoin=off'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET @@optimizer_switch= @optimiser_switch_save; +DROP TABLE t1, t2, t3; +End of 5.5 tests # End of 10.0 tests # # MDEV-9487: Server crashes in Time_and_counter_tracker::incr_loops diff --git a/mysql-test/r/subselect_no_mat.result b/mysql-test/r/subselect_no_mat.result index 8b395fae5ce..72f30bbd21f 100644 --- a/mysql-test/r/subselect_no_mat.result +++ b/mysql-test/r/subselect_no_mat.result @@ -7207,6 +7207,32 @@ NULL # SELECT (SELECT MAX(sq.f2) FROM t1) FROM (SELECT * FROM t2) AS sq WHERE f2 = 2; # drop table t1, t2; +# +# MDEV-13933: Wrong results in COUNT() query with EXISTS and exists_to_in +# (5.5 test) +# +SET @optimiser_switch_save= @@optimizer_switch; +CREATE TABLE t1 (a INT NOT NULL); +INSERT INTO t1 VALUES (1),(1),(1),(5),(5); +CREATE TABLE t2 (b INT); +INSERT INTO t2 VALUES (5),(1); +CREATE TABLE t3 (c INT, KEY(c)); +INSERT INTO t3 VALUES (5),(5); +SET optimizer_switch='semijoin=on'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET optimizer_switch='semijoin=off'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET @@optimizer_switch= @optimiser_switch_save; +DROP TABLE t1, t2, t3; +End of 5.5 tests # End of 10.0 tests # # MDEV-9487: Server crashes in Time_and_counter_tracker::incr_loops diff --git a/mysql-test/r/subselect_no_opts.result b/mysql-test/r/subselect_no_opts.result index fbc5fccfb9b..de075d3245f 100644 --- a/mysql-test/r/subselect_no_opts.result +++ b/mysql-test/r/subselect_no_opts.result @@ -7205,6 +7205,32 @@ NULL # SELECT (SELECT MAX(sq.f2) FROM t1) FROM (SELECT * FROM t2) AS sq WHERE f2 = 2; # drop table t1, t2; +# +# MDEV-13933: Wrong results in COUNT() query with EXISTS and exists_to_in +# (5.5 test) +# +SET @optimiser_switch_save= @@optimizer_switch; +CREATE TABLE t1 (a INT NOT NULL); +INSERT INTO t1 VALUES (1),(1),(1),(5),(5); +CREATE TABLE t2 (b INT); +INSERT INTO t2 VALUES (5),(1); +CREATE TABLE t3 (c INT, KEY(c)); +INSERT INTO t3 VALUES (5),(5); +SET optimizer_switch='semijoin=on'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET optimizer_switch='semijoin=off'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET @@optimizer_switch= @optimiser_switch_save; +DROP TABLE t1, t2, t3; +End of 5.5 tests # End of 10.0 tests # # MDEV-9487: Server crashes in Time_and_counter_tracker::incr_loops diff --git a/mysql-test/r/subselect_no_scache.result b/mysql-test/r/subselect_no_scache.result index 446d2649d1f..a594f5f85b9 100644 --- a/mysql-test/r/subselect_no_scache.result +++ b/mysql-test/r/subselect_no_scache.result @@ -7220,6 +7220,32 @@ NULL # SELECT (SELECT MAX(sq.f2) FROM t1) FROM (SELECT * FROM t2) AS sq WHERE f2 = 2; # drop table t1, t2; +# +# MDEV-13933: Wrong results in COUNT() query with EXISTS and exists_to_in +# (5.5 test) +# +SET @optimiser_switch_save= @@optimizer_switch; +CREATE TABLE t1 (a INT NOT NULL); +INSERT INTO t1 VALUES (1),(1),(1),(5),(5); +CREATE TABLE t2 (b INT); +INSERT INTO t2 VALUES (5),(1); +CREATE TABLE t3 (c INT, KEY(c)); +INSERT INTO t3 VALUES (5),(5); +SET optimizer_switch='semijoin=on'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET optimizer_switch='semijoin=off'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET @@optimizer_switch= @optimiser_switch_save; +DROP TABLE t1, t2, t3; +End of 5.5 tests # End of 10.0 tests # # MDEV-9487: Server crashes in Time_and_counter_tracker::incr_loops diff --git a/mysql-test/r/subselect_no_semijoin.result b/mysql-test/r/subselect_no_semijoin.result index 97ff454cf51..e068b28b017 100644 --- a/mysql-test/r/subselect_no_semijoin.result +++ b/mysql-test/r/subselect_no_semijoin.result @@ -7205,6 +7205,32 @@ NULL # SELECT (SELECT MAX(sq.f2) FROM t1) FROM (SELECT * FROM t2) AS sq WHERE f2 = 2; # drop table t1, t2; +# +# MDEV-13933: Wrong results in COUNT() query with EXISTS and exists_to_in +# (5.5 test) +# +SET @optimiser_switch_save= @@optimizer_switch; +CREATE TABLE t1 (a INT NOT NULL); +INSERT INTO t1 VALUES (1),(1),(1),(5),(5); +CREATE TABLE t2 (b INT); +INSERT INTO t2 VALUES (5),(1); +CREATE TABLE t3 (c INT, KEY(c)); +INSERT INTO t3 VALUES (5),(5); +SET optimizer_switch='semijoin=on'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET optimizer_switch='semijoin=off'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET @@optimizer_switch= @optimiser_switch_save; +DROP TABLE t1, t2, t3; +End of 5.5 tests # End of 10.0 tests # # MDEV-9487: Server crashes in Time_and_counter_tracker::incr_loops diff --git a/mysql-test/suite/binlog/r/binlog_killed.result b/mysql-test/suite/binlog/r/binlog_killed.result index d751b1f9002..ae851c38e5f 100644 --- a/mysql-test/suite/binlog/r/binlog_killed.result +++ b/mysql-test/suite/binlog/r/binlog_killed.result @@ -185,90 +185,88 @@ connection con3; MI: MyISAM, INNODB BEGIN; INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); +INSERT INTO t1 VALUES (NULL, 2); connection con1; KILL ID; include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info master-bin.000002 # Format_desc # # SERVER_VERSION, BINLOG_VERSION master-bin.000002 # Gtid_list # # [#-#-#] -master-bin.000002 # Binlog_checkpoint # # master-bin.000001 -master-bin.000002 # Binlog_checkpoint # # master-bin.000002 master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=3 master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 1) master-bin.000002 # Query # # COMMIT master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=4 -master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 2) master-bin.000002 # Query # # ROLLBACK disconnect con3; connect con3, localhost, root,,; connection con3; IM: INNODB, MyISAM BEGIN; -INSERT INTO t1 VALUES (NULL, 1); -INSERT INTO t2 VALUES (NULL, 1); +INSERT INTO t1 VALUES (NULL, 3); +INSERT INTO t2 VALUES (NULL, 4); connection con1; KILL ID; include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=4 -master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 4) master-bin.000002 # Query # # COMMIT master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=5 -master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 3) master-bin.000002 # Query # # ROLLBACK disconnect con3; connect con3, localhost, root,,; connection con3; IMI: INNODB, MyISAM, INNODB BEGIN; -INSERT INTO t1 VALUES (NULL, 1); -INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); +INSERT INTO t1 VALUES (NULL, 5); +INSERT INTO t2 VALUES (NULL, 6); +INSERT INTO t1 VALUES (NULL, 7); connection con1; KILL ID; include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=5 -master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 6) master-bin.000002 # Query # # COMMIT master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=6 -master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 5) master-bin.000002 # Intvar # # INSERT_ID=7 -master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 7) master-bin.000002 # Query # # ROLLBACK disconnect con3; connect con3, localhost, root,,; connection con3; MI2: MyISAM, INNODB, MyISAM, INNODB BEGIN; -INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); -INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); +INSERT INTO t2 VALUES (NULL, 8); +INSERT INTO t1 VALUES (NULL, 9); +INSERT INTO t2 VALUES (NULL, 10); +INSERT INTO t1 VALUES (NULL, 11); connection con1; KILL ID; include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=6 -master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 8) master-bin.000002 # Query # # COMMIT master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=7 -master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 10) master-bin.000002 # Query # # COMMIT master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=8 -master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 9) master-bin.000002 # Intvar # # INSERT_ID=9 -master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 11) master-bin.000002 # Query # # ROLLBACK connection default; disconnect con1; diff --git a/mysql-test/suite/binlog/t/binlog_killed.test b/mysql-test/suite/binlog/t/binlog_killed.test index 8ac223603a3..7c3a262d2c1 100644 --- a/mysql-test/suite/binlog/t/binlog_killed.test +++ b/mysql-test/suite/binlog/t/binlog_killed.test @@ -10,6 +10,13 @@ call mtr.add_suppression("Unsafe statement written to the binary log using state -- source include/not_embedded.inc +# +# Avoid printing binlog checkpoints +# + +--let $skip_checkpoint_events=1 + + ### ### bug#22725 : incorrect killed error in binlogged query ### @@ -362,16 +369,20 @@ connect (con3, localhost, root,,); connection con3; let $ID= `select connection_id()`; +--let $threads_connected=`select variable_value from information_schema.global_status where variable_name="threads_connected"` +--let wait_condition=select variable_value < $threads_connected from information_schema.global_status where variable_name="threads_connected" + --echo MI: MyISAM, INNODB BEGIN; INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); +INSERT INTO t1 VALUES (NULL, 2); #Connection con1 as killer throughout the test connection con1; --replace_result $ID ID --eval KILL $ID let $binlog_file= query_get_value(SHOW MASTER STATUS, File, 1); +--source include/wait_condition.inc --let $binlog_start= 4 --source include/show_binlog_events.inc --let $binlog_killed_pos=query_get_value(SHOW MASTER STATUS, Position, 1) @@ -383,12 +394,13 @@ let $ID= `select connection_id()`; --echo IM: INNODB, MyISAM BEGIN; -INSERT INTO t1 VALUES (NULL, 1); -INSERT INTO t2 VALUES (NULL, 1); +INSERT INTO t1 VALUES (NULL, 3); +INSERT INTO t2 VALUES (NULL, 4); connection con1; --replace_result $ID ID --eval KILL $ID +--source include/wait_condition.inc --let $binlog_start= $binlog_killed_pos --source include/show_binlog_events.inc --let $binlog_killed_pos=query_get_value(SHOW MASTER STATUS, Position, 1) @@ -400,13 +412,14 @@ let $ID= `select connection_id()`; --echo IMI: INNODB, MyISAM, INNODB BEGIN; -INSERT INTO t1 VALUES (NULL, 1); -INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); +INSERT INTO t1 VALUES (NULL, 5); +INSERT INTO t2 VALUES (NULL, 6); +INSERT INTO t1 VALUES (NULL, 7); connection con1; --replace_result $ID ID --eval KILL $ID +--source include/wait_condition.inc --let $binlog_start= $binlog_killed_pos --source include/show_binlog_events.inc --let $binlog_killed_pos=query_get_value(SHOW MASTER STATUS, Position, 1) @@ -418,14 +431,15 @@ let $ID= `select connection_id()`; --echo MI2: MyISAM, INNODB, MyISAM, INNODB BEGIN; -INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); -INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); +INSERT INTO t2 VALUES (NULL, 8); +INSERT INTO t1 VALUES (NULL, 9); +INSERT INTO t2 VALUES (NULL, 10); +INSERT INTO t1 VALUES (NULL, 11); connection con1; --replace_result $ID ID --eval KILL $ID +--source include/wait_condition.inc --let $binlog_start= $binlog_killed_pos --source include/show_binlog_events.inc diff --git a/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result b/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result index afba16db42c..23e8a1c98c8 100644 --- a/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result +++ b/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result @@ -17,6 +17,13 @@ WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS FOUND 1 /InnoDB: Upgrade after a crash is not supported. This redo log was created before MariaDB 10\.2\.2, and it appears corrupted/ in mysqld.1.err +# empty redo log from before MariaDB 10.2.2 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +COUNT(*) +1 +FOUND 1 /InnoDB: Upgrading redo log:/ in mysqld.1.err # redo log from "after" MariaDB 10.2.2, but with invalid header checksum SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' @@ -93,25 +100,38 @@ AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS FOUND 1 /InnoDB: MLOG_FILE_NAME incorrect:bigot/ in mysqld.1.err FOUND 1 /len 22; hex 38000000000012860cb7809781e800066269676f7400; asc 8 bigot ;/ in mysqld.1.err -# missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT +# 10.2 missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS -NOT FOUND /InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42$/ in mysqld.1.err -# Clean 10.2 redo log +FOUND 1 /InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42/ in mysqld.1.err +# 10.3 missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS -InnoDB YES Supports transactions, row-level locking, foreign keys and encryption for tables YES YES YES -FOUND 1 /InnoDB: Upgrading redo log:/ in mysqld.1.err +FOUND 2 /InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42/ in mysqld.1.err +# Empty 10.3 redo log +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +COUNT(*) +1 +FOUND 1 /InnoDB: .* started; log sequence number 1213970/ in mysqld.1.err +# Empty 10.2 redo log +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +COUNT(*) +1 +FOUND 2 /InnoDB: Upgrading redo log:/ in mysqld.1.err # Minimal MariaDB 10.1.21 encrypted redo log SELECT COUNT(*) `1` FROM INFORMATION_SCHEMA.ENGINES WHERE engine='innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); 1 1 -FOUND 1 /InnoDB: Encrypting redo log/ in mysqld.1.err +FOUND 2 /InnoDB: Encrypting redo log/ in mysqld.1.err ib_buffer_pool ib_logfile0 ib_logfile1 diff --git a/mysql-test/suite/innodb/r/deadlock_detect.result b/mysql-test/suite/innodb/r/deadlock_detect.result index c3e3794ed21..4e14eff34b2 100644 --- a/mysql-test/suite/innodb/r/deadlock_detect.result +++ b/mysql-test/suite/innodb/r/deadlock_detect.result @@ -8,21 +8,15 @@ PRIMARY KEY(id) INSERT INTO t1 VALUES(1), (2), (3); BEGIN; SELECT * FROM t1 WHERE id = 1 FOR UPDATE; -id -1 connect con1,localhost,root,,; BEGIN; SELECT * FROM t1 WHERE id = 2 FOR UPDATE; -id -2 SELECT * FROM t1 WHERE id = 1 FOR UPDATE; connection default; SELECT * FROM t1 WHERE id = 2 FOR UPDATE; connection con1; -ERROR HY000: Lock wait timeout exceeded; try restarting transaction ROLLBACK; connection default; -ERROR HY000: Lock wait timeout exceeded; try restarting transaction ROLLBACK; DROP TABLE t1; disconnect con1; diff --git a/mysql-test/suite/innodb/r/innodb-index-online.result b/mysql-test/suite/innodb/r/innodb-index-online.result index 5427f13043d..3134c64c8c6 100644 --- a/mysql-test/suite/innodb/r/innodb-index-online.result +++ b/mysql-test/suite/innodb/r/innodb-index-online.result @@ -470,8 +470,6 @@ ddl_online_create_index 0 ddl_pending_alter_table 0 ddl_sort_file_alter_table 0 ddl_log_file_alter_table 2 -connection con1; -disconnect con1; connection default; SHOW CREATE TABLE t1; Table Create Table @@ -498,6 +496,25 @@ ERROR 42000: Duplicate key name 'c2h' SET DEBUG_SYNC = 'RESET'; SET GLOBAL innodb_monitor_disable = module_ddl; DROP TABLE t1; +# +# MDEV-13205 assertion !dict_index_is_online_ddl(index) upon ALTER TABLE +# +CREATE TABLE t1 (c VARCHAR(64)) ENGINE=InnoDB; +SET DEBUG_SYNC = 'row_log_apply_before SIGNAL t1u_created WAIT_FOR dup_done'; +ALTER TABLE t1 ADD UNIQUE(c); +connection con1; +SET DEBUG_SYNC = 'now WAIT_FOR t1u_created'; +BEGIN; +INSERT INTO t1 VALUES('bar'),('bar'); +SET DEBUG_SYNC = 'now SIGNAL dup_done'; +connection default; +ERROR 23000: Duplicate entry 'bar' for key 'c' +SET DEBUG_SYNC = 'RESET'; +disconnect con1; +CREATE TABLE t2 (c VARCHAR(64)) ENGINE=InnoDB; +ALTER TABLE t2 ADD FOREIGN KEY (c) REFERENCES t1 (c); +ERROR HY000: Can't create table `test`.`#sql-temporary` (errno: 150 "Foreign key constraint is incorrectly formed") +DROP TABLE t2,t1; SET GLOBAL innodb_file_per_table = @global_innodb_file_per_table_orig; SET GLOBAL innodb_monitor_enable = default; SET GLOBAL innodb_monitor_disable = default; diff --git a/mysql-test/suite/innodb/r/innodb-lru-force-no-free-page.result b/mysql-test/suite/innodb/r/innodb-lru-force-no-free-page.result new file mode 100644 index 00000000000..09e53b59d9d --- /dev/null +++ b/mysql-test/suite/innodb/r/innodb-lru-force-no-free-page.result @@ -0,0 +1,10 @@ +call mtr.add_suppression("InnoDB: Difficult to find free blocks in the buffer pool"); +SET @saved_debug = @@SESSION.debug_dbug; +SET SESSION debug_dbug="+d,ib_lru_force_no_free_page"; +CREATE TABLE t1 (j LONGBLOB) ENGINE = InnoDB; +BEGIN; +INSERT INTO t1 VALUES (repeat('abcdefghijklmnopqrstuvwxyz',200)); +COMMIT; +SET debug_dbug = @saved_debug; +DROP TABLE t1; +FOUND 1 /InnoDB: Difficult to find free blocks / in mysqld.1.err diff --git a/mysql-test/suite/innodb/r/log_corruption.result b/mysql-test/suite/innodb/r/log_corruption.result index 903dc7f6cf4..b783a809a35 100644 --- a/mysql-test/suite/innodb/r/log_corruption.result +++ b/mysql-test/suite/innodb/r/log_corruption.result @@ -17,6 +17,13 @@ WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS FOUND 1 /InnoDB: Upgrade after a crash is not supported. This redo log was created before MariaDB 10\.2\.2, and it appears corrupted/ in mysqld.1.err +# empty redo log from before MariaDB 10.2.2 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +COUNT(*) +1 +FOUND 1 /InnoDB: Upgrading redo log:/ in mysqld.1.err # redo log from "after" MariaDB 10.2.2, but with invalid header checksum SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' @@ -93,19 +100,32 @@ AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS FOUND 1 /InnoDB: MLOG_FILE_NAME incorrect:bigot/ in mysqld.1.err FOUND 1 /len 22; hex 38000000000012860cb7809781e800066269676f7400; asc 8 bigot ;/ in mysqld.1.err -# missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT +# 10.2 missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS -NOT FOUND /InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42$/ in mysqld.1.err -# Clean 10.2 redo log +FOUND 1 /InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42/ in mysqld.1.err +# 10.3 missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS -InnoDB YES Supports transactions, row-level locking, foreign keys and encryption for tables YES YES YES -FOUND 1 /InnoDB: Upgrading redo log:/ in mysqld.1.err +FOUND 2 /InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42/ in mysqld.1.err +# Empty 10.3 redo log +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +COUNT(*) +1 +FOUND 1 /InnoDB: .* started; log sequence number 1213970/ in mysqld.1.err +# Empty 10.2 redo log +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +COUNT(*) +1 +FOUND 2 /InnoDB: Upgrading redo log:/ in mysqld.1.err # Minimal MariaDB 10.1.21 encrypted redo log SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' diff --git a/mysql-test/suite/innodb/r/truncate_inject.result b/mysql-test/suite/innodb/r/truncate_inject.result new file mode 100644 index 00000000000..5ec532a0f83 --- /dev/null +++ b/mysql-test/suite/innodb/r/truncate_inject.result @@ -0,0 +1,114 @@ +SET @save_dbug = @@SESSION.debug_dbug; +call mtr.add_suppression("InnoDB: Flagged corruption of .* in table `test`\\.`t` in TRUNCATE TABLE"); +# 1. Error in assigning undo logs for truncate action +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), INDEX ck(c)) +ENGINE = InnoDB; +insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); +check table t; +Table Op Msg_type Msg_text +test.t check status OK +SET debug_dbug = '+d,ib_err_trunc_assigning_undo_log'; +truncate table t; +ERROR HY000: Got error 168 "Unknown (generic) error from engine" from storage engine InnoDB +SET debug_dbug = @save_dbug; +check table t; +Table Op Msg_type Msg_text +test.t check status OK +select * from t; +i f c +1 1.1 a +2 2.2 b +3 3.3 c +# 2. Error while preparing for truncate +SET debug_dbug = '+d,ib_err_trunc_preparing_for_truncate'; +truncate table t; +ERROR HY000: Got error 168 "Unknown (generic) error from engine" from storage engine InnoDB +SET debug_dbug = @save_dbug; +check table t; +Table Op Msg_type Msg_text +test.t check status OK +select * from t; +i f c +1 1.1 a +2 2.2 b +3 3.3 c +# 3. Error while dropping/creating indexes +SET debug_dbug = '+d,ib_err_trunc_drop_index'; +truncate table t; +ERROR HY000: Got error 168 "Unknown (generic) error from engine" from storage engine InnoDB +SET debug_dbug = @save_dbug; +check table t; +Table Op Msg_type Msg_text +test.t check Warning InnoDB: Index PRIMARY is marked as corrupted +test.t check error Corrupt +select * from t; +Got one of the listed errors +drop table t; +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), INDEX ck(c)) +ENGINE = InnoDB; +insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); +check table t; +Table Op Msg_type Msg_text +test.t check status OK +SET debug_dbug = '+d,ib_err_trunc_create_index'; +truncate table t; +ERROR HY000: Got error 168 "Unknown (generic) error from engine" from storage engine InnoDB +SET debug_dbug = @save_dbug; +check table t; +Table Op Msg_type Msg_text +test.t check Warning InnoDB: Index PRIMARY is marked as corrupted +test.t check error Corrupt +select * from t; +Got one of the listed errors +drop table t; +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), INDEX ck(c)) +ENGINE = InnoDB; +insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); +check table t; +Table Op Msg_type Msg_text +test.t check status OK +SET debug_dbug = '+d,ib_err_trunc_temp_recreate_index'; +truncate table t; +SET debug_dbug = @save_dbug; +check table t; +Table Op Msg_type Msg_text +test.t check status OK +select * from t; +i f c +drop table t; +# 4. Error while completing truncate of table involving FTS +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), +FULLTEXT INDEX(c)) ENGINE = InnoDB; +insert into t values (1, 1.1, 'mysql is now oracle company'), +(2, 2.2, 'innodb is part of mysql'), +(3, 3.3, 'innodb is default storage engine of mysql'); +check table t; +Table Op Msg_type Msg_text +test.t check status OK +SET debug_dbug = '+d,ib_err_trunc_temp_recreate_index'; +truncate table t; +SET debug_dbug = @save_dbug; +check table t; +Table Op Msg_type Msg_text +test.t check status OK +select * from t; +i f c +drop table t; +# 5. Error while updating sys-tables +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), +FULLTEXT INDEX(c)) ENGINE = InnoDB; +insert into t values (1, 1.1, 'mysql is now oracle company'), +(2, 2.2, 'innodb is part of mysql'), +(3, 3.3, 'innodb is default storage engine of mysql'); +check table t; +Table Op Msg_type Msg_text +test.t check status OK +SET debug_dbug = '+d,ib_err_trunc_temp_recreate_index'; +truncate table t; +SET debug_dbug = @save_dbug; +check table t; +Table Op Msg_type Msg_text +test.t check status OK +select * from t order by i; +i f c +drop table t; diff --git a/mysql-test/suite/innodb/t/deadlock_detect.test b/mysql-test/suite/innodb/t/deadlock_detect.test index 85d8c1f67f2..babdb54719f 100644 --- a/mysql-test/suite/innodb/t/deadlock_detect.test +++ b/mysql-test/suite/innodb/t/deadlock_detect.test @@ -18,6 +18,8 @@ CREATE TABLE t1( INSERT INTO t1 VALUES(1), (2), (3); +# We are not interested query results, only errors +--disable_result_log BEGIN; SELECT * FROM t1 WHERE id = 1 FOR UPDATE; @@ -39,12 +41,22 @@ reap; ROLLBACK; +# +# Note here that con1 is the older transaction as it +# query started wait first. Thus, con1 gets lock +# wait timeout first. There is possibility that +# default connection gets lock timeout also or +# as con1 is rolled back it gets the locks it waited +# and does the update. +# connection default; ---error ER_LOCK_WAIT_TIMEOUT +--error 0,ER_LOCK_WAIT_TIMEOUT reap; ROLLBACK; +--enable_result_log + DROP TABLE t1; disconnect con1; diff --git a/mysql-test/suite/innodb/t/innodb-index-online.test b/mysql-test/suite/innodb/t/innodb-index-online.test index 5509d985edc..efe1c796cf1 100644 --- a/mysql-test/suite/innodb/t/innodb-index-online.test +++ b/mysql-test/suite/innodb/t/innodb-index-online.test @@ -457,8 +457,6 @@ connection default; reap; --enable_parsing #remove below con1 disconnect if above test case is enabled -connection con1; -disconnect con1; connection default; SHOW CREATE TABLE t1; @@ -474,6 +472,31 @@ SET GLOBAL innodb_monitor_disable = module_ddl; DROP TABLE t1; +--echo # +--echo # MDEV-13205 assertion !dict_index_is_online_ddl(index) upon ALTER TABLE +--echo # +CREATE TABLE t1 (c VARCHAR(64)) ENGINE=InnoDB; +SET DEBUG_SYNC = 'row_log_apply_before SIGNAL t1u_created WAIT_FOR dup_done'; +send ALTER TABLE t1 ADD UNIQUE(c); + +connection con1; +SET DEBUG_SYNC = 'now WAIT_FOR t1u_created'; +BEGIN; +INSERT INTO t1 VALUES('bar'),('bar'); +SET DEBUG_SYNC = 'now SIGNAL dup_done'; + +connection default; +--error ER_DUP_ENTRY +reap; + +SET DEBUG_SYNC = 'RESET'; +disconnect con1; +CREATE TABLE t2 (c VARCHAR(64)) ENGINE=InnoDB; +--replace_regex /#sql-[0-9a-f_]*/#sql-temporary/ +--error ER_CANT_CREATE_TABLE +ALTER TABLE t2 ADD FOREIGN KEY (c) REFERENCES t1 (c); +DROP TABLE t2,t1; + # Check that all connections opened by test cases in this file are really # gone so execution of other tests won't be affected by their presence. --source include/wait_until_count_sessions.inc diff --git a/mysql-test/suite/innodb/t/innodb-lru-force-no-free-page.test b/mysql-test/suite/innodb/t/innodb-lru-force-no-free-page.test new file mode 100644 index 00000000000..d4f08b5afe6 --- /dev/null +++ b/mysql-test/suite/innodb/t/innodb-lru-force-no-free-page.test @@ -0,0 +1,24 @@ +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/not_embedded.inc + +call mtr.add_suppression("InnoDB: Difficult to find free blocks in the buffer pool"); + +SET @saved_debug = @@SESSION.debug_dbug; +SET SESSION debug_dbug="+d,ib_lru_force_no_free_page"; + +CREATE TABLE t1 (j LONGBLOB) ENGINE = InnoDB; +BEGIN; +INSERT INTO t1 VALUES (repeat('abcdefghijklmnopqrstuvwxyz',200)); +COMMIT; + +SET debug_dbug = @saved_debug; + +DROP TABLE t1; + +# +# There should be only one message +# +let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; +--let SEARCH_PATTERN=InnoDB: Difficult to find free blocks +--source include/search_pattern_in_file.inc diff --git a/mysql-test/suite/innodb/t/innodb-master.opt b/mysql-test/suite/innodb/t/innodb-master.opt index 5266978e4f0..2e71d62206d 100644 --- a/mysql-test/suite/innodb/t/innodb-master.opt +++ b/mysql-test/suite/innodb/t/innodb-master.opt @@ -2,3 +2,5 @@ --default-storage-engine=MyISAM --innodb-strict-mode=0 --innodb-file-per-table=0 +--loose-innodb-track-changed-pages +--loose-innodb-log-archive diff --git a/mysql-test/suite/innodb/t/innodb_bug14147491.test b/mysql-test/suite/innodb/t/innodb_bug14147491.test index 620b93aae04..44b9d16ca78 100644 --- a/mysql-test/suite/innodb/t/innodb_bug14147491.test +++ b/mysql-test/suite/innodb/t/innodb_bug14147491.test @@ -8,12 +8,10 @@ -- source include/not_encrypted.inc --disable_query_log -call mtr.add_suppression("InnoDB: Table `test`.`t1` is corrupted. Please drop the table and recreate."); -call mtr.add_suppression("InnoDB: Cannot open table test/t1 from the internal data dictionary of InnoDB though the .frm file for the table exists. Please refer to http://dev.mysql.com/doc/refman/5.7/en/innodb-troubleshooting.html for how to resolve the issue."); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed file read of tablespace test/t1 page \[page id: space=[0-9]+, page number=[0-9]+\]. You may have to recover from a backup."); -call mtr.add_suppression("InnoDB: We detected index corruption in an InnoDB type table.*"); -call mtr.add_suppression("mysqld: Index for table 't1' is corrupt; try to repair it"); -call mtr.add_suppression("mysqld.exe: Index for table 't1' is corrupt; try to repair it"); +call mtr.add_suppression("InnoDB: Table `test`\\.`t1` is corrupted\\. Please drop the table and recreate\\."); +call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed file read of tablespace test/t1 page"); +call mtr.add_suppression("InnoDB: We detected index corruption in an InnoDB type table"); +call mtr.add_suppression("Index for table 't1' is corrupt; try to repair it"); --enable_query_log --echo # Ensure that purge will not crash on the table after we corrupt it. diff --git a/mysql-test/suite/innodb/t/log_corruption.test b/mysql-test/suite/innodb/t/log_corruption.test index 20763d26bdc..ca5d945afb7 100644 --- a/mysql-test/suite/innodb/t/log_corruption.test +++ b/mysql-test/suite/innodb/t/log_corruption.test @@ -16,7 +16,7 @@ call mtr.add_suppression("InnoDB: Log scan aborted at LSN"); call mtr.add_suppression("InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42\\r?$"); call mtr.add_suppression("InnoDB: Obtaining redo log encryption key version 1 failed"); call mtr.add_suppression("InnoDB: Decrypting checkpoint failed"); -call mtr.add_suppression("`mysql\\.transaction_registry` does not exist"); +call mtr.add_suppression("InnoDB: Are you sure you are using the right ib_logfiles to start up the database\\? Log sequence number in the ib_logfiles is 1213964,"); --enable_query_log let bugdir= $MYSQLTEST_VARDIR/tmp/log_corruption; @@ -141,6 +141,24 @@ eval $check_no_innodb; let SEARCH_PATTERN=InnoDB: Upgrade after a crash is not supported. This redo log was created before MariaDB 10\\.2\\.2, and it appears corrupted; --source include/search_pattern_in_file.inc +--echo # empty redo log from before MariaDB 10.2.2 +perl; +die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0"; +binmode OUT; +die unless seek(OUT, 0x800, 0); +print OUT pack("NnnNx[496]N", 0x80000944, 12, 12, 0, 0xb2a); +close OUT or die; +EOF +--let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=1m +--source include/start_mysqld.inc +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +--source include/shutdown_mysqld.inc +--let SEARCH_PATTERN= InnoDB: Upgrading redo log: +--source include/search_pattern_in_file.inc +--let $restart_parameters= $dirs + --echo # redo log from "after" MariaDB 10.2.2, but with invalid header checksum perl; die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0"; @@ -337,7 +355,7 @@ let SEARCH_PATTERN=InnoDB: MLOG_FILE_NAME incorrect:bigot; --let SEARCH_PATTERN= len 22; hex 38000000000012860cb7809781e800066269676f7400; asc 8 bigot ; --source include/search_pattern_in_file.inc ---echo # missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT +--echo # 10.2 missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT perl; die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0"; binmode OUT; @@ -365,27 +383,68 @@ EOF --source include/start_mysqld.inc eval $check_no_innodb; --source include/shutdown_mysqld.inc ---let SEARCH_PATTERN= InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42\$ +--let SEARCH_PATTERN= InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42 --source include/search_pattern_in_file.inc ---echo # Clean 10.2 redo log +--echo # 10.3 missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT +perl; +die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0"; +binmode OUT; +print OUT pack("Nx[5]nx[5]", 103, 0x1286), "MariaDB 10.3.1"; +print OUT pack("x[478]N", 0x85021a0f); +close OUT or die; +EOF + +--source include/start_mysqld.inc +eval $check_no_innodb; +--source include/shutdown_mysqld.inc +--let SEARCH_PATTERN= InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42 +--source include/search_pattern_in_file.inc + +--echo # Empty 10.3 redo log perl; die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0"; binmode OUT; die unless seek(OUT, 0x800, 0); -print OUT pack("H*", "800009440022000c00000001"); -# dummy padding (MLOG_DUMMY_RECORD) -print OUT " " x 6; -# MLOG_CHECKPOINT record -print OUT pack("CNN", 56, 0, 0x12860c); -# padding (MLOG_DUMMY_RECORD) and block checksum -print OUT " " x 481, pack("N", 0xe9b21b7b); +print OUT pack("NnnNx[496]N", 0x80000944, 12, 12, 1, 0x46c8a2a2); close OUT or die; EOF --let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=1m --source include/start_mysqld.inc -eval $check_no_innodb; +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +--source include/shutdown_mysqld.inc +--let SEARCH_PATTERN= InnoDB: .* started; log sequence number 1213970 +--source include/search_pattern_in_file.inc + +--echo # Empty 10.2 redo log +perl; +die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0"; +binmode OUT; +# header block +print OUT pack("Nx[5]nx[5]", 1, 0x1286); +print OUT "ibbackup was here!!!1!"; +print OUT pack("x[470]N", 0x52b54540); +# In encryption.innodb_log_corruption the previous step would +# replace the block with an encrypted one and update the checkpoint. +# Restore them. +# invalid (all-zero) checkpoint page 1 and an empty log page +print OUT chr(0) x 1024; +# valid checkpoint block 2 +print OUT pack("x[12]NNNx[264]", 0x12860c, 0, 0x80c); +# pointer to the MLOG_CHECKPOINT record, and checkpoint page checksum +print OUT pack("H*x[204]NNN", "590DBAACFE922582", 0x128612, 0, 0x101741b); +# log page +print OUT pack("NnnNx[496]N", 0x80000944, 12, 12, 1, 0x46c8a2a2); +close OUT or die; +EOF + +--source include/start_mysqld.inc +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); --source include/shutdown_mysqld.inc --let SEARCH_PATTERN= InnoDB: Upgrading redo log: --source include/search_pattern_in_file.inc diff --git a/mysql-test/suite/innodb/t/truncate_inject.test b/mysql-test/suite/innodb/t/truncate_inject.test new file mode 100644 index 00000000000..35e516324bb --- /dev/null +++ b/mysql-test/suite/innodb/t/truncate_inject.test @@ -0,0 +1,97 @@ +# This test is based on innodb_zip.wl6501_error_1 in MySQL 5.7. + +--source include/have_innodb.inc +--source include/innodb_row_format.inc +--source include/have_debug.inc + +SET @save_dbug = @@SESSION.debug_dbug; + +call mtr.add_suppression("InnoDB: Flagged corruption of .* in table `test`\\.`t` in TRUNCATE TABLE"); + +--echo # 1. Error in assigning undo logs for truncate action +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), INDEX ck(c)) +ENGINE = InnoDB; +insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); +check table t; +# +SET debug_dbug = '+d,ib_err_trunc_assigning_undo_log'; +--error ER_GET_ERRNO +truncate table t; +SET debug_dbug = @save_dbug; +check table t; +select * from t; + +--echo # 2. Error while preparing for truncate +SET debug_dbug = '+d,ib_err_trunc_preparing_for_truncate'; +--error ER_GET_ERRNO +truncate table t; +SET debug_dbug = @save_dbug; +check table t; +select * from t; + +--echo # 3. Error while dropping/creating indexes +SET debug_dbug = '+d,ib_err_trunc_drop_index'; +--error ER_GET_ERRNO +truncate table t; +SET debug_dbug = @save_dbug; +check table t; +--error ER_TABLE_CORRUPT,ER_GET_ERRNO +select * from t; +drop table t; + +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), INDEX ck(c)) +ENGINE = InnoDB; +insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); +check table t; + +SET debug_dbug = '+d,ib_err_trunc_create_index'; +--error ER_GET_ERRNO +truncate table t; +SET debug_dbug = @save_dbug; +check table t; +--error ER_TABLE_CORRUPT,ER_GET_ERRNO +select * from t; +drop table t; + +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), INDEX ck(c)) +ENGINE = InnoDB; +insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); +check table t; + +SET debug_dbug = '+d,ib_err_trunc_temp_recreate_index'; +truncate table t; +SET debug_dbug = @save_dbug; + +check table t; +select * from t; +drop table t; + +--echo # 4. Error while completing truncate of table involving FTS +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), +FULLTEXT INDEX(c)) ENGINE = InnoDB; +insert into t values (1, 1.1, 'mysql is now oracle company'), + (2, 2.2, 'innodb is part of mysql'), + (3, 3.3, 'innodb is default storage engine of mysql'); +check table t; +SET debug_dbug = '+d,ib_err_trunc_temp_recreate_index'; +truncate table t; +SET debug_dbug = @save_dbug; + +check table t; +select * from t; +drop table t; + +--echo # 5. Error while updating sys-tables +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), +FULLTEXT INDEX(c)) ENGINE = InnoDB; +insert into t values (1, 1.1, 'mysql is now oracle company'), + (2, 2.2, 'innodb is part of mysql'), + (3, 3.3, 'innodb is default storage engine of mysql'); +check table t; +SET debug_dbug = '+d,ib_err_trunc_temp_recreate_index'; +truncate table t; +SET debug_dbug = @save_dbug; + +check table t; +select * from t order by i; +drop table t; diff --git a/mysql-test/suite/innodb_fts/r/innodb-fts-ddl.result b/mysql-test/suite/innodb_fts/r/innodb-fts-ddl.result index 983f254cf90..cd7d8f03923 100644 --- a/mysql-test/suite/innodb_fts/r/innodb-fts-ddl.result +++ b/mysql-test/suite/innodb_fts/r/innodb-fts-ddl.result @@ -162,11 +162,15 @@ INSERT INTO articles (FTS_DOC_ID, title, body) VALUES (14,'1001 MySQL Tricks','1. Never run mysqld as root. 2. ...'), (19, 'MySQL vs. YourSQL','In the following database comparison ...'), (20, 'MySQL Security','When configured properly, MySQL ...'); -ALTER TABLE articles ADD FULLTEXT INDEX idx3 (title), -ADD FULLTEXT INDEX idx5 (title); -ERROR HY000: InnoDB presently supports one FULLTEXT index creation at a time -CREATE FULLTEXT INDEX idx on articles (title); -ALTER TABLE articles ADD FULLTEXT INDEX idx3 (title); +ALTER TABLE articles ADD FULLTEXT INDEX idx (title), +ADD FULLTEXT INDEX idx3 (title), ALGORITHM=INPLACE; +ERROR 0A000: ALGORITHM=INPLACE is not supported. Reason: InnoDB presently supports one FULLTEXT index creation at a time. Try ALGORITHM=COPY +ALTER TABLE articles ADD FULLTEXT INDEX idx (title), +ADD FULLTEXT INDEX idx3 (title); +affected rows: 6 +info: Records: 6 Duplicates: 0 Warnings: 1 +Warnings: +Note 1831 Duplicate index `idx3`. This is deprecated and will be disallowed in a future release ALTER TABLE articles ADD INDEX t20 (title(20)), LOCK=NONE; ALTER TABLE articles DROP INDEX t20; INSERT INTO articles (FTS_DOC_ID, title, body) VALUES diff --git a/mysql-test/suite/innodb_fts/t/innodb-fts-ddl.test b/mysql-test/suite/innodb_fts/t/innodb-fts-ddl.test index 10dc1462c98..23065a97002 100644 --- a/mysql-test/suite/innodb_fts/t/innodb-fts-ddl.test +++ b/mysql-test/suite/innodb_fts/t/innodb-fts-ddl.test @@ -195,12 +195,13 @@ INSERT INTO articles (FTS_DOC_ID, title, body) VALUES (19, 'MySQL vs. YourSQL','In the following database comparison ...'), (20, 'MySQL Security','When configured properly, MySQL ...'); ---error ER_INNODB_FT_LIMIT -ALTER TABLE articles ADD FULLTEXT INDEX idx3 (title), - ADD FULLTEXT INDEX idx5 (title); - -CREATE FULLTEXT INDEX idx on articles (title); -ALTER TABLE articles ADD FULLTEXT INDEX idx3 (title); +--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON +ALTER TABLE articles ADD FULLTEXT INDEX idx (title), + ADD FULLTEXT INDEX idx3 (title), ALGORITHM=INPLACE; +--enable_info +ALTER TABLE articles ADD FULLTEXT INDEX idx (title), + ADD FULLTEXT INDEX idx3 (title); +--disable_info ALTER TABLE articles ADD INDEX t20 (title(20)), LOCK=NONE; ALTER TABLE articles DROP INDEX t20; diff --git a/mysql-test/suite/innodb_zip/include/innodb_wl6501_error.inc b/mysql-test/suite/innodb_zip/include/innodb_wl6501_error.inc deleted file mode 100644 index 03400c79bef..00000000000 --- a/mysql-test/suite/innodb_zip/include/innodb_wl6501_error.inc +++ /dev/null @@ -1,225 +0,0 @@ -# -# WL#6501: make truncate table atomic -# - ---source include/have_innodb.inc ---source include/have_debug.inc - ---disable_query_log -# suppress expected warnings -call mtr.add_suppression("Unable to truncate FTS index for table"); -call mtr.add_suppression("Unable to assign a new identifier to table " - "`.*`\.`.*` after truncating it"); -call mtr.add_suppression("Flagged corruption of .* in table " - "`.*`\.`.*` in TRUNCATE TABLE"); -call mtr.add_suppression("Parent table of FTS auxiliary table " - ".*\/.* not found"); ---enable_query_log -################################################################################ -# -# Will test following scenarios: -# 1. Error in assigning undo logs for truncate action. -# 2. Error while preparing for truncate. -# 3. Error while dropping/creating indexes. -# 4. Error while completing truncate of table involving FTS. -# 5. Error while updating sys-tables. -# -################################################################################ - -#----------------------------------------------------------------------------- -# -# create test-bed -# -let $per_table = `select @@innodb_file_per_table`; - -eval set global innodb_file_per_table = on; -let $WL6501_TMP_DIR = `select @@tmpdir`; -let $WL6501_DATA_DIR = `select @@datadir`; -set innodb_strict_mode=off; - -#----------------------------------------------------------------------------- -# -# 1. Error in assigning undo logs for truncate action. -# ---echo "1. Error in assigning undo logs for truncate action." -eval set global innodb_file_per_table = $wl6501_file_per_table; ---disable_warnings -eval create $wl6501_temp table t ( - i int, f float, c char, - primary key pk(i), unique findex(f), index ck(c)) - engine = innodb row_format = $wl6501_row_fmt - key_block_size = $wl6501_kbs; ---enable_warnings -insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); -select * from t; -#check table t; -# -set session debug = "+d,ib_err_trunc_assigning_undo_log"; ---error ER_GET_ERRNO -truncate table t; -set session debug = "-d,ib_err_trunc_assigning_undo_log"; -# -#check table t; -select * from t; -drop table t; - -#----------------------------------------------------------------------------- -# -# 2. Error while preparing for truncate. -# ---echo "2. Error while preparing for truncate." -eval set global innodb_file_per_table = $wl6501_file_per_table; ---disable_warnings -eval create $wl6501_temp table t ( - i int, f float, c char, - primary key pk(i), unique findex(f), index ck(c)) - engine = innodb row_format = $wl6501_row_fmt - key_block_size = $wl6501_kbs; ---enable_warnings -insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); -select * from t; -#check table t; -# -set session debug = "+d,ib_err_trunc_preparing_for_truncate"; ---error ER_GET_ERRNO -truncate table t; -set session debug = "-d,ib_err_trunc_preparing_for_truncate"; -# -#check table t; -select * from t; -drop table t; - -#----------------------------------------------------------------------------- -# -# 3. Error while dropping/creating indexes -# ---echo "3. Error while dropping/creating indexes" -eval set global innodb_file_per_table = $wl6501_file_per_table; ---disable_warnings -eval create $wl6501_temp table t ( - i int, f float, c char, - primary key pk(i), unique findex(f), index ck(c)) - engine = innodb row_format = $wl6501_row_fmt - key_block_size = $wl6501_kbs; ---enable_warnings -insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); -select * from t; -#check table t; -# -set session debug = "+d,ib_err_trunc_drop_index"; ---error ER_GET_ERRNO -truncate table t; -set session debug = "-d,ib_err_trunc_drop_index"; -# -#check table t; ---error ER_TABLE_CORRUPT, 1030 -select * from t; -drop table t; -# -# -eval set global innodb_file_per_table = $wl6501_file_per_table; ---disable_warnings -eval create $wl6501_temp table t ( - i int, f float, c char, - primary key pk(i), unique findex(f), index ck(c)) - engine = innodb row_format = $wl6501_row_fmt - key_block_size = $wl6501_kbs; ---enable_warnings -insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); -select * from t; -#check table t; -# -set session debug = "+d,ib_err_trunc_create_index"; ---error ER_GET_ERRNO -truncate table t; -set session debug = "-d,ib_err_trunc_create_index"; -# -#check table t; ---error ER_TABLE_CORRUPT, 1030 -select * from t; -drop table t; -# -# -eval set global innodb_file_per_table = $wl6501_file_per_table; ---disable_warnings -eval create temporary table t ( - i int, f float, c char, - primary key pk(i), unique findex(f), index ck(c)) - engine = innodb row_format = $wl6501_row_fmt - key_block_size = $wl6501_kbs; ---enable_warnings -insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); -select * from t; -#check table t; -# -set session debug = "+d,ib_err_trunc_temp_recreate_index"; ---error ER_GET_ERRNO -truncate table t; -set session debug = "-d,ib_err_trunc_temp_recreate_index"; -# -#check table t; ---error ER_TABLE_CORRUPT, 1030 -select * from t; -drop table t; - -#----------------------------------------------------------------------------- -# -# 4. Error while completing truncate of table involving FTS. -# ---echo "4. Error while completing truncate of table involving FTS." -eval set global innodb_file_per_table = $wl6501_file_per_table; ---disable_warnings -eval create $wl6501_temp table t (i int, f float, c char(100), - primary key pk(i), index fk(f), fulltext index ck(c)) - engine=innodb row_format=$wl6501_row_fmt - key_block_size=$wl6501_kbs; ---enable_warnings -insert into t values (1, 1.1, 'mysql is now oracle company'), - (2, 2.2, 'innodb is part of mysql'), - (3, 3.3, 'innodb is default storage engine of mysql'); -select * from t; -#check table t; -# -set session debug = "+d,ib_err_trunc_during_fts_trunc"; ---error ER_GET_ERRNO -truncate table t; -set session debug = "-d,ib_err_trunc_during_fts_trunc"; -# -#check table t; ---error ER_TABLE_CORRUPT, 1030 -select * from t; -drop table t; - -#----------------------------------------------------------------------------- -# -# 5. Error while updating sys-tables. -# ---echo "5. Error while updating sys-tables." -eval set global innodb_file_per_table = $wl6501_file_per_table; ---disable_warnings -eval create $wl6501_temp table t (i int, f float, c char(100), - primary key pk(i), index fk(f), fulltext index ck(c)) - engine=innodb row_format=$wl6501_row_fmt - key_block_size=$wl6501_kbs; ---enable_warnings -insert into t values (1, 1.1, 'mysql is now oracle company'), - (2, 2.2, 'innodb is part of mysql'), - (3, 3.3, 'innodb is default storage engine of mysql'); -select * from t order by i; -#check table t; -# -set session debug = "+d,ib_err_trunc_during_sys_table_update"; ---error ER_GET_ERRNO -truncate table t; -set session debug = "-d,ib_err_trunc_during_sys_table_update"; -# -#check table t; ---error ER_TABLE_CORRUPT, 1030 -select * from t order by i; -drop table t; - -#----------------------------------------------------------------------------- -# -# remove test-bed -# -eval set global innodb_file_per_table = $per_table; diff --git a/mysql-test/suite/mariabackup/huge_lsn.opt b/mysql-test/suite/mariabackup/huge_lsn.opt new file mode 100644 index 00000000000..74a6450a1ef --- /dev/null +++ b/mysql-test/suite/mariabackup/huge_lsn.opt @@ -0,0 +1,6 @@ +--innodb-encrypt-log=ON +--plugin-load-add=$FILE_KEY_MANAGEMENT_SO +--loose-file-key-management +--loose-file-key-management-filekey=FILE:$MTR_SUITE_DIR/filekeys-data.key +--loose-file-key-management-filename=$MTR_SUITE_DIR/filekeys-data.enc +--loose-file-key-management-encryption-algorithm=aes_cbc diff --git a/mysql-test/suite/mariabackup/huge_lsn.result b/mysql-test/suite/mariabackup/huge_lsn.result index f8c4d3b5fb3..e7c4cc9471d 100644 --- a/mysql-test/suite/mariabackup/huge_lsn.result +++ b/mysql-test/suite/mariabackup/huge_lsn.result @@ -1,10 +1,11 @@ # # MDEV-13416 mariabackup fails with EFAULT "Bad Address" # -FOUND 1 /InnoDB: 5\.7\.\d+ started; log sequence number 17592186044428/ in mysqld.1.err +FOUND 1 /InnoDB: New log files created, LSN=175964\d{8}/ in mysqld.1.err CREATE TABLE t(i INT) ENGINE INNODB; INSERT INTO t VALUES(1); # xtrabackup backup +SET GLOBAL innodb_flush_log_at_trx_commit=1; INSERT INTO t VALUES(2); # xtrabackup prepare # shutdown server diff --git a/mysql-test/suite/mariabackup/huge_lsn.test b/mysql-test/suite/mariabackup/huge_lsn.test index 8b1ae338274..417d40c8e7a 100644 --- a/mysql-test/suite/mariabackup/huge_lsn.test +++ b/mysql-test/suite/mariabackup/huge_lsn.test @@ -1,4 +1,5 @@ --source include/not_embedded.inc +--source include/have_file_key_management.inc --echo # --echo # MDEV-13416 mariabackup fails with EFAULT "Bad Address" @@ -16,7 +17,7 @@ binmode FILE; my $ps= $ENV{INNODB_PAGE_SIZE}; my $page; die "Unable to read $file" unless sysread(FILE, $page, $ps) == $ps; -substr($page,26,8) = pack("NN", 4096, 0); +substr($page,26,8) = pack("NN", 4096, ~1024); substr($page,0,4)=pack("N",0xdeadbeef); substr($page,$ps-8,4)=pack("N",0xdeadbeef); sysseek(FILE, 0, 0) || die "Unable to rewind $file\n"; @@ -27,8 +28,9 @@ EOF --remove_files_wildcard $MYSQLD_DATADIR ib_logfile* --source include/start_mysqld.inc +let SEARCH_RANGE= -50000; let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; ---let SEARCH_PATTERN= InnoDB: 5\.7\.\d+ started; log sequence number 17592186044428 +--let SEARCH_PATTERN= InnoDB: New log files created, LSN=175964\d{8} --source include/search_pattern_in_file.inc CREATE TABLE t(i INT) ENGINE INNODB; @@ -39,6 +41,7 @@ let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --disable_result_log exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; --enable_result_log +SET GLOBAL innodb_flush_log_at_trx_commit=1; INSERT INTO t VALUES(2); echo # xtrabackup prepare; --disable_result_log diff --git a/mysql-test/suite/rpl/r/rpl_row_end_of_statement_loss.result b/mysql-test/suite/rpl/r/rpl_row_end_of_statement_loss.result new file mode 100644 index 00000000000..dc6a67b48d2 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_row_end_of_statement_loss.result @@ -0,0 +1,42 @@ +include/master-slave.inc +[connection master] +connection slave; +call mtr.add_suppression("Slave IO thread did not receive an expected Rows-log end-of-statement"); +call mtr.add_suppression("Relay log write failure: could not queue event from master"); +SET @save_debug= @@global.debug; +SET GLOBAL debug_dbug="+d,simulate_stmt_end_rows_event_loss"; +include/stop_slave.inc +CHANGE MASTER TO master_host = '127.0.0.1', master_port = MASTER_PORT, MASTER_USE_GTID=SLAVE_POS; +connection master; +CREATE TABLE t (a INT, b text(8192));; +INSERT INTO t values (1, repeat('b', 8192)), (1, repeat('b', 8192)); +connection slave; +START SLAVE IO_THREAD; +include/wait_for_slave_io_error.inc [errno=1595] +SET GLOBAL debug_dbug="-d,simulate_stmt_end_rows_event_loss"; +include/start_slave.inc +connection master; +connection slave; +connection slave; +include/stop_slave.inc +connection master; +SET @save_log_bin_compress= @@GLOBAL.log_bin_compress; +SET @save_log_bin_compress_min_len= @@GLOBAL.log_bin_compress_min_len; +SET @@GLOBAL.log_bin_compress=ON; +SET @@GLOBAL.log_bin_compress_min_len=10; +INSERT INTO t values (2, repeat('b', 8192)), (2, repeat('b', 8192)); +connection slave; +SET GLOBAL debug_dbug="+d,simulate_stmt_end_rows_event_loss"; +START SLAVE IO_THREAD; +include/wait_for_slave_io_error.inc [errno=1595] +SET GLOBAL debug_dbug="-d,simulate_stmt_end_rows_event_loss"; +include/start_slave.inc +connection master; +connection slave; +connection master; +SET @@GLOBAL.log_bin_compress= @save_log_bin_compress; +SET @@GLOBAL.log_bin_compress_min_len= @save_log_bin_compress_min_len; +DROP TABLE t; +connection slave; +SET GLOBAL debug_dbug= @save_debug; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss-master.opt b/mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss-master.opt new file mode 100644 index 00000000000..144bbca0730 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss-master.opt @@ -0,0 +1,2 @@ +--binlog-row-event-max-size=8192 + diff --git a/mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss.test b/mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss.test new file mode 100644 index 00000000000..5b2d99f3bf1 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss.test @@ -0,0 +1,66 @@ +--source include/have_debug.inc +--source include/have_binlog_format_row.inc +--source include/master-slave.inc + +# Loss of STMT_END flagged event must error out the IO thread +--connection slave +call mtr.add_suppression("Slave IO thread did not receive an expected Rows-log end-of-statement"); +call mtr.add_suppression("Relay log write failure: could not queue event from master"); + +SET @save_debug= @@global.debug; +SET GLOBAL debug_dbug="+d,simulate_stmt_end_rows_event_loss"; +--source include/stop_slave.inc +--replace_result $MASTER_MYPORT MASTER_PORT +--eval CHANGE MASTER TO master_host = '127.0.0.1', master_port = $MASTER_MYPORT, MASTER_USE_GTID=SLAVE_POS + +--connection master +--let $max_row_size=8192 +--eval CREATE TABLE t (a INT, b text($max_row_size)); +--eval INSERT INTO t values (1, repeat('b', $max_row_size)), (1, repeat('b', $max_row_size)) + +# Prove that the missed STMT_END marked rows-event causes the io thread stop. +--connection slave +START SLAVE IO_THREAD; +--let $slave_io_errno=1595 +--source include/wait_for_slave_io_error.inc +SET GLOBAL debug_dbug="-d,simulate_stmt_end_rows_event_loss"; +--source include/start_slave.inc + +--connection master +sync_slave_with_master; + +# Compressed version of the above +--connection slave +--source include/stop_slave.inc + +--connection master +SET @save_log_bin_compress= @@GLOBAL.log_bin_compress; +SET @save_log_bin_compress_min_len= @@GLOBAL.log_bin_compress_min_len; + +SET @@GLOBAL.log_bin_compress=ON; +SET @@GLOBAL.log_bin_compress_min_len=10; + +--eval INSERT INTO t values (2, repeat('b', $max_row_size)), (2, repeat('b', $max_row_size)) + +# Prove that the missed STMT_END marked rows-event causes the io thread stop. +--connection slave +SET GLOBAL debug_dbug="+d,simulate_stmt_end_rows_event_loss"; +START SLAVE IO_THREAD; +--let $slave_io_errno=1595 +--source include/wait_for_slave_io_error.inc +SET GLOBAL debug_dbug="-d,simulate_stmt_end_rows_event_loss"; +--source include/start_slave.inc + +--connection master +sync_slave_with_master; + +# cleanup + +--connection master +SET @@GLOBAL.log_bin_compress= @save_log_bin_compress; +SET @@GLOBAL.log_bin_compress_min_len= @save_log_bin_compress_min_len; +DROP TABLE t; +sync_slave_with_master; +SET GLOBAL debug_dbug= @save_debug; + +--source include/rpl_end.inc diff --git a/mysql-test/suite/sql_sequence/create.result b/mysql-test/suite/sql_sequence/create.result index 015b5ac762b..20d16822575 100644 --- a/mysql-test/suite/sql_sequence/create.result +++ b/mysql-test/suite/sql_sequence/create.result @@ -482,6 +482,7 @@ select previous value for t1; previous value for t1 1 CREATE TEMPORARY SEQUENCE t1 start with 100 minvalue 100 maxvalue 200 increment by 1 cache 10 engine=innodb; +START TRANSACTION WITH CONSISTENT SNAPSHOT; select previous value for t1; previous value for t1 NULL diff --git a/mysql-test/suite/sql_sequence/create.test b/mysql-test/suite/sql_sequence/create.test index 6696e78db92..91411bebfde 100644 --- a/mysql-test/suite/sql_sequence/create.test +++ b/mysql-test/suite/sql_sequence/create.test @@ -362,6 +362,7 @@ CREATE SEQUENCE t1 start with 1 minvalue 1 maxvalue 10 increment by 1 cache 10 e select next value for t1; select previous value for t1; CREATE TEMPORARY SEQUENCE t1 start with 100 minvalue 100 maxvalue 200 increment by 1 cache 10 engine=innodb; +START TRANSACTION WITH CONSISTENT SNAPSHOT; select previous value for t1; select next value for t1; select previous value for t1; diff --git a/mysql-test/t/cte_nonrecursive.test b/mysql-test/t/cte_nonrecursive.test index 1f21dbcd36d..9436665bfee 100644 --- a/mysql-test/t/cte_nonrecursive.test +++ b/mysql-test/t/cte_nonrecursive.test @@ -853,3 +853,32 @@ execute stmt; deallocate prepare stmt; drop table t1; + +--echo # +--echo # MDEV-14852: CTE using temporary table in query +--echo # with two references to the CTE +--echo # + +create temporary table t1 (i int); +insert into t1 values (5),(4),(1),(2),(3); + +with +c1 as (select i from t1), +c2 as (select i from c1 where c1.i=2) +select i from c1 where i > 3 union select i from c2; + +drop table t1; + +create table t1 (term char(10)); +create temporary table t2 (term char(10)); + +insert into t1 values ('TERM01'),('TERM02'),('TERM03'); +insert into t2 values ('TERM02'),('TERM03'),('TERM04'); + +with c1 as (select * from t1), c2 as (select * from t2) +(select * from c1 left outer join c2 on c1.term = c2.term) +union all +(select * from c1 right outer join c2 on c1.term = c2.term + where c1.term is null); + +drop table t1,t2; diff --git a/mysql-test/t/cte_recursive.test b/mysql-test/t/cte_recursive.test index 12013d43c1f..d2e33b165f7 100644 --- a/mysql-test/t/cte_recursive.test +++ b/mysql-test/t/cte_recursive.test @@ -2000,6 +2000,93 @@ set @var= drop table t1; --echo # +--echo # mdev-14777: crash caused by the same as in mdev-14755 +--echo # + +--source include/have_sequence.inc + +CREATE TABLE t1 (i1 int NOT NULL, i2 int); +CREATE TABLE t2 (d1 int NOT NULL PRIMARY KEY); +CREATE TABLE t3 (i int ); + +insert into t1 select seq,seq from seq_1_to_100000; +insert into t2 select seq from seq_1000_to_100000; +insert into t3 select seq from seq_1_to_1000; + +SELECT * +FROM +( + SELECT * + FROM + ( + WITH RECURSIVE rt AS + ( + SELECT i2 P, i1 C FROM t1 WHERE i1 IN (SELECT d1 FROM t2) + UNION + SELECT t1.i2 P, rt.C C FROM t1, rt + ) + SELECT C,P + FROM ( SELECT P,C FROM rt WHERE NOT EXISTS (SELECT 1 FROM t1) ) Y + ) X + WHERE 1 = 1 +) K, t3; + +drop table t1,t2,t3; + +--echo # +--echo # mdev-14879: subquery with recursive reference in WHERE of CTE +--echo # + +create table flights +(departure varchar(32), + arrival varchar(32), + carrier varchar(20), + flight_number char(7)); + +insert into flights values +('Seattle', 'Frankfurt', 'Lufthansa', 'LH 491'), +('Seattle', 'Chicago', 'American', 'AA 2573'), +('Seattle', 'Los Angeles', 'Alaska Air', 'AS 410'), +('Chicago', 'New York', 'American', 'AA 375'), +('Chicago', 'Montreal', 'Air Canada', 'AC 3053'), +('Los Angeles', 'New York', 'Delta', 'DL 1197'), +('Moscow', 'Tokyo', 'Aeroflot', 'SU 264'), +('New York', 'Paris', 'Air France', 'AF 23'), +('Frankfurt', 'Moscow', 'Lufthansa', 'LH 1444'), +('Tokyo', 'Seattle', 'ANA', 'NH 178'), +('Los Angeles', 'Tokyo', 'ANA', 'NH 175'), +('Moscow', 'Los Angeles', 'Aeroflot', 'SU 106'), +('Montreal', 'Paris', 'Air Canada', 'AC 870'), +('Cairo', 'Paris', 'Air France', 'AF 503'), +('New York', 'Seattle', 'American', 'AA 45'), +('Paris', 'Chicago', 'Air France', 'AF 6734'); + +with recursive destinations (city) as +( select a.arrival from flights a where a.departure='Cairo' + union + select b.arrival from destinations r, flights b where r.city=b.departure) +select * from destinations; + +set standard_compliant_cte=0; + +let $q= +with recursive destinations (city, legs) as +( + select a.arrival, 1 from flights a where a.departure='Cairo' + union + select b.arrival, r.legs + 1 from destinations r, flights b + where r.city=b.departure and b.arrival not in (select city from destinations) +) +select * from destinations; + +eval $q; +eval explain extended $q; + +set standard_compliant_cte=default; + +drop table flights; + +--echo # --echo # MDEV-14217 [db crash] Recursive CTE when SELECT includes new field --echo # diff --git a/mysql-test/t/derived_cond_pushdown.test b/mysql-test/t/derived_cond_pushdown.test index f6e902336b3..fa977f7ec4b 100644 --- a/mysql-test/t/derived_cond_pushdown.test +++ b/mysql-test/t/derived_cond_pushdown.test @@ -2141,6 +2141,31 @@ eval explain format=json $q; drop table t1; --echo # +--echo # MDEV-13454: consequence of mdev-14368 fixed for 5.5 +--echo # + +SET sql_mode = 'ONLY_FULL_GROUP_BY'; + +create table t1 (id int, id2 int); +insert into t1 values (1,1),(2,3),(3,4),(7,2); + +create table t2(id2 int); +insert t2 values (1),(2),(3); + +let $q= +SELECT * FROM t1 + LEFT OUTER JOIN + (SELECT id2, COUNT(*) as ct FROM t2 GROUP BY id2) vc USING (id2) +WHERE (vc.ct>0); + +eval $q; +eval EXPLAIN FORMAT=JSON $q; + +DROP TABLE t1,t2; + +SET sql_mode = DEFAULT; + +--echo # --echo # MDEV-10855: Pushdown into derived with window functions --echo # diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test index ce0b949fcac..c5cec99cebf 100644 --- a/mysql-test/t/subselect.test +++ b/mysql-test/t/subselect.test @@ -6078,6 +6078,33 @@ SELECT (SELECT MAX(sq.f2) FROM t1) FROM (SELECT * FROM t2) AS sq WHERE f2 = 2; --echo # drop table t1, t2; +--echo # +--echo # MDEV-13933: Wrong results in COUNT() query with EXISTS and exists_to_in +--echo # (5.5 test) +--echo # +SET @optimiser_switch_save= @@optimizer_switch; + +CREATE TABLE t1 (a INT NOT NULL); +INSERT INTO t1 VALUES (1),(1),(1),(5),(5); + +CREATE TABLE t2 (b INT); +INSERT INTO t2 VALUES (5),(1); + +CREATE TABLE t3 (c INT, KEY(c)); +INSERT INTO t3 VALUES (5),(5); + +SET optimizer_switch='semijoin=on'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); + +SET optimizer_switch='semijoin=off'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); + +SET @@optimizer_switch= @optimiser_switch_save; +DROP TABLE t1, t2, t3; + +--echo End of 5.5 tests --echo # End of 10.0 tests --echo # diff --git a/mysys/my_atomic_writes.c b/mysys/my_atomic_writes.c index 0b54a207713..7f1e353c121 100644 --- a/mysys/my_atomic_writes.c +++ b/mysys/my_atomic_writes.c @@ -259,7 +259,7 @@ static my_bool shannon_has_atomic_write(File file, int page_size) ************************************************************************/ /* - Initalize automic write sub systems. + Initialize automic write sub systems. Checks if we have any devices that supports atomic write */ diff --git a/mysys/my_context.c b/mysys/my_context.c index cf10738bdbd..5423f59d19b 100644 --- a/mysys/my_context.c +++ b/mysys/my_context.c @@ -707,7 +707,7 @@ my_context_continue(struct my_context *c) { /* This seems to be a common trick to run ConvertThreadToFiber() only on the - first occurence in a thread, in a way that works on multiple Windows + first occurrence in a thread, in a way that works on multiple Windows versions. */ void *current_fiber= GetCurrentFiber(); diff --git a/mysys/my_getopt.c b/mysys/my_getopt.c index 0ea062988f6..9e617366ed5 100644 --- a/mysys/my_getopt.c +++ b/mysys/my_getopt.c @@ -236,7 +236,7 @@ int handle_options(int *argc, char ***argv, { is_cmdline_arg= 1; - /* save the separator too if skip unkown options */ + /* save the separator too if skip unknown options */ if (my_getopt_skip_unknown) (*argv)[argvpos++]= cur_arg; else diff --git a/mysys/thr_mutex.c b/mysys/thr_mutex.c index 49cb3ea600f..268a2b80f63 100644 --- a/mysys/thr_mutex.c +++ b/mysys/thr_mutex.c @@ -240,7 +240,7 @@ int safe_mutex_lock(safe_mutex_t *mp, myf my_flags, const char *file, if (!mp->file) { fprintf(stderr, - "safe_mutex: Trying to lock unitialized mutex at %s, line %d\n", + "safe_mutex: Trying to lock uninitialized mutex at %s, line %d\n", file, line); fflush(stderr); abort(); @@ -585,7 +585,7 @@ int safe_mutex_destroy(safe_mutex_t *mp, const char *file, uint line) if (!mp->file) { fprintf(stderr, - "safe_mutex: Trying to destroy unitialized mutex at %s, line %d\n", + "safe_mutex: Trying to destroy uninitialized mutex at %s, line %d\n", file, line); fflush(stderr); abort(); diff --git a/sql/field.cc b/sql/field.cc index 642ad6a65e9..8de823cee7c 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -8533,6 +8533,7 @@ uchar *Field_blob::pack(uchar *to, const uchar *from, uint max_length) @return New pointer into memory based on from + length of the data */ + const uchar *Field_blob::unpack(uchar *to, const uchar *from, const uchar *from_end, uint param_data) { @@ -8548,7 +8549,6 @@ const uchar *Field_blob::unpack(uchar *to, const uchar *from, if (from + master_packlength + length > from_end) DBUG_RETURN(0); set_ptr(length, const_cast<uchar*> (from) + master_packlength); - DBUG_DUMP("record", to, table->s->reclength); DBUG_RETURN(from + master_packlength + length); } diff --git a/sql/item_func.cc b/sql/item_func.cc index ac7ed75e7f9..93ddddfde61 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -2724,7 +2724,7 @@ bool Item_func_min_max::get_date_native(MYSQL_TIME *ltime, ulonglong fuzzy_date) ltime->hour+= (ltime->month * 32 + ltime->day) * 24; ltime->year= ltime->month= ltime->day= 0; if (adjust_time_range_with_warn(ltime, - std::min<uint>(decimals, TIME_SECOND_PART_DIGITS))) + MY_MIN(decimals, TIME_SECOND_PART_DIGITS))) return (null_value= true); } diff --git a/sql/item_sum.cc b/sql/item_sum.cc index 945224c2623..be9aefa7338 100644 --- a/sql/item_sum.cc +++ b/sql/item_sum.cc @@ -2531,7 +2531,7 @@ bool Item_sum_bit::remove_as_window(ulonglong value) } // Prevent overflow; - num_values_added = std::min(num_values_added, num_values_added - 1); + num_values_added = MY_MIN(num_values_added, num_values_added - 1); set_bits_from_counters(); return 0; } @@ -2544,7 +2544,7 @@ bool Item_sum_bit::add_as_window(ulonglong value) bit_counters[i]+= (value & (1ULL << i)) ? 1 : 0; } // Prevent overflow; - num_values_added = std::max(num_values_added, num_values_added + 1); + num_values_added = MY_MAX(num_values_added, num_values_added + 1); set_bits_from_counters(); return 0; } diff --git a/sql/key.cc b/sql/key.cc index 3ee083e560f..ab93e8a0437 100644 --- a/sql/key.cc +++ b/sql/key.cc @@ -1,4 +1,5 @@ /* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2018, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -21,9 +22,6 @@ #include "key.h" // key_rec_cmp #include "field.h" // Field -using std::min; -using std::max; - /* Search after a key that starts with 'field' @@ -135,7 +133,7 @@ void key_copy(uchar *to_key, const uchar *from_record, KEY *key_info, Don't copy data for null values The -1 below is to subtract the null byte which is already handled */ - length= min<uint>(key_length, key_part->store_length-1); + length= MY_MIN(key_length, uint(key_part->store_length)-1); if (with_zerofill) bzero((char*) to_key, length); continue; @@ -145,7 +143,7 @@ void key_copy(uchar *to_key, const uchar *from_record, KEY *key_info, key_part->key_part_flag & HA_VAR_LENGTH_PART) { key_length-= HA_KEY_BLOB_LENGTH; - length= min<uint>(key_length, key_part->length); + length= MY_MIN(key_length, key_part->length); uint bytes= key_part->field->get_key_image(to_key, length, Field::itRAW); if (with_zerofill && bytes < length) bzero((char*) to_key + bytes, length - bytes); @@ -153,7 +151,7 @@ void key_copy(uchar *to_key, const uchar *from_record, KEY *key_info, } else { - length= min<uint>(key_length, key_part->length); + length= MY_MIN(key_length, key_part->length); Field *field= key_part->field; CHARSET_INFO *cs= field->charset(); uint bytes= field->get_key_image(to_key, length, Field::itRAW); @@ -205,7 +203,7 @@ void key_restore(uchar *to_record, const uchar *from_key, KEY *key_info, Don't copy data for null bytes The -1 below is to subtract the null byte which is already handled */ - length= min<uint>(key_length, key_part->store_length-1); + length= MY_MIN(key_length, uint(key_part->store_length)-1); continue; } } @@ -247,7 +245,7 @@ void key_restore(uchar *to_record, const uchar *from_key, KEY *key_info, my_ptrdiff_t ptrdiff= to_record - field->table->record[0]; field->move_field_offset(ptrdiff); key_length-= HA_KEY_BLOB_LENGTH; - length= min<uint>(key_length, key_part->length); + length= MY_MIN(key_length, key_part->length); old_map= dbug_tmp_use_all_columns(field->table, field->table->write_set); field->set_key_image(from_key, length); dbug_tmp_restore_column_map(field->table->write_set, old_map); @@ -256,7 +254,7 @@ void key_restore(uchar *to_record, const uchar *from_key, KEY *key_info, } else { - length= min<uint>(key_length, key_part->length); + length= MY_MIN(key_length, key_part->length); /* skip the byte with 'uneven' bits, if used */ memcpy(to_record + key_part->offset, from_key + used_uneven_bits , (size_t) length - used_uneven_bits); @@ -314,7 +312,7 @@ bool key_cmp_if_same(TABLE *table,const uchar *key,uint idx,uint key_length) return 1; continue; } - length= min((uint) (key_end-key), store_length); + length= MY_MIN((uint) (key_end-key), store_length); if (!(key_part->key_type & (FIELDFLAG_NUMBER+FIELDFLAG_BINARY+ FIELDFLAG_PACK))) { @@ -392,7 +390,7 @@ void field_unpack(String *to, Field *field, const uchar *rec, uint max_length, tmp.length(charpos); } if (max_length < field->pack_length()) - tmp.length(min(tmp.length(),max_length)); + tmp.length(MY_MIN(tmp.length(),max_length)); ErrConvString err(&tmp); to->append(err.ptr()); } diff --git a/sql/log.cc b/sql/log.cc index 6923a6241cd..742b0a03e90 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -7108,7 +7108,7 @@ int MYSQL_BIN_LOG::write_cache(THD *thd, IO_CACHE *cache) int4store(ev + EVENT_LEN_OFFSET, ev_len + writer.checksum_len); writer.remains= ev_len; - if (writer.write(ev, std::min<uint>(ev_len, length - hdr_offs))) + if (writer.write(ev, MY_MIN(ev_len, length - hdr_offs))) DBUG_RETURN(ER_ERROR_ON_WRITE); /* next event header at ... */ diff --git a/sql/log_event.cc b/sql/log_event.cc index ca659085228..c40e3be94c8 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -56,8 +56,6 @@ #define my_b_write_string(A, B) my_b_write((A), (uchar*)(B), (uint) (sizeof(B) - 1)) -using std::max; - /** BINLOG_CHECKSUM variable. */ @@ -1799,8 +1797,8 @@ int Log_event::read_log_event(IO_CACHE* file, String* packet, if (data_len < LOG_EVENT_MINIMAL_HEADER_LEN) DBUG_RETURN(LOG_READ_BOGUS); - if (data_len > max(max_allowed_packet, - opt_binlog_rows_event_max_size + MAX_LOG_EVENT_HEADER)) + if (data_len > MY_MAX(max_allowed_packet, + opt_binlog_rows_event_max_size + MAX_LOG_EVENT_HEADER)) DBUG_RETURN(LOG_READ_TOO_LARGE); if (likely(data_len > LOG_EVENT_MINIMAL_HEADER_LEN)) diff --git a/sql/mdl.h b/sql/mdl.h index f30c976ac94..be9cc806ec2 100644 --- a/sql/mdl.h +++ b/sql/mdl.h @@ -21,8 +21,6 @@ #include <mysql_com.h> #include <lf.h> -#include <algorithm> - class THD; class MDL_context; @@ -373,8 +371,7 @@ public: character set is utf-8, we can safely assume that no character starts with a zero byte. */ - using std::min; - return memcmp(m_ptr, rhs->m_ptr, min(m_length, rhs->m_length)); + return memcmp(m_ptr, rhs->m_ptr, MY_MIN(m_length, rhs->m_length)); } MDL_key(const MDL_key *rhs) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 067b2e70ba6..29b12934550 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -2928,16 +2928,17 @@ void unlink_thd(THD *thd) DBUG_ENTER("unlink_thd"); DBUG_PRINT("enter", ("thd: %p", thd)); + thd->cleanup(); + thd->add_status_to_global(); + unlink_not_visible_thd(thd); + /* Do not decrement when its wsrep system thread. wsrep_applier is set for applier as well as rollbacker threads. */ if (IF_WSREP(!thd->wsrep_applier, 1)) dec_connection_count(thd->scheduler); - thd->cleanup(); - thd->add_status_to_global(); - unlink_not_visible_thd(thd); thd->free_connection(); DBUG_VOID_RETURN; diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc index dada23508b5..9d6b67e845e 100644 --- a/sql/opt_subselect.cc +++ b/sql/opt_subselect.cc @@ -2662,7 +2662,8 @@ void advance_sj_state(JOIN *join, table_map remaining_tables, uint idx, LooseScan detector in best_access_path) */ remaining_tables &= ~new_join_tab->table->map; - table_map dups_producing_tables; + table_map dups_producing_tables, prev_dups_producing_tables, + prev_sjm_lookup_tables; if (idx == join->const_tables) dups_producing_tables= 0; @@ -2673,7 +2674,7 @@ void advance_sj_state(JOIN *join, table_map remaining_tables, uint idx, if ((emb_sj_nest= new_join_tab->emb_sj_nest)) dups_producing_tables |= emb_sj_nest->sj_inner_tables; - Semi_join_strategy_picker **strategy; + Semi_join_strategy_picker **strategy, **prev_strategy; if (idx == join->const_tables) { /* First table, initialize pickers */ @@ -2725,23 +2726,54 @@ void advance_sj_state(JOIN *join, table_map remaining_tables, uint idx, 3. We have no clue what to do about fanount of semi-join Y. */ if ((dups_producing_tables & handled_fanout) || - (read_time < *current_read_time && + (read_time < *current_read_time && !(handled_fanout & pos->inner_tables_handled_with_other_sjs))) { - /* Mark strategy as used */ - (*strategy)->mark_used(); - pos->sj_strategy= sj_strategy; - if (sj_strategy == SJ_OPT_MATERIALIZE) - join->sjm_lookup_tables |= handled_fanout; + DBUG_ASSERT(pos->sj_strategy != sj_strategy); + /* + If the strategy choosen first time or + the strategy replace strategy which was used to exectly the same + tables + */ + if (pos->sj_strategy == SJ_OPT_NONE || + handled_fanout == + (prev_dups_producing_tables ^ dups_producing_tables)) + { + prev_strategy= strategy; + if (pos->sj_strategy == SJ_OPT_NONE) + { + prev_dups_producing_tables= dups_producing_tables; + prev_sjm_lookup_tables= join->sjm_lookup_tables; + } + /* Mark strategy as used */ + (*strategy)->mark_used(); + pos->sj_strategy= sj_strategy; + if (sj_strategy == SJ_OPT_MATERIALIZE) + join->sjm_lookup_tables |= handled_fanout; + else + join->sjm_lookup_tables &= ~handled_fanout; + *current_read_time= read_time; + *current_record_count= rec_count; + dups_producing_tables &= ~handled_fanout; + //TODO: update bitmap of semi-joins that were handled together with + // others. + if (is_multiple_semi_joins(join, join->positions, idx, + handled_fanout)) + pos->inner_tables_handled_with_other_sjs |= handled_fanout; + } else - join->sjm_lookup_tables &= ~handled_fanout; - *current_read_time= read_time; - *current_record_count= rec_count; - dups_producing_tables &= ~handled_fanout; - //TODO: update bitmap of semi-joins that were handled together with - // others. - if (is_multiple_semi_joins(join, join->positions, idx, handled_fanout)) - pos->inner_tables_handled_with_other_sjs |= handled_fanout; + { + /* Conflict fall to most general variant */ + (*prev_strategy)->set_empty(); + dups_producing_tables= prev_dups_producing_tables; + join->sjm_lookup_tables= prev_sjm_lookup_tables; + // mark it 'none' to avpoid loops + pos->sj_strategy= SJ_OPT_NONE; + // next skip to last; + strategy= pickers + + (sizeof(pickers)/sizeof(Semi_join_strategy_picker*) - 3); + continue; + } } else { diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index 9f48f908102..78e6165a551 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -582,7 +582,7 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, bool table_opened= false; TABLE *table; list_element *delete_list= 0, *next, *cur, **next_ptr_ptr, **best_ptr_ptr; - uint64_t best_sub_id; + uint64 best_sub_id; element *elem; ulonglong thd_saved_option= thd->variables.option_bits; Query_tables_list lex_backup; diff --git a/sql/rpl_mi.h b/sql/rpl_mi.h index 14d74dc4bb7..260c35e8c04 100644 --- a/sql/rpl_mi.h +++ b/sql/rpl_mi.h @@ -133,6 +133,19 @@ public: extern TYPELIB slave_parallel_mode_typelib; +typedef struct st_rows_event_tracker +{ + char binlog_file_name[FN_REFLEN]; + my_off_t first_seen; + my_off_t last_seen; + bool stmt_end_seen; + void update(const char* file_name, my_off_t pos, + const char* buf, + const Format_description_log_event *fdle); + void reset(); + bool check_and_report(const char* file_name, my_off_t pos); +} Rows_event_tracker; + /***************************************************************************** Replication IO Thread @@ -301,6 +314,14 @@ class Master_info : public Slave_reporting_capability uint64 gtid_reconnect_event_skip_count; /* gtid_event_seen is false until we receive first GTID event from master. */ bool gtid_event_seen; + /** + The struct holds some history of Rows- log-event reading/queuing + by the receiver thread. Its fields are updated per each such event + at time of queue_event(), and they are checked to detect + the Rows- event group integrity violation at time of first non-Rows- + event gets handled. + */ + Rows_event_tracker rows_event_tracker; bool in_start_all_slaves, in_stop_all_slaves; bool in_flush_all_relay_logs; uint users; /* Active user for object */ diff --git a/sql/set_var.cc b/sql/set_var.cc index 311b33bc0dd..097124a0b3a 100644 --- a/sql/set_var.cc +++ b/sql/set_var.cc @@ -1323,7 +1323,7 @@ resolve_engine_list_item(THD *thd, plugin_ref *list, uint32 *idx, { LEX_CSTRING item_str; plugin_ref ref; - uint32_t i; + uint32 i; THD *thd_or_null = (temp_copy ? thd : NULL); item_str.str= pos; diff --git a/sql/slave.cc b/sql/slave.cc index 90ad8bbdef1..b69e92a8363 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -3659,7 +3659,8 @@ static ulong read_event(MYSQL* mysql, Master_info *mi, bool* suppress_warnings, we suppress prints to .err file as long as the reconnect happens without problems */ - *suppress_warnings= TRUE; + *suppress_warnings= + global_system_variables.log_warnings < 2 ? TRUE : FALSE; } else { @@ -4589,6 +4590,7 @@ pthread_handler_t handle_slave_io(void *arg) mi->abort_slave = 0; mysql_mutex_unlock(&mi->run_lock); mysql_cond_broadcast(&mi->start_cond); + mi->rows_event_tracker.reset(); DBUG_PRINT("master_info",("log_file_name: '%s' position: %llu", mi->master_log_name, mi->master_log_pos)); @@ -4672,6 +4674,10 @@ connected: */ mi->gtid_reconnect_event_skip_count= mi->events_queued_since_last_gtid; mi->gtid_event_seen= false; + /* + Reset stale state of the rows-event group tracker at reconnect. + */ + mi->rows_event_tracker.reset(); } #ifdef ENABLED_DEBUG_SYNC @@ -6077,7 +6083,7 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) char* new_buf = NULL; char new_buf_arr[4096]; bool is_malloc = false; - + bool is_rows_event= false; /* FD_q must have been prepared for the first R_a event inside get_master_version_and_clock() @@ -6511,11 +6517,11 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) got_gtid_event= true; if (mi->using_gtid == Master_info::USE_GTID_NO) goto default_action; - if (unlikely(!mi->gtid_event_seen)) + if (unlikely(mi->gtid_reconnect_event_skip_count)) { - mi->gtid_event_seen= true; - if (mi->gtid_reconnect_event_skip_count) + if (likely(!mi->gtid_event_seen)) { + mi->gtid_event_seen= true; /* If we are reconnecting, and we need to skip a partial event group already queued to the relay log before the reconnect, then we check @@ -6544,13 +6550,45 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) rpl_slave_state_tostring_helper(&error_msg, &event_gtid, &first); goto err; } + if (global_system_variables.log_warnings > 1) + { + bool first= true; + StringBuffer<1024> gtid_text; + rpl_slave_state_tostring_helper(>id_text, &mi->last_queued_gtid, + &first); + sql_print_information("Slave IO thread is reconnected to " + "receive Gtid_log_event %s. It is to skip %llu " + "already received events including the gtid one", + gtid_text.ptr(), + mi->events_queued_since_last_gtid); + } + goto default_action; } - } + else + { + bool first; + StringBuffer<1024> gtid_text; - if (unlikely(mi->gtid_reconnect_event_skip_count)) - { - goto default_action; + gtid_text.append(STRING_WITH_LEN("Last received gtid: ")); + first= true; + rpl_slave_state_tostring_helper(>id_text, &mi->last_queued_gtid, + &first); + gtid_text.append(STRING_WITH_LEN(", currently received: ")); + first= true; + rpl_slave_state_tostring_helper(>id_text, &event_gtid, &first); + + error= ER_SLAVE_RELAY_LOG_WRITE_FAILURE; + sql_print_error("Slave IO thread has received a new Gtid_log_event " + "while skipping already logged events " + "after reconnect. %s. %llu remains to be skipped. " + "The number of originally read events was %llu", + gtid_text.ptr(), + mi->gtid_reconnect_event_skip_count, + mi->events_queued_since_last_gtid); + goto err; + } } + mi->gtid_event_seen= true; /* We have successfully queued to relay log everything before this GTID, so @@ -6617,8 +6655,34 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) goto err; } } - buf = new_buf; is_compress_event = true; + buf = new_buf; + /* + As we are uncertain about compressed V2 rows events, we don't track + them + */ + if (LOG_EVENT_IS_ROW_V2((Log_event_type) buf[EVENT_TYPE_OFFSET])) + goto default_action; + /* fall through */ + case WRITE_ROWS_EVENT_V1: + case UPDATE_ROWS_EVENT_V1: + case DELETE_ROWS_EVENT_V1: + case WRITE_ROWS_EVENT: + case UPDATE_ROWS_EVENT: + case DELETE_ROWS_EVENT: + { + is_rows_event= true; + mi->rows_event_tracker.update(mi->master_log_name, + mi->master_log_pos, + buf, + mi->rli.relay_log. + description_event_for_queue); + + DBUG_EXECUTE_IF("simulate_stmt_end_rows_event_loss", + { + mi->rows_event_tracker.stmt_end_seen= false; + }); + } goto default_action; #ifndef DBUG_OFF @@ -6677,6 +6741,21 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) } /* + Integrity of Rows- event group check. + A sequence of Rows- events must end with STMT_END_F flagged one. + Even when Heartbeat event interrupts Rows- events flow this must indicate a + malfunction e.g logging on the master. + */ + if (((uchar) buf[EVENT_TYPE_OFFSET] != HEARTBEAT_LOG_EVENT) && + !is_rows_event && + mi->rows_event_tracker.check_and_report(mi->master_log_name, + mi->master_log_pos)) + { + error= ER_SLAVE_RELAY_LOG_WRITE_FAILURE; + goto err; + } + + /* If we filter events master-side (eg. @@skip_replication), we will see holes in the event positions from the master. If we see such a hole, adjust mi->master_log_pos accordingly so we maintain the correct position (for @@ -6845,6 +6924,21 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) The whole of the current event group is queued. So in case of reconnect we can start from after the current GTID. */ + if (mi->gtid_reconnect_event_skip_count) + { + bool first= true; + StringBuffer<1024> gtid_text; + + rpl_slave_state_tostring_helper(>id_text, &mi->last_queued_gtid, + &first); + sql_print_error("Slave IO thread received a terminal event from " + "group %s whose retrieval was interrupted " + "with reconnect. We still had %llu events to read. " + "The number of originally read events was %llu", + gtid_text.ptr(), + mi->gtid_reconnect_event_skip_count, + mi->events_queued_since_last_gtid); + } mi->gtid_current_pos.update(&mi->last_queued_gtid); mi->events_queued_since_last_gtid= 0; @@ -7844,6 +7938,92 @@ bool rpl_master_erroneous_autoinc(THD *thd) return FALSE; } + +static bool get_row_event_stmt_end(const char* buf, + const Format_description_log_event *fdle) +{ + uint8 const common_header_len= fdle->common_header_len; + Log_event_type event_type= (Log_event_type)(uchar)buf[EVENT_TYPE_OFFSET]; + + uint8 const post_header_len= fdle->post_header_len[event_type-1]; + const char *flag_start= buf + common_header_len; + /* + The term 4 below signifies that master is of 'an intermediate source', see + Rows_log_event::Rows_log_event. + */ + flag_start += RW_MAPID_OFFSET + (post_header_len == 6) ? 4 : RW_FLAGS_OFFSET; + + return (uint2korr(flag_start) & Rows_log_event::STMT_END_F) != 0; +} + + +/* + Reset log event tracking data. +*/ + +void Rows_event_tracker::reset() +{ + binlog_file_name[0]= 0; + first_seen= last_seen= 0; + stmt_end_seen= false; +} + + +/* + Update log event tracking data. + + The first- and last- seen event binlog position get memorized, as + well as the end-of-statement status of the last one. +*/ + +void Rows_event_tracker::update(const char* file_name, my_off_t pos, + const char* buf, + const Format_description_log_event *fdle) +{ + if (!first_seen) + { + first_seen= pos; + strmake(binlog_file_name, file_name, sizeof(binlog_file_name) - 1); + } + last_seen= pos; + DBUG_ASSERT(stmt_end_seen == 0); // We can only have one + stmt_end_seen= get_row_event_stmt_end(buf, fdle); +}; + + +/** + The function is called at next event reading + after a sequence of Rows- log-events. It checks the end-of-statement status + of the past sequence to report on any isssue. + In the positive case the tracker gets reset. + + @return true when the Rows- event group integrity found compromised, + false otherwise. +*/ +bool Rows_event_tracker::check_and_report(const char* file_name, + my_off_t pos) +{ + if (last_seen) + { + // there was at least one "block" event previously + if (!stmt_end_seen) + { + sql_print_error("Slave IO thread did not receive an expected " + "Rows-log end-of-statement for event starting " + "at log '%s' position %llu " + "whose last block was seen at log '%s' position %llu. " + "The end-of-statement should have been delivered " + "before the current one at log '%s' position %llu", + binlog_file_name, first_seen, + binlog_file_name, last_seen, file_name, pos); + return true; + } + reset(); + } + + return false; +} + /** @} (end of group Replication) */ diff --git a/sql/sql_class.h b/sql/sql_class.h index 5469171f0e3..28c603dd6b3 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -837,7 +837,7 @@ typedef struct system_status_var ulonglong table_open_cache_overflows; double last_query_cost; double cpu_time, busy_time; - uint32_t threads_running; + uint32 threads_running; /* Don't initialize */ /* Memory used for thread local storage */ int64 max_local_memory_used; diff --git a/sql/sql_cte.cc b/sql/sql_cte.cc index 9608436226c..7da088cdfd5 100644 --- a/sql/sql_cte.cc +++ b/sql/sql_cte.cc @@ -833,12 +833,19 @@ st_select_lex_unit *With_element::clone_parsed_spec(THD *thd, parse_status= parse_sql(thd, &parser_state, 0); if (parse_status) goto err; + + if (check_dependencies_in_with_clauses(lex->with_clauses_list)) + goto err; + spec_tables= lex->query_tables; spec_tables_tail= 0; for (TABLE_LIST *tbl= spec_tables; tbl; tbl= tbl->next_global) { + if (!tbl->derived && !tbl->schema_table && + thd->open_temporary_table(tbl)) + goto err; spec_tables_tail= tbl; } if (check_table_access(thd, SELECT_ACL, spec_tables, FALSE, UINT_MAX, FALSE)) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index b058af531c6..44c8236023c 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -3274,6 +3274,9 @@ mysql_execute_command(THD *thd) thd->get_stmt_da()->opt_clear_warning_info(thd->query_id); } + if (check_dependencies_in_with_clauses(thd->lex->with_clauses_list)) + DBUG_RETURN(1); + #ifdef HAVE_REPLICATION if (unlikely(thd->slave_thread)) { @@ -3731,14 +3734,6 @@ mysql_execute_command(THD *thd) ulong privileges_requested= lex->exchange ? SELECT_ACL | FILE_ACL : SELECT_ACL; - /* - The same function must be called for DML commands - when CTEs are supported in DML statements - */ - res= check_dependencies_in_with_clauses(thd->lex->with_clauses_list); - if (res) - break; - if (all_tables) res= check_table_access(thd, privileges_requested, @@ -4165,8 +4160,7 @@ mysql_execute_command(THD *thd) /* Copy temporarily the statement flags to thd for lock_table_names() */ uint save_thd_create_info_options= thd->lex->create_info.options; thd->lex->create_info.options|= create_info.options; - if (!(res= check_dependencies_in_with_clauses(lex->with_clauses_list))) - res= open_and_lock_tables(thd, create_info, lex->query_tables, TRUE, 0); + res= open_and_lock_tables(thd, create_info, lex->query_tables, TRUE, 0); thd->lex->create_info.options= save_thd_create_info_options; if (res) { @@ -4783,8 +4777,7 @@ end_with_restore_list: unit->set_limit(select_lex); - if (!(res= check_dependencies_in_with_clauses(lex->with_clauses_list)) && - !(res=open_and_lock_tables(thd, all_tables, TRUE, 0))) + if (!(res=open_and_lock_tables(thd, all_tables, TRUE, 0))) { MYSQL_INSERT_SELECT_START(thd->query()); /* @@ -5115,9 +5108,6 @@ end_with_restore_list: { List<set_var_base> *lex_var_list= &lex->var_list; - if (check_dependencies_in_with_clauses(thd->lex->with_clauses_list)) - goto error; - if ((check_table_access(thd, SELECT_ACL, all_tables, FALSE, UINT_MAX, FALSE) || open_and_lock_tables(thd, all_tables, TRUE, 0))) goto error; @@ -6436,8 +6426,6 @@ static bool execute_sqlcom_select(THD *thd, TABLE_LIST *all_tables) new (thd->mem_root) Item_int(thd, (ulonglong) thd->variables.select_limit); } - if (check_dependencies_in_with_clauses(lex->with_clauses_list)) - return 1; if (thd->variables.vers_alter_history == VERS_ALTER_HISTORY_SURVIVE) { diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc index b31c2f36db1..80ac2b22a86 100644 --- a/sql/sql_prepare.cc +++ b/sql/sql_prepare.cc @@ -1516,8 +1516,6 @@ static int mysql_test_select(Prepared_statement *stmt, lex->select_lex.context.resolve_in_select_list= TRUE; ulong privilege= lex->exchange ? SELECT_ACL | FILE_ACL : SELECT_ACL; - if (check_dependencies_in_with_clauses(lex->with_clauses_list)) - goto error; if (tables) { if (check_table_access(thd, privilege, tables, FALSE, UINT_MAX, FALSE)) @@ -1784,9 +1782,6 @@ static bool mysql_test_create_table(Prepared_statement *stmt) if (create_table_precheck(thd, tables, create_table)) DBUG_RETURN(TRUE); - if (check_dependencies_in_with_clauses(lex->with_clauses_list)) - DBUG_RETURN(TRUE); - if (select_lex->item_list.elements) { /* Base table and temporary table are not in the same name space. */ @@ -2177,9 +2172,6 @@ static bool mysql_test_insert_select(Prepared_statement *stmt, if (insert_precheck(stmt->thd, tables)) return 1; - if (check_dependencies_in_with_clauses(lex->with_clauses_list)) - return 1; - /* store it, because mysql_insert_select_prepare_tester change it */ first_local_table= lex->select_lex.table_list.first; DBUG_ASSERT(first_local_table != 0); @@ -2282,6 +2274,9 @@ static bool check_prepared_statement(Prepared_statement *stmt) if (tables) thd->get_stmt_da()->opt_clear_warning_info(thd->query_id); + if (check_dependencies_in_with_clauses(thd->lex->with_clauses_list)) + goto error; + if (sql_command_flags[sql_command] & CF_HA_CLOSE) mysql_ha_rm_tables(thd, tables); diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 0527eef4fde..d6fe06173a2 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -18358,7 +18358,7 @@ bool create_internal_tmp_table(TABLE *table, KEY *keyinfo, table->in_use->inc_status_created_tmp_disk_tables(); table->in_use->inc_status_created_tmp_tables(); share->db_record_offset= 1; - table->created= TRUE; + table->set_created(); DBUG_RETURN(0); err: DBUG_RETURN(1); @@ -18869,8 +18869,8 @@ int rr_sequential_and_unpack(READ_RECORD *info) */ bool instantiate_tmp_table(TABLE *table, KEY *keyinfo, - MARIA_COLUMNDEF *start_recinfo, - MARIA_COLUMNDEF **recinfo, + TMP_ENGINE_COLUMNDEF *start_recinfo, + TMP_ENGINE_COLUMNDEF **recinfo, ulonglong options) { if (table->s->db_type() == TMP_ENGINE_HTON) diff --git a/sql/sql_select.h b/sql/sql_select.h index fbadca2255d..298097f1b98 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -2348,8 +2348,8 @@ bool create_internal_tmp_table(TABLE *table, KEY *keyinfo, TMP_ENGINE_COLUMNDEF **recinfo, ulonglong options); bool instantiate_tmp_table(TABLE *table, KEY *keyinfo, - MARIA_COLUMNDEF *start_recinfo, - MARIA_COLUMNDEF **recinfo, + TMP_ENGINE_COLUMNDEF *start_recinfo, + TMP_ENGINE_COLUMNDEF **recinfo, ulonglong options); bool open_tmp_table(TABLE *table); void setup_tmp_table_column_bitmaps(TABLE *table, uchar *bitmaps); diff --git a/sql/sql_union.cc b/sql/sql_union.cc index dda57cc5b4d..f18d30a45c0 100644 --- a/sql/sql_union.cc +++ b/sql/sql_union.cc @@ -1710,6 +1710,7 @@ bool st_select_lex_unit::exec_recursive() sq; sq= sq->next_with_rec_ref) { + sq->reset(); sq->engine->force_reexecution(); } diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index fc13537d364..cda0636146f 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -3,7 +3,7 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -482,10 +482,10 @@ inconsistent: /* In fact, because we only ever append fields to the 'default value' record, it is also OK to perform READ UNCOMMITTED and then ignore any extra fields, provided that - trx_rw_is_active(DB_TRX_ID). */ + trx_sys->rw_trx_hash.find(DB_TRX_ID). */ if (rec_offs_n_fields(offsets) > index->n_fields - && !trx_rw_is_active(row_get_rec_trx_id(rec, index, offsets), - NULL, false)) { + && !trx_sys->rw_trx_hash.find(row_get_rec_trx_id(rec, index, + offsets))) { goto inconsistent; } @@ -919,8 +919,7 @@ search tuple should be performed in the B-tree. InnoDB does an insert immediately after the cursor. Thus, the cursor may end up on a user record, or on a page infimum record. */ dberr_t -btr_cur_search_to_nth_level( -/*========================*/ +btr_cur_search_to_nth_level_func( dict_index_t* index, /*!< in: index */ ulint level, /*!< in: the tree level of search */ const dtuple_t* tuple, /*!< in: data tuple; NOTE: n_fields_cmp in @@ -935,17 +934,16 @@ btr_cur_search_to_nth_level( cursor->left_block is used to store a pointer to the left neighbor page, in the cases BTR_SEARCH_PREV and BTR_MODIFY_PREV; - NOTE that if has_search_latch - is != 0, we maybe do not have a latch set - on the cursor page, we assume - the caller uses his search latch - to protect the record! */ + NOTE that if ahi_latch, we might not have a + cursor page latch, we assume that ahi_latch + protects the record! */ btr_cur_t* cursor, /*!< in/out: tree cursor; the cursor page is s- or x-latched, but see also above! */ - ulint has_search_latch, - /*!< in: info on the latch mode the - caller currently has on search system: - RW_S_LATCH, or 0 */ +#ifdef BTR_CUR_HASH_ADAPT + rw_lock_t* ahi_latch, + /*!< in: currently held btr_search_latch + (in RW_S_LATCH mode), or NULL */ +#endif /* BTR_CUR_HASH_ADAPT */ const char* file, /*!< in: file name */ unsigned line, /*!< in: line where called */ mtr_t* mtr, /*!< in: mtr */ @@ -1123,17 +1121,15 @@ btr_cur_search_to_nth_level( && mode != PAGE_CUR_LE_OR_EXTENDS # endif /* PAGE_CUR_LE_OR_EXTENDS */ && !dict_index_is_spatial(index) - /* If !has_search_latch, we do a dirty read of + /* If !ahi_latch, we do a dirty read of btr_search_enabled below, and btr_search_guess_on_hash() will have to check it again. */ && btr_search_enabled && !modify_external && !(tuple->info_bits & REC_INFO_MIN_REC_FLAG) - && rw_lock_get_writer(btr_get_search_latch(index)) - == RW_LOCK_NOT_LOCKED && btr_search_guess_on_hash(index, info, tuple, mode, latch_mode, cursor, - has_search_latch, mtr)) { + ahi_latch, mtr)) { /* Search using the hash index succeeded */ @@ -1154,10 +1150,12 @@ btr_cur_search_to_nth_level( /* If the hash search did not succeed, do binary search down the tree */ - if (has_search_latch) { +#ifdef BTR_CUR_HASH_ADAPT + if (ahi_latch) { /* Release possible search latch to obey latching order */ - btr_search_s_unlock(index); + rw_lock_s_unlock(ahi_latch); } +#endif /* BTR_CUR_HASH_ADAPT */ /* Store the position of the tree latch we push to mtr so that we know how to release it when we have latched leaf node(s) */ @@ -2224,15 +2222,17 @@ func_exit: ut_free(prev_tree_savepoints); } - if (has_search_latch) { - btr_search_s_lock(index); - } - if (mbr_adj) { /* remember that we will need to adjust parent MBR */ cursor->rtr_info->mbr_adj = true; } +#ifdef BTR_CUR_HASH_ADAPT + if (ahi_latch) { + rw_lock_s_lock(ahi_latch); + } +#endif /* BTR_CUR_HASH_ADAPT */ + DBUG_RETURN(err); } @@ -3302,10 +3302,14 @@ fail_err: ut_ad(entry->info_bits == REC_INFO_DEFAULT_ROW); ut_ad(index->is_instant()); ut_ad(flags == BTR_NO_LOCKING_FLAG); - } else if (!reorg && cursor->flag == BTR_CUR_HASH) { - btr_search_update_hash_node_on_insert(cursor); } else { - btr_search_update_hash_on_insert(cursor); + rw_lock_t* ahi_latch = btr_get_search_latch(index); + if (!reorg && cursor->flag == BTR_CUR_HASH) { + btr_search_update_hash_node_on_insert( + cursor, ahi_latch); + } else { + btr_search_update_hash_on_insert(cursor, ahi_latch); + } } #endif /* BTR_CUR_HASH_ADAPT */ @@ -3511,7 +3515,8 @@ btr_cur_pessimistic_insert( ut_ad((flags & ~BTR_KEEP_IBUF_BITMAP) == BTR_NO_LOCKING_FLAG); } else { - btr_search_update_hash_on_insert(cursor); + btr_search_update_hash_on_insert( + cursor, btr_get_search_latch(index)); } #endif /* BTR_CUR_HASH_ADAPT */ if (inherit && !(flags & BTR_NO_LOCKING_FLAG)) { @@ -3922,34 +3927,39 @@ btr_cur_update_in_place( || row_get_rec_trx_id(rec, index, offsets)); #ifdef BTR_CUR_HASH_ADAPT - if (block->index) { - /* TO DO: Can we skip this if none of the fields - index->search_info->curr_n_fields - are being updated? */ - - /* The function row_upd_changes_ord_field_binary works only - if the update vector was built for a clustered index, we must - NOT call it if index is secondary */ + { + rw_lock_t* ahi_latch = block->index + ? btr_get_search_latch(block->index) : NULL; + if (ahi_latch) { + /* TO DO: Can we skip this if none of the fields + index->search_info->curr_n_fields + are being updated? */ + + /* The function row_upd_changes_ord_field_binary + does not work on a secondary index. */ + + if (!dict_index_is_clust(index) + || row_upd_changes_ord_field_binary( + index, update, thr, NULL, NULL)) { + ut_ad(!(update->info_bits + & REC_INFO_MIN_REC_FLAG)); + /* Remove possible hash index pointer + to this record */ + btr_search_update_hash_on_delete(cursor); + } - if (!dict_index_is_clust(index) - || row_upd_changes_ord_field_binary(index, update, thr, - NULL, NULL)) { - ut_ad(!(update->info_bits & REC_INFO_MIN_REC_FLAG)); - /* Remove possible hash index pointer to this record */ - btr_search_update_hash_on_delete(cursor); + rw_lock_x_lock(ahi_latch); } - btr_search_x_lock(index); - } - - assert_block_ahi_valid(block); + assert_block_ahi_valid(block); #endif /* BTR_CUR_HASH_ADAPT */ - row_upd_rec_in_place(rec, index, offsets, update, page_zip); + row_upd_rec_in_place(rec, index, offsets, update, page_zip); #ifdef BTR_CUR_HASH_ADAPT - if (block->index) { - btr_search_x_unlock(index); + if (ahi_latch) { + rw_lock_x_unlock(ahi_latch); + } } #endif /* BTR_CUR_HASH_ADAPT */ diff --git a/storage/innobase/btr/btr0pcur.cc b/storage/innobase/btr/btr0pcur.cc index 2b85c764a3b..56f6da81ded 100644 --- a/storage/innobase/btr/btr0pcur.cc +++ b/storage/innobase/btr/btr0pcur.cc @@ -353,7 +353,11 @@ btr_pcur_restore_position_func( } btr_pcur_open_with_no_init_func(index, tuple, mode, latch_mode, - cursor, 0, file, line, mtr); + cursor, +#ifdef BTR_CUR_HASH_ADAPT + NULL, +#endif /* BTR_CUR_HASH_ADAPT */ + file, line, mtr); /* Restore the old search mode */ cursor->search_mode = old_mode; diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 6613e4c59f8..ccd3e2f129f 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -45,10 +45,10 @@ Created 2/17/1996 Heikki Tuuri /** Is search system enabled. Search system is protected by array of latches. */ -char btr_search_enabled = true; +char btr_search_enabled; /** Number of adaptive hash index partition. */ -ulong btr_ahi_parts = 8; +ulong btr_ahi_parts; #ifdef UNIV_SEARCH_PERF_STAT /** Number of successful adaptive hash index lookups */ @@ -177,23 +177,6 @@ btr_search_get_n_fields( return(btr_search_get_n_fields(cursor->n_fields, cursor->n_bytes)); } -/********************************************************************//** -Builds a hash index on a page with the given parameters. If the page already -has a hash index with different parameters, the old hash index is removed. -If index is non-NULL, this function checks if n_fields and n_bytes are -sensible values, and does not build a hash index if not. */ -static -void -btr_search_build_page_hash_index( -/*=============================*/ - dict_index_t* index, /*!< in: index for which to build, or NULL if - not known */ - buf_block_t* block, /*!< in: index page, s- or x-latched */ - ulint n_fields,/*!< in: hash this many full fields */ - ulint n_bytes,/*!< in: hash this many bytes from the next - field */ - ibool left_side);/*!< in: hash for searches from left side? */ - /** This function should be called before reserving any btr search mutex, if the intended operation might add nodes to the search system hash table. Because of the latching order, once we have reserved the btr search system @@ -206,13 +189,13 @@ will not guarantee success. @param[in] index index handler */ static void -btr_search_check_free_space_in_heap(dict_index_t* index) +btr_search_check_free_space_in_heap(const dict_index_t* index) { hash_table_t* table; mem_heap_t* heap; - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_S)); - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); + ut_ad(!btr_search_own_any(RW_LOCK_S)); + ut_ad(!btr_search_own_any(RW_LOCK_X)); table = btr_get_search_table(index); @@ -224,8 +207,9 @@ btr_search_check_free_space_in_heap(dict_index_t* index) if (heap->free_block == NULL) { buf_block_t* block = buf_block_alloc(NULL); + rw_lock_t* ahi_latch = btr_get_search_latch(index); - btr_search_x_lock(index); + rw_lock_x_lock(ahi_latch); if (btr_search_enabled && heap->free_block == NULL) { @@ -234,7 +218,7 @@ btr_search_check_free_space_in_heap(dict_index_t* index) buf_block_free(block); } - btr_search_x_unlock(index); + rw_lock_x_unlock(ahi_latch); } } @@ -361,9 +345,6 @@ btr_search_disable_ref_count( for (index = dict_table_get_first_index(table); index != NULL; index = dict_table_get_next_index(index)) { - - ut_ad(rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); - index->search_info->ref_count = 0; } } @@ -457,12 +438,10 @@ btr_search_info_get_ref_count( ut_ad(info); - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_S)); - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); - - btr_search_s_lock(index); + rw_lock_t* ahi_latch = btr_get_search_latch(index); + rw_lock_s_lock(ahi_latch); ret = info->ref_count; - btr_search_s_unlock(index); + rw_lock_s_unlock(ahi_latch); return(ret); } @@ -482,8 +461,8 @@ btr_search_info_update_hash( ulint n_unique; int cmp; - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_S)); - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); + ut_ad(!btr_search_own_any(RW_LOCK_S)); + ut_ad(!btr_search_own_any(RW_LOCK_X)); if (dict_index_is_ibuf(index)) { /* So many deletes are performed on an insert buffer tree @@ -590,14 +569,14 @@ semaphore, to save CPU time! Do not assume the fields are consistent. @param[in,out] block buffer block @param[in] cursor cursor */ static -ibool +bool btr_search_update_block_hash_info( btr_search_t* info, buf_block_t* block, const btr_cur_t* cursor) { - ut_ad(!rw_lock_own(btr_get_search_latch(cursor->index), RW_LOCK_S)); - ut_ad(!rw_lock_own(btr_get_search_latch(cursor->index), RW_LOCK_X)); + ut_ad(!btr_search_own_any(RW_LOCK_S)); + ut_ad(!btr_search_own_any(RW_LOCK_X)); ut_ad(rw_lock_own(&block->lock, RW_LOCK_S) || rw_lock_own(&block->lock, RW_LOCK_X)); @@ -644,11 +623,11 @@ btr_search_update_block_hash_info( /* Build a new hash index on the page */ - return(TRUE); + return(true); } } - return(FALSE); + return(false); } /** Updates a hash node reference when it has been unsuccessfully used in a @@ -688,8 +667,8 @@ btr_search_update_hash_ref( } ut_ad(block->page.id.space() == index->space); - ut_a(index == cursor->index); - ut_a(!dict_index_is_ibuf(index)); + ut_ad(index == cursor->index); + ut_ad(!dict_index_is_ibuf(index)); if ((info->n_hash_potential > 0) && (block->curr_n_fields == info->n_fields) @@ -714,7 +693,6 @@ btr_search_update_hash_ref( if (UNIV_LIKELY_NULL(heap)) { mem_heap_free(heap); } - ut_ad(rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); ha_insert_for_fold(btr_get_search_table(index), fold, block, rec); @@ -723,61 +701,6 @@ btr_search_update_hash_ref( } } -/** Updates the search info. -@param[in,out] info search info -@param[in] cursor cursor which was just positioned */ -void -btr_search_info_update_slow( - btr_search_t* info, - btr_cur_t* cursor) -{ - buf_block_t* block; - ibool build_index; - - ut_ad(!rw_lock_own(btr_get_search_latch(cursor->index), RW_LOCK_S)); - ut_ad(!rw_lock_own(btr_get_search_latch(cursor->index), RW_LOCK_X)); - - block = btr_cur_get_block(cursor); - - /* NOTE that the following two function calls do NOT protect - info or block->n_fields etc. with any semaphore, to save CPU time! - We cannot assume the fields are consistent when we return from - those functions! */ - - btr_search_info_update_hash(info, cursor); - - build_index = btr_search_update_block_hash_info(info, block, cursor); - - if (build_index || (cursor->flag == BTR_CUR_HASH_FAIL)) { - - btr_search_check_free_space_in_heap(cursor->index); - } - - if (cursor->flag == BTR_CUR_HASH_FAIL) { - /* Update the hash node reference, if appropriate */ - -#ifdef UNIV_SEARCH_PERF_STAT - btr_search_n_hash_fail++; -#endif /* UNIV_SEARCH_PERF_STAT */ - - btr_search_x_lock(cursor->index); - - btr_search_update_hash_ref(info, block, cursor); - - btr_search_x_unlock(cursor->index); - } - - if (build_index) { - /* Note that since we did not protect block->n_fields etc. - with any semaphore, the values can be inconsistent. We have - to check inside the function call that they make sense. */ - btr_search_build_page_hash_index(cursor->index, block, - block->n_fields, - block->n_bytes, - block->left_side); - } -} - /** Checks if a guessed position for a tree cursor is right. Note that if mode is PAGE_CUR_LE, which is used in inserts, and the function returns TRUE, then cursor->up_match and cursor->low_match both have sensible values. @@ -791,16 +714,14 @@ TRUE, then cursor->up_match and cursor->low_match both have sensible values. previous record to check our guess! @param[in] tuple data tuple @param[in] mode PAGE_CUR_L, PAGE_CUR_LE, PAGE_CUR_G, PAGE_CUR_GE -@param[in] mtr mini transaction -@return TRUE if success */ +@return whether a match was found */ static -ibool +bool btr_search_check_guess( btr_cur_t* cursor, - ibool can_only_compare_to_cursor_rec, + bool can_only_compare_to_cursor_rec, const dtuple_t* tuple, - ulint mode, - mtr_t* mtr) + ulint mode) { rec_t* rec; ulint n_unique; @@ -862,14 +783,13 @@ btr_search_check_guess( match = 0; if ((mode == PAGE_CUR_G) || (mode == PAGE_CUR_GE)) { - rec_t* prev_rec; - ut_ad(!page_rec_is_infimum(rec)); - prev_rec = page_rec_get_prev(rec); + const rec_t* prev_rec = page_rec_get_prev(rec); if (page_rec_is_infimum(prev_rec)) { - success = btr_page_get_prev(page_align(prev_rec), mtr) + success = *reinterpret_cast<const uint32_t*>( + page_align(prev_rec) + FIL_PAGE_PREV) == FIL_NULL; goto exit_func; @@ -884,17 +804,14 @@ btr_search_check_guess( } else { success = cmp >= 0; } - - goto exit_func; } else { - rec_t* next_rec; - ut_ad(!page_rec_is_supremum(rec)); - next_rec = page_rec_get_next(rec); + const rec_t* next_rec = page_rec_get_next(rec); if (page_rec_is_supremum(next_rec)) { - if (btr_page_get_next(page_align(next_rec), mtr) + if (*reinterpret_cast<const uint32_t*>( + page_align(next_rec) + FIL_PAGE_NEXT) == FIL_NULL) { cursor->up_match = 0; @@ -953,12 +870,11 @@ both have sensible values. we assume the caller uses his search latch to protect the record! @param[out] cursor tree cursor -@param[in] has_search_latch - latch mode the caller currently has on - search system: RW_S/X_LATCH or 0 +@param[in] ahi_latch the adaptive hash index latch being held, + or NULL @param[in] mtr mini transaction -@return TRUE if succeeded */ -ibool +@return whether the search succeeded */ +bool btr_search_guess_on_hash( dict_index_t* index, btr_search_t* info, @@ -966,7 +882,7 @@ btr_search_guess_on_hash( ulint mode, ulint latch_mode, btr_cur_t* cursor, - ulint has_search_latch, + rw_lock_t* ahi_latch, mtr_t* mtr) { const rec_t* rec; @@ -976,6 +892,8 @@ btr_search_guess_on_hash( btr_cur_t cursor2; btr_pcur_t pcur; #endif + ut_ad(!ahi_latch || rw_lock_own(ahi_latch, RW_LOCK_S) + || rw_lock_own(ahi_latch, RW_LOCK_X)); if (!btr_search_enabled) { return(FALSE); @@ -983,6 +901,7 @@ btr_search_guess_on_hash( ut_ad(index && info && tuple && cursor && mtr); ut_ad(!dict_index_is_ibuf(index)); + ut_ad(!ahi_latch || ahi_latch == btr_get_search_latch(index)); ut_ad((latch_mode == BTR_SEARCH_LEAF) || (latch_mode == BTR_MODIFY_LEAF)); @@ -1015,28 +934,26 @@ btr_search_guess_on_hash( cursor->fold = fold; cursor->flag = BTR_CUR_HASH; - if (!has_search_latch) { - btr_search_s_lock(index); + rw_lock_t* use_latch = ahi_latch ? NULL : btr_get_search_latch(index); - if (!btr_search_enabled) { - btr_search_s_unlock(index); + if (use_latch) { + rw_lock_s_lock(use_latch); - btr_search_failure(info, cursor); - - return(FALSE); + if (!btr_search_enabled) { + goto fail; } + } else { + ut_ad(btr_search_enabled); + ut_ad(rw_lock_own(ahi_latch, RW_LOCK_S)); } - ut_ad(rw_lock_get_writer(btr_get_search_latch(index)) != RW_LOCK_X); - ut_ad(rw_lock_get_reader_count(btr_get_search_latch(index)) > 0); - rec = (rec_t*) ha_search_and_get_data( - btr_get_search_table(index), fold); + btr_get_search_table(index), fold); if (rec == NULL) { - - if (!has_search_latch) { - btr_search_s_unlock(index); + if (use_latch) { +fail: + rw_lock_s_unlock(use_latch); } btr_search_failure(info, cursor); @@ -1046,22 +963,15 @@ btr_search_guess_on_hash( buf_block_t* block = buf_block_from_ahi(rec); - if (!has_search_latch) { + if (use_latch) { if (!buf_page_get_known_nowait( latch_mode, block, BUF_MAKE_YOUNG, __FILE__, __LINE__, mtr)) { - - if (!has_search_latch) { - btr_search_s_unlock(index); - } - - btr_search_failure(info, cursor); - - return(FALSE); + goto fail; } - btr_search_s_unlock(index); + rw_lock_s_unlock(use_latch); buf_block_dbg_add_level(block, SYNC_TREE_NODE_FROM_HASH); } @@ -1070,7 +980,7 @@ btr_search_guess_on_hash( ut_ad(buf_block_get_state(block) == BUF_BLOCK_REMOVE_HASH); - if (!has_search_latch) { + if (!ahi_latch) { btr_leaf_page_release(block, latch_mode, mtr); } @@ -1092,11 +1002,9 @@ btr_search_guess_on_hash( record to determine if our guess for the cursor position is right. */ if (index_id != btr_page_get_index_id(block->frame) - || !btr_search_check_guess(cursor, - has_search_latch, - tuple, mode, mtr)) { + || !btr_search_check_guess(cursor, !!ahi_latch, tuple, mode)) { - if (!has_search_latch) { + if (!ahi_latch) { btr_leaf_page_release(block, latch_mode, mtr); } @@ -1117,7 +1025,7 @@ btr_search_guess_on_hash( info->last_hash_succ = FALSE; /* Currently, does not work if the following fails: */ - ut_ad(!has_search_latch); + ut_ad(!ahi_latch); btr_leaf_page_release(block, latch_mode, mtr); @@ -1150,7 +1058,7 @@ btr_search_guess_on_hash( #ifdef UNIV_SEARCH_PERF_STAT btr_search_n_succ++; #endif - if (!has_search_latch && buf_page_peek_if_too_old(&block->page)) { + if (!ahi_latch && buf_page_peek_if_too_old(&block->page)) { buf_page_make_young(&block->page); } @@ -1198,6 +1106,8 @@ retry: /* This debug check uses a dirty read that could theoretically cause false positives while buf_pool_clear_hash_index() is executing. */ assert_block_ahi_valid(block); + ut_ad(!btr_search_own_any(RW_LOCK_S)); + ut_ad(!btr_search_own_any(RW_LOCK_X)); if (index == NULL) { return; @@ -1221,9 +1131,6 @@ retry: % btr_ahi_parts; latch = btr_search_latches[ahi_slot]; - ut_ad(!btr_search_own_any(RW_LOCK_S)); - ut_ad(!btr_search_own_any(RW_LOCK_X)); - rw_lock_s_lock(latch); assert_block_ahi_valid(block); @@ -1426,6 +1333,7 @@ If index is non-NULL, this function checks if n_fields and n_bytes are sensible, and does not build a hash index if not. @param[in,out] index index for which to build. @param[in,out] block index page, s-/x- latched. +@param[in,out] ahi_latch the adaptive search latch @param[in] n_fields hash this many full fields @param[in] n_bytes hash this many bytes of the next field @param[in] left_side hash for searches from left side */ @@ -1434,12 +1342,11 @@ void btr_search_build_page_hash_index( dict_index_t* index, buf_block_t* block, + rw_lock_t* ahi_latch, ulint n_fields, ulint n_bytes, ibool left_side) { - hash_table_t* table; - page_t* page; const rec_t* rec; const rec_t* next_rec; ulint fold; @@ -1461,29 +1368,26 @@ btr_search_build_page_hash_index( } rec_offs_init(offsets_); + ut_ad(ahi_latch == btr_get_search_latch(index)); ut_ad(index); ut_ad(block->page.id.space() == index->space); ut_a(!dict_index_is_ibuf(index)); ut_ad(page_is_leaf(block->frame)); - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); ut_ad(rw_lock_own(&(block->lock), RW_LOCK_S) || rw_lock_own(&(block->lock), RW_LOCK_X)); - btr_search_s_lock(index); + rw_lock_s_lock(ahi_latch); - table = btr_get_search_table(index); - page = buf_block_get_frame(block); - - if (block->index && ((block->curr_n_fields != n_fields) - || (block->curr_n_bytes != n_bytes) - || (block->curr_left_side != left_side))) { + const bool rebuild = block->index + && (block->curr_n_fields != n_fields + || block->curr_n_bytes != n_bytes + || block->curr_left_side != left_side); - btr_search_s_unlock(index); + rw_lock_s_unlock(ahi_latch); + if (rebuild) { btr_search_drop_page_hash_index(block); - } else { - btr_search_s_unlock(index); } /* Check that the values for hash index build are sensible */ @@ -1498,6 +1402,7 @@ btr_search_build_page_hash_index( return; } + page_t* page = buf_block_get_frame(block); n_recs = page_get_n_recs(page); if (n_recs == 0) { @@ -1580,7 +1485,8 @@ btr_search_build_page_hash_index( btr_search_check_free_space_in_heap(index); - btr_search_x_lock(index); + hash_table_t* table = btr_get_search_table(index); + rw_lock_x_lock(ahi_latch); if (!btr_search_enabled) { goto exit_func; @@ -1618,7 +1524,7 @@ btr_search_build_page_hash_index( MONITOR_INC_VALUE(MONITOR_ADAPTIVE_HASH_ROW_ADDED, n_cached); exit_func: assert_block_ahi_valid(block); - btr_search_x_unlock(index); + rw_lock_x_unlock(ahi_latch); ut_free(folds); ut_free(recs); @@ -1627,6 +1533,60 @@ exit_func: } } +/** Updates the search info. +@param[in,out] info search info +@param[in,out] cursor cursor which was just positioned */ +void +btr_search_info_update_slow(btr_search_t* info, btr_cur_t* cursor) +{ + rw_lock_t* ahi_latch = btr_get_search_latch(cursor->index); + + ut_ad(!rw_lock_own(ahi_latch, RW_LOCK_S)); + ut_ad(!rw_lock_own(ahi_latch, RW_LOCK_X)); + + buf_block_t* block = btr_cur_get_block(cursor); + + /* NOTE that the following two function calls do NOT protect + info or block->n_fields etc. with any semaphore, to save CPU time! + We cannot assume the fields are consistent when we return from + those functions! */ + + btr_search_info_update_hash(info, cursor); + + bool build_index = btr_search_update_block_hash_info( + info, block, cursor); + + if (build_index || (cursor->flag == BTR_CUR_HASH_FAIL)) { + + btr_search_check_free_space_in_heap(cursor->index); + } + + if (cursor->flag == BTR_CUR_HASH_FAIL) { + /* Update the hash node reference, if appropriate */ + +#ifdef UNIV_SEARCH_PERF_STAT + btr_search_n_hash_fail++; +#endif /* UNIV_SEARCH_PERF_STAT */ + + rw_lock_x_lock(ahi_latch); + + btr_search_update_hash_ref(info, block, cursor); + + rw_lock_x_unlock(ahi_latch); + } + + if (build_index) { + /* Note that since we did not protect block->n_fields etc. + with any semaphore, the values can be inconsistent. We have + to check inside the function call that they make sense. */ + btr_search_build_page_hash_index(cursor->index, block, + ahi_latch, + block->n_fields, + block->n_bytes, + block->left_side); + } +} + /** Move or delete hash entries for moved records, usually in a page split. If new_block is already hashed, then any hash index for block is dropped. If new_block is not hashed, and block is hashed, then a new hash index is @@ -1645,19 +1605,27 @@ btr_search_move_or_delete_hash_entries( return; } + dict_index_t* index = block->index; + if (!index) { + index = new_block->index; + } else { + ut_ad(!new_block->index || index == new_block->index); + } assert_block_ahi_valid(block); assert_block_ahi_valid(new_block); + rw_lock_t* ahi_latch = index ? btr_get_search_latch(index) : NULL; + if (new_block->index) { btr_search_drop_page_hash_index(block); return; } - dict_index_t* index = block->index; if (!index) { return; } - btr_search_s_lock(index); + + rw_lock_s_lock(ahi_latch); if (block->index) { ulint n_fields = block->curr_n_fields; @@ -1668,19 +1636,20 @@ btr_search_move_or_delete_hash_entries( new_block->n_bytes = block->curr_n_bytes; new_block->left_side = left_side; - btr_search_s_unlock(index); + rw_lock_s_unlock(ahi_latch); ut_a(n_fields > 0 || n_bytes > 0); btr_search_build_page_hash_index( - index, new_block, n_fields, n_bytes, left_side); + index, new_block, ahi_latch, + n_fields, n_bytes, left_side); ut_ad(n_fields == block->curr_n_fields); ut_ad(n_bytes == block->curr_n_bytes); ut_ad(left_side == block->curr_left_side); return; } - btr_search_s_unlock(index); + rw_lock_s_unlock(ahi_latch); } /** Updates the page hash index when a single record is deleted from a page. @@ -1735,7 +1704,9 @@ btr_search_update_hash_on_delete(btr_cur_t* cursor) mem_heap_free(heap); } - btr_search_x_lock(index); + rw_lock_t* ahi_latch = btr_get_search_latch(index); + + rw_lock_x_lock(ahi_latch); assert_block_ahi_valid(block); if (block->index) { @@ -1751,21 +1722,25 @@ btr_search_update_hash_on_delete(btr_cur_t* cursor) assert_block_ahi_valid(block); } - btr_search_x_unlock(index); + rw_lock_x_unlock(ahi_latch); } /** Updates the page hash index when a single record is inserted on a page. @param[in] cursor cursor which was positioned to the place to insert using btr_cur_search_, and the new record has been - inserted next to the cursor. */ + inserted next to the cursor. +@param[in] ahi_latch the adaptive hash index latch */ void -btr_search_update_hash_node_on_insert(btr_cur_t* cursor) +btr_search_update_hash_node_on_insert(btr_cur_t* cursor, rw_lock_t* ahi_latch) { hash_table_t* table; buf_block_t* block; dict_index_t* index; rec_t* rec; + ut_ad(ahi_latch == btr_get_search_latch(cursor->index)); + ut_ad(!btr_search_own_any(RW_LOCK_S)); + ut_ad(!btr_search_own_any(RW_LOCK_X)); #ifdef MYSQL_INDEX_DISABLE_AHI if (cursor->index->disable_ahi) return; #endif @@ -1788,8 +1763,7 @@ btr_search_update_hash_node_on_insert(btr_cur_t* cursor) ut_a(cursor->index == index); ut_a(!dict_index_is_ibuf(index)); - - btr_search_x_lock(index); + rw_lock_x_lock(ahi_latch); if (!block->index) { @@ -1813,11 +1787,11 @@ btr_search_update_hash_node_on_insert(btr_cur_t* cursor) func_exit: assert_block_ahi_valid(block); - btr_search_x_unlock(index); + rw_lock_x_unlock(ahi_latch); } else { - btr_search_x_unlock(index); + rw_lock_x_unlock(ahi_latch); - btr_search_update_hash_on_insert(cursor); + btr_search_update_hash_on_insert(cursor, ahi_latch); } } @@ -1825,9 +1799,10 @@ func_exit: @param[in,out] cursor cursor which was positioned to the place to insert using btr_cur_search_..., and the new record has been inserted next - to the cursor */ + to the cursor +@param[in] ahi_latch the adaptive hash index latch */ void -btr_search_update_hash_on_insert(btr_cur_t* cursor) +btr_search_update_hash_on_insert(btr_cur_t* cursor, rw_lock_t* ahi_latch) { hash_table_t* table; buf_block_t* block; @@ -1841,13 +1816,16 @@ btr_search_update_hash_on_insert(btr_cur_t* cursor) ulint n_fields; ulint n_bytes; ibool left_side; - ibool locked = FALSE; + bool locked = false; mem_heap_t* heap = NULL; ulint offsets_[REC_OFFS_NORMAL_SIZE]; ulint* offsets = offsets_; rec_offs_init(offsets_); + ut_ad(ahi_latch == btr_get_search_latch(cursor->index)); ut_ad(page_is_leaf(btr_cur_get_page(cursor))); + ut_ad(!btr_search_own_any(RW_LOCK_S)); + ut_ad(!btr_search_own_any(RW_LOCK_X)); #ifdef MYSQL_INDEX_DISABLE_AHI if (cursor->index->disable_ahi) return; #endif @@ -1906,10 +1884,8 @@ btr_search_update_hash_on_insert(btr_cur_t* cursor) fold = rec_fold(rec, offsets, n_fields, n_bytes, index->id); } else { if (left_side) { - - btr_search_x_lock(index); - - locked = TRUE; + locked = true; + rw_lock_x_lock(ahi_latch); if (!btr_search_enabled) { goto function_exit; @@ -1924,10 +1900,8 @@ btr_search_update_hash_on_insert(btr_cur_t* cursor) if (fold != ins_fold) { if (!locked) { - - btr_search_x_lock(index); - - locked = TRUE; + locked = true; + rw_lock_x_lock(ahi_latch); if (!btr_search_enabled) { goto function_exit; @@ -1945,11 +1919,9 @@ check_next_rec: if (page_rec_is_supremum(next_rec)) { if (!left_side) { - if (!locked) { - btr_search_x_lock(index); - - locked = TRUE; + locked = true; + rw_lock_x_lock(ahi_latch); if (!btr_search_enabled) { goto function_exit; @@ -1965,10 +1937,8 @@ check_next_rec: if (ins_fold != next_fold) { if (!locked) { - - btr_search_x_lock(index); - - locked = TRUE; + locked = true; + rw_lock_x_lock(ahi_latch); if (!btr_search_enabled) { goto function_exit; @@ -1987,8 +1957,9 @@ function_exit: mem_heap_free(heap); } if (locked) { - btr_search_x_unlock(index); + rw_lock_x_unlock(ahi_latch); } + ut_ad(!rw_lock_own(ahi_latch, RW_LOCK_X)); } #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index df47a49152c..df3154b2130 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -443,9 +443,6 @@ bool buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) MY_ATTRIBUTE((nonnull)); -/* prototypes for new functions added to ha_innodb.cc */ -trx_t* innobase_get_trx(); - /********************************************************************//** Gets the smallest oldest_modification lsn for any page in the pool. Returns zero if all modified pages have been flushed to disk. @@ -4353,11 +4350,7 @@ loop: << ". The most probable cause" " of this error may be that the" " table has been corrupted." - " You can try to fix this" - " problem by using" - " innodb_force_recovery." - " Please see " REFMAN " for more" - " details. Aborting..."; + " See https://mariadb.com/kb/en/library/xtradbinnodb-recovery-modes/"; } #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 26814418033..2d68b8e9ccd 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -80,6 +80,10 @@ static const ulint BUF_LRU_SEARCH_SCAN_THRESHOLD = 100; frames in the buffer pool, we set this to TRUE */ static bool buf_lru_switched_on_innodb_mon = false; +/** True if diagnostic message about difficult to find free blocks +in the buffer bool has already printed. */ +static bool buf_lru_free_blocks_error_printed; + /******************************************************************//** These statistics are not 'of' LRU but 'for' LRU. We keep count of I/O and page_zip_decompress() operations. Based on the statistics, @@ -692,22 +696,26 @@ buf_flush_dirty_pages( /** Empty the flush list for all pages belonging to a tablespace. @param[in] id tablespace identifier @param[in] observer flush observer, - or NULL if nothing is to be written -@param[in] drop_ahi whether to drop the adaptive hash index */ + or NULL if nothing is to be written */ void buf_LRU_flush_or_remove_pages( ulint id, - FlushObserver* observer, - bool drop_ahi) + FlushObserver* observer +#ifdef BTR_CUR_HASH_ADAPT + , bool drop_ahi /*!< whether to drop the adaptive hash index */ +#endif /* BTR_CUR_HASH_ADAPT */ + ) { /* Pages in the system tablespace must never be discarded. */ ut_ad(id || observer); for (ulint i = 0; i < srv_buf_pool_instances; i++) { buf_pool_t* buf_pool = buf_pool_from_array(i); +#ifdef BTR_CUR_HASH_ADAPT if (drop_ahi) { buf_LRU_drop_page_hash_for_tablespace(buf_pool, id); } +#endif /* BTR_CUR_HASH_ADAPT */ buf_flush_dirty_pages(buf_pool, id, observer); } @@ -1079,8 +1087,6 @@ buf_LRU_get_free_block( bool freed = false; ulint n_iterations = 0; ulint flush_failures = 0; - bool mon_value_was = false; - bool started_monitor = false; MONITOR_INC(MONITOR_LRU_GET_FREE_SEARCH); loop: @@ -1088,6 +1094,11 @@ loop: buf_LRU_check_size_of_non_data_objects(buf_pool); + DBUG_EXECUTE_IF("ib_lru_force_no_free_page", + if (!buf_lru_free_blocks_error_printed) { + n_iterations = 21; + goto not_found;}); + /* If there is a block in the free list, take it */ block = buf_LRU_get_free_only(buf_pool); @@ -1097,11 +1108,6 @@ loop: ut_ad(buf_pool_from_block(block) == buf_pool); memset(&block->page.zip, 0, sizeof block->page.zip); - if (started_monitor) { - srv_print_innodb_monitor = - static_cast<my_bool>(mon_value_was); - } - block->skip_flush_check = false; block->page.flush_observer = NULL; return(block); @@ -1131,24 +1137,24 @@ loop: } } +#ifndef DBUG_OFF +not_found: +#endif + buf_pool_mutex_exit(buf_pool); if (freed) { goto loop; } - if (n_iterations > 20 + if (n_iterations > 20 && !buf_lru_free_blocks_error_printed && srv_buf_pool_old_size == srv_buf_pool_size) { ib::warn() << "Difficult to find free blocks in the buffer pool" " (" << n_iterations << " search iterations)! " << flush_failures << " failed attempts to" - " flush a page! Consider increasing the buffer pool" - " size. It is also possible that in your Unix version" - " fsync is very slow, or completely frozen inside" - " the OS kernel. Then upgrading to a newer version" - " of your operating system may help. Look at the" - " number of fsyncs in diagnostic info below." + " flush a page!" + " Consider increasing innodb_buffer_pool_size." " Pending flushes (fsync) log: " << fil_n_pending_log_flushes << "; buffer pool: " @@ -1156,13 +1162,9 @@ loop: << ". " << os_n_file_reads << " OS file reads, " << os_n_file_writes << " OS file writes, " << os_n_fsyncs - << " OS fsyncs. Starting InnoDB Monitor to print" - " further diagnostics to the standard output."; + << " OS fsyncs."; - mon_value_was = srv_print_innodb_monitor; - started_monitor = true; - srv_print_innodb_monitor = true; - os_event_set(srv_monitor_event); + buf_lru_free_blocks_error_printed = true; } /* If we have scanned the whole LRU and still are unable to diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index af4b5cb7347..86a50e70eb3 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1630,7 +1630,11 @@ dict_table_rename_in_cache( return(DB_OUT_OF_MEMORY); } - fil_delete_tablespace(table->space, true); + fil_delete_tablespace(table->space +#ifdef BTR_CUR_HASH_ADAPT + , true +#endif /* BTR_CUR_HASH_ADAPT */ + ); /* Delete any temp file hanging around. */ if (os_file_status(filepath, &exists, &ftype) @@ -3434,6 +3438,7 @@ dict_foreign_find_index( && !(index->type & DICT_FTS) && !dict_index_is_spatial(index) && !index->to_be_dropped + && !dict_index_is_online_ddl(index) && dict_foreign_qualify_index( table, col_names, columns, n_cols, index, types_idx, diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index dd02295eb42..fa65bda4490 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2017, MariaDB Corporation. +Copyright (c) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -566,7 +566,6 @@ bool fil_node_open_file( fil_node_t* node) { - os_offset_t size_bytes; bool success; bool read_only_mode; fil_space_t* space = node->space; @@ -611,7 +610,7 @@ retry: return(false); } - size_bytes = os_file_get_size(node->handle); + os_offset_t size_bytes = os_file_get_size(node->handle); ut_a(size_bytes != (os_offset_t) -1); ut_a(space->purpose != FIL_TYPE_LOG); @@ -694,20 +693,17 @@ retry: space->free_len = free_len; if (first_time_open) { - ulint extent_size; - - extent_size = psize * FSP_EXTENT_SIZE; - - /* After apply-incremental, tablespaces are not extended - to a whole megabyte. Do not cut off valid data. */ - /* Truncate the size to a multiple of extent size. */ - if (size_bytes >= extent_size) { - size_bytes = ut_2pow_round(size_bytes, - extent_size); + ulint mask = psize * FSP_EXTENT_SIZE - 1; + + if (size_bytes <= mask) { + /* .ibd files start smaller than an + extent size. Do not truncate valid data. */ + } else { + size_bytes &= ~os_offset_t(mask); } - node->size = static_cast<ulint>(size_bytes / psize); + node->size = ulint(size_bytes / psize); space->size += node->size; } } @@ -2981,10 +2977,14 @@ fil_table_accessible(const dict_table_t* table) /** Delete a tablespace and associated .ibd file. @param[in] id tablespace identifier -@param[in] drop_ahi whether to drop the adaptive hash index @return DB_SUCCESS or error */ dberr_t -fil_delete_tablespace(ulint id, bool drop_ahi) +fil_delete_tablespace( + ulint id +#ifdef BTR_CUR_HASH_ADAPT + , bool drop_ahi /*!< whether to drop the adaptive hash index */ +#endif /* BTR_CUR_HASH_ADAPT */ + ) { char* path = 0; fil_space_t* space = 0; @@ -3027,7 +3027,11 @@ fil_delete_tablespace(ulint id, bool drop_ahi) To deal with potential read requests, we will check the ::stop_new_ops flag in fil_io(). */ - buf_LRU_flush_or_remove_pages(id, NULL, drop_ahi); + buf_LRU_flush_or_remove_pages(id, NULL +#ifdef BTR_CUR_HASH_ADAPT + , drop_ahi +#endif /* BTR_CUR_HASH_ADAPT */ + ); /* If it is a delete then also delete any generated files, otherwise when we drop the database the remove directory will fail. */ @@ -3307,7 +3311,11 @@ fil_discard_tablespace( { dberr_t err; - switch (err = fil_delete_tablespace(id, true)) { + switch (err = fil_delete_tablespace(id +#ifdef BTR_CUR_HASH_ADAPT + , true +#endif /* BTR_CUR_HASH_ADAPT */ + )) { case DB_SUCCESS: break; diff --git a/storage/innobase/gis/gis0sea.cc b/storage/innobase/gis/gis0sea.cc index 87ebd9ad34a..dcf8cc6f781 100644 --- a/storage/innobase/gis/gis0sea.cc +++ b/storage/innobase/gis/gis0sea.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -773,8 +773,9 @@ rtr_page_get_father_node_ptr( error << ". You should dump + drop + reimport the table to" " fix the corruption. If the crash happens at" - " database startup, see " REFMAN - "forcing-innodb-recovery.html about forcing" + " database startup, see " + "https://mariadb.com/kb/en/library/xtradbinnodb-recovery-modes/" + " about forcing" " recovery. Then dump + drop + reimport."; } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index e8b1ea7350e..2236abf593c 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. Copyright (c) 2012, Facebook Inc. @@ -2819,13 +2819,17 @@ check_trx_exists( return(trx); } -/************************************************************************* -Gets current trx. */ -trx_t* -innobase_get_trx() +/** + Gets current trx. + + This function may be called during InnoDB initialisation, when + innodb_hton_ptr->slot is not yet set to meaningful value. +*/ + +trx_t *current_trx() { THD *thd=current_thd; - if (likely(thd != 0)) { + if (likely(thd != 0) && innodb_hton_ptr->slot != HA_SLOT_UNDEF) { trx_t*& trx = thd_to_trx(thd); return(trx); } else { @@ -21981,7 +21985,8 @@ const char* BUG_REPORT_MSG = "Submit a detailed bug report to https://jira.mariadb.org/"; const char* FORCE_RECOVERY_MSG = - "Please refer to " REFMAN "forcing-innodb-recovery.html" + "Please refer to " + "https://mariadb.com/kb/en/library/xtradbinnodb-recovery-modes/" " for information about forcing recovery."; const char* ERROR_CREATING_MSG = @@ -21989,17 +21994,17 @@ const char* ERROR_CREATING_MSG = const char* OPERATING_SYSTEM_ERROR_MSG = "Some operating system error numbers are described at" - " " REFMAN "operating-system-error-codes.html"; + " https://mariadb.com/kb/en/library/operating-system-error-codes/"; const char* FOREIGN_KEY_CONSTRAINTS_MSG = - "Please refer to " REFMAN "innodb-foreign-key-constraints.html" + "Please refer to https://mariadb.com/kb/en/library/foreign-keys/" " for correct foreign key definition."; const char* SET_TRANSACTION_MSG = - "Please refer to " REFMAN "set-transaction.html"; + "Please refer to https://mariadb.com/kb/en/library/set-transaction/"; const char* INNODB_PARAMETERS_MSG = - "Please refer to " REFMAN "innodb-parameters.html"; + "Please refer to https://mariadb.com/kb/en/library/xtradbinnodb-server-system-variables/"; /********************************************************************** Converts an identifier from my_charset_filename to UTF-8 charset. diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index a8cb749c5b1..8c1ba192b17 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -1177,6 +1177,7 @@ next_column: /* If the table already contains fulltext indexes, refuse to rebuild the table natively altogether. */ if (m_prebuilt->table->fts) { +cannot_create_many_fulltext_index: ha_alter_info->unsupported_reason = innobase_get_err_msg( ER_INNODB_FT_LIMIT); DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); @@ -1206,6 +1207,7 @@ next_column: We could also do ADD SPATIAL INDEX by implementing row_log_apply() for it. */ + bool add_fulltext = false; for (uint i = 0; i < ha_alter_info->index_add_count; i++) { const KEY* key = @@ -1217,16 +1219,18 @@ next_column: | HA_PACK_KEY | HA_GENERATED_KEY | HA_BINARY_PACK_KEY))); + if (add_fulltext) { + goto cannot_create_many_fulltext_index; + } + add_fulltext = true; ha_alter_info->unsupported_reason = innobase_get_err_msg( ER_ALTER_OPERATION_NOT_SUPPORTED_REASON_FTS); online = false; - break; } - if (key->flags & HA_SPATIAL) { + if (online && (key->flags & HA_SPATIAL)) { ha_alter_info->unsupported_reason = innobase_get_err_msg( ER_ALTER_OPERATION_NOT_SUPPORTED_REASON_GIS); online = false; - break; } } } @@ -5326,35 +5330,18 @@ new_clustered_failed: ctx->prepare_instant(); } - if (!ctx->is_instant()) { - if (num_fts_index > 1) { - my_error(ER_INNODB_FT_LIMIT, MYF(0)); - goto error_handled; - } - - if (!ctx->online) { - /* This is not an online operation (LOCK=NONE). */ - } else if (ctx->add_autoinc == ULINT_UNDEFINED - && num_fts_index == 0 - && (!innobase_need_rebuild(ha_alter_info, old_table) - || !innobase_fulltext_exist(altered_table))) { - /* InnoDB can perform an online operation - (LOCK=NONE). */ - } else { - size_t query_length; - /* This should have been blocked in - check_if_supported_inplace_alter(). */ - ut_ad(0); - my_error(ER_NOT_SUPPORTED_YET, MYF(0), - innobase_get_stmt_unsafe( - ctx->prebuilt->trx->mysql_thd, - &query_length)); - goto error_handled; - } - } - if (ctx->need_rebuild()) { not_instant_add_column: + DBUG_ASSERT(ctx->need_rebuild()); + DBUG_ASSERT(!ctx->is_instant()); + DBUG_ASSERT(num_fts_index <= 1); + DBUG_ASSERT(!ctx->online || num_fts_index == 0); + DBUG_ASSERT(!ctx->online + || ctx->add_autoinc == ULINT_UNDEFINED); + DBUG_ASSERT(!ctx->online + || !innobase_need_rebuild(ha_alter_info, old_table) + || !innobase_fulltext_exist(altered_table)); + uint32_t key_id = FIL_DEFAULT_ENCRYPTION_KEY; fil_encryption_t mode = FIL_ENCRYPTION_DEFAULT; @@ -5454,7 +5441,7 @@ new_table_failed: ut_ad(index->trx_id == ctx->trx->id); if (index->type & DICT_FTS) { - DBUG_ASSERT(num_fts_index); + DBUG_ASSERT(num_fts_index == 1); DBUG_ASSERT(!fts_index); DBUG_ASSERT(index->type == DICT_FTS); fts_index = ctx->add_index[a]; @@ -5539,7 +5526,7 @@ error_handling_drop_uncached: /* If ADD INDEX with LOCK=NONE has been requested, allocate a modification log. */ if (index->type & DICT_FTS) { - DBUG_ASSERT(num_fts_index); + DBUG_ASSERT(num_fts_index == 1); DBUG_ASSERT(!fts_index); DBUG_ASSERT(index->type == DICT_FTS); fts_index = ctx->add_index[a]; diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index 76f13325e2a..8d8fe0bc236 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -177,8 +177,7 @@ Note that if mode is PAGE_CUR_LE, which is used in inserts, then cursor->up_match and cursor->low_match both will have sensible values. If mode is PAGE_CUR_GE, then up_match will a have a sensible value. */ dberr_t -btr_cur_search_to_nth_level( -/*========================*/ +btr_cur_search_to_nth_level_func( dict_index_t* index, /*!< in: index */ ulint level, /*!< in: the tree level of search */ const dtuple_t* tuple, /*!< in: data tuple; NOTE: n_fields_cmp in @@ -197,23 +196,29 @@ btr_cur_search_to_nth_level( cursor->left_block is used to store a pointer to the left neighbor page, in the cases BTR_SEARCH_PREV and BTR_MODIFY_PREV; - NOTE that if has_search_latch - is != 0, we maybe do not have a latch set - on the cursor page, we assume - the caller uses his search latch - to protect the record! */ + NOTE that if ahi_latch, we might not have a + cursor page latch, we assume that ahi_latch + protects the record! */ btr_cur_t* cursor, /*!< in/out: tree cursor; the cursor page is s- or x-latched, but see also above! */ - ulint has_search_latch, - /*!< in: latch mode the caller - currently has on search system: - RW_S_LATCH, or 0 */ +#ifdef BTR_CUR_HASH_ADAPT + rw_lock_t* ahi_latch, + /*!< in: currently held btr_search_latch + (in RW_S_LATCH mode), or NULL */ +#endif /* BTR_CUR_HASH_ADAPT */ const char* file, /*!< in: file name */ unsigned line, /*!< in: line where called */ mtr_t* mtr, /*!< in/out: mini-transaction */ ib_uint64_t autoinc = 0); /*!< in: PAGE_ROOT_AUTO_INC to be written (0 if none) */ +#ifdef BTR_CUR_HASH_ADAPT +# define btr_cur_search_to_nth_level(i,l,t,m,lm,c,a,fi,li,mtr) \ + btr_cur_search_to_nth_level_func(i,l,t,m,lm,c,a,fi,li,mtr) +#else /* BTR_CUR_HASH_ADAPT */ +# define btr_cur_search_to_nth_level(i,l,t,m,lm,c,a,fi,li,mtr) \ + btr_cur_search_to_nth_level_func(i,l,t,m,lm,c,fi,li,mtr) +#endif /* BTR_CUR_HASH_ADAPT */ /*****************************************************************//** Opens a cursor at either end of an index. diff --git a/storage/innobase/include/btr0pcur.h b/storage/innobase/include/btr0pcur.h index fab934ca0ee..1d8690a3c90 100644 --- a/storage/innobase/include/btr0pcur.h +++ b/storage/innobase/include/btr0pcur.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -136,20 +136,25 @@ btr_pcur_open_with_no_init_func( may end up on the previous page of the record! */ ulint latch_mode,/*!< in: BTR_SEARCH_LEAF, ...; - NOTE that if has_search_latch != 0 then - we maybe do not acquire a latch on the cursor - page, but assume that the caller uses his - btr search latch to protect the record! */ + NOTE that if ahi_latch then we might not + acquire a cursor page latch, but assume + that the ahi_latch protects the record! */ btr_pcur_t* cursor, /*!< in: memory buffer for persistent cursor */ - ulint has_search_latch, - /*!< in: latch mode the caller - currently has on search system: - RW_S_LATCH, or 0 */ +#ifdef BTR_CUR_HASH_ADAPT + rw_lock_t* ahi_latch, + /*!< in: adaptive hash index latch held + by the caller, or NULL if none */ +#endif /* BTR_CUR_HASH_ADAPT */ const char* file, /*!< in: file name */ unsigned line, /*!< in: line where called */ mtr_t* mtr); /*!< in: mtr */ -#define btr_pcur_open_with_no_init(ix,t,md,l,cur,has,m) \ - btr_pcur_open_with_no_init_func(ix,t,md,l,cur,has,__FILE__,__LINE__,m) +#ifdef BTR_CUR_HASH_ADAPT +# define btr_pcur_open_with_no_init(ix,t,md,l,cur,ahi,m) \ + btr_pcur_open_with_no_init_func(ix,t,md,l,cur,ahi,__FILE__,__LINE__,m) +#else /* BTR_CUR_HASH_ADAPT */ +# define btr_pcur_open_with_no_init(ix,t,md,l,cur,ahi,m) \ + btr_pcur_open_with_no_init_func(ix,t,md,l,cur,__FILE__,__LINE__,m) +#endif /* BTR_CUR_HASH_ADAPT */ /*****************************************************************//** Opens a persistent cursor at either end of an index. */ diff --git a/storage/innobase/include/btr0pcur.ic b/storage/innobase/include/btr0pcur.ic index 4490942a2bb..cbe1d76b094 100644 --- a/storage/innobase/include/btr0pcur.ic +++ b/storage/innobase/include/btr0pcur.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -454,9 +454,12 @@ btr_pcur_open_low( ut_ad(!dict_index_is_spatial(index)); - err = btr_cur_search_to_nth_level( - index, level, tuple, mode, latch_mode, - btr_cursor, 0, file, line, mtr, autoinc); + err = btr_cur_search_to_nth_level_func( + index, level, tuple, mode, latch_mode, btr_cursor, +#ifdef BTR_CUR_HASH_ADAPT + NULL, +#endif /* BTR_CUR_HASH_ADAPT */ + file, line, mtr, autoinc); if (err != DB_SUCCESS) { ib::warn() << " Error code: " << err @@ -491,15 +494,15 @@ btr_pcur_open_with_no_init_func( may end up on the previous page of the record! */ ulint latch_mode,/*!< in: BTR_SEARCH_LEAF, ...; - NOTE that if has_search_latch != 0 then - we maybe do not acquire a latch on the cursor - page, but assume that the caller uses his - btr search latch to protect the record! */ + NOTE that if ahi_latch then we might not + acquire a cursor page latch, but assume + that the ahi_latch protects the record! */ btr_pcur_t* cursor, /*!< in: memory buffer for persistent cursor */ - ulint has_search_latch, - /*!< in: latch mode the caller - currently has on search system: - RW_S_LATCH, or 0 */ +#ifdef BTR_CUR_HASH_ADAPT + rw_lock_t* ahi_latch, + /*!< in: adaptive hash index latch held + by the caller, or NULL if none */ +#endif /* BTR_CUR_HASH_ADAPT */ const char* file, /*!< in: file name */ unsigned line, /*!< in: line where called */ mtr_t* mtr) /*!< in: mtr */ @@ -514,9 +517,12 @@ btr_pcur_open_with_no_init_func( btr_cursor = btr_pcur_get_btr_cur(cursor); - err = btr_cur_search_to_nth_level( + err = btr_cur_search_to_nth_level_func( index, 0, tuple, mode, latch_mode, btr_cursor, - has_search_latch, file, line, mtr); +#ifdef BTR_CUR_HASH_ADAPT + ahi_latch, +#endif /* BTR_CUR_HASH_ADAPT */ + file, line, mtr); cursor->pos_state = BTR_PCUR_IS_POSITIONED; diff --git a/storage/innobase/include/btr0sea.h b/storage/innobase/include/btr0sea.h index bd1a72fc3ac..cddf846438d 100644 --- a/storage/innobase/include/btr0sea.h +++ b/storage/innobase/include/btr0sea.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -91,12 +91,11 @@ both have sensible values. we assume the caller uses his search latch to protect the record! @param[out] cursor tree cursor -@param[in] has_search_latch - latch mode the caller currently has on - search system: RW_S/X_LATCH or 0 +@param[in] ahi_latch the adaptive hash index latch being held, + or NULL @param[in] mtr mini transaction -@return TRUE if succeeded */ -ibool +@return whether the search succeeded */ +bool btr_search_guess_on_hash( dict_index_t* index, btr_search_t* info, @@ -104,7 +103,7 @@ btr_search_guess_on_hash( ulint mode, ulint latch_mode, btr_cur_t* cursor, - ulint has_search_latch, + rw_lock_t* ahi_latch, mtr_t* mtr); /** Move or delete hash entries for moved records, usually in a page split. @@ -140,17 +139,19 @@ btr_search_drop_page_hash_when_freed( /** Updates the page hash index when a single record is inserted on a page. @param[in] cursor cursor which was positioned to the place to insert using btr_cur_search_, and the new record has been - inserted next to the cursor. */ + inserted next to the cursor. +@param[in] ahi_latch the adaptive hash index latch */ void -btr_search_update_hash_node_on_insert(btr_cur_t* cursor); +btr_search_update_hash_node_on_insert(btr_cur_t* cursor, rw_lock_t* ahi_latch); /** Updates the page hash index when a single record is inserted on a page. -@param[in] cursor cursor which was positioned to the +@param[in,out] cursor cursor which was positioned to the place to insert using btr_cur_search_..., and the new record has been inserted next - to the cursor */ + to the cursor +@param[in] ahi_latch the adaptive hash index latch */ void -btr_search_update_hash_on_insert(btr_cur_t* cursor); +btr_search_update_hash_on_insert(btr_cur_t* cursor, rw_lock_t* ahi_latch); /** Updates the page hash index when a single record is deleted from a page. @param[in] cursor cursor which was positioned on the record to delete @@ -163,18 +164,6 @@ btr_search_update_hash_on_delete(btr_cur_t* cursor); bool btr_search_validate(); -/** X-Lock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_x_lock(const dict_index_t* index); - -/** X-Unlock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_x_unlock(const dict_index_t* index); - /** Lock all search latches in exclusive mode. */ UNIV_INLINE void @@ -185,18 +174,6 @@ UNIV_INLINE void btr_search_x_unlock_all(); -/** S-Lock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_s_lock(const dict_index_t* index); - -/** S-Unlock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_s_unlock(const dict_index_t* index); - /** Lock all search latches in shared mode. */ UNIV_INLINE void @@ -243,15 +220,11 @@ btr_get_search_table(const dict_index_t* index); #else /* BTR_CUR_HASH_ADAPT */ # define btr_search_sys_create(size) # define btr_search_drop_page_hash_index(block) -# define btr_search_s_lock(index) -# define btr_search_s_unlock(index) # define btr_search_s_lock_all(index) # define btr_search_s_unlock_all(index) -# define btr_search_x_lock(index) -# define btr_search_x_unlock(index) # define btr_search_info_update(index, cursor) # define btr_search_move_or_delete_hash_entries(new_block, block) -# define btr_search_update_hash_on_insert(cursor) +# define btr_search_update_hash_on_insert(cursor, ahi_latch) # define btr_search_update_hash_on_delete(cursor) # define btr_search_sys_resize(hash_size) #endif /* BTR_CUR_HASH_ADAPT */ diff --git a/storage/innobase/include/btr0sea.ic b/storage/innobase/include/btr0sea.ic index b5a7536a2b4..e0052a98639 100644 --- a/storage/innobase/include/btr0sea.ic +++ b/storage/innobase/include/btr0sea.ic @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -45,13 +46,11 @@ btr_search_info_create(mem_heap_t* heap) } #ifdef BTR_CUR_HASH_ADAPT -/*********************************************************************//** -Updates the search info. */ +/** Updates the search info. +@param[in,out] info search info +@param[in,out] cursor cursor which was just positioned */ void -btr_search_info_update_slow( -/*========================*/ - btr_search_t* info, /*!< in/out: search info */ - btr_cur_t* cursor);/*!< in: cursor which was just positioned */ +btr_search_info_update_slow(btr_search_t* info, btr_cur_t* cursor); /*********************************************************************//** Updates the search info. */ @@ -62,8 +61,8 @@ btr_search_info_update( dict_index_t* index, /*!< in: index of the cursor */ btr_cur_t* cursor) /*!< in: cursor which was just positioned */ { - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_S)); - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); + ut_ad(!btr_search_own_any(RW_LOCK_S)); + ut_ad(!btr_search_own_any(RW_LOCK_X)); if (dict_index_is_spatial(index) || !btr_search_enabled) { return; @@ -87,24 +86,6 @@ btr_search_info_update( btr_search_info_update_slow(info, cursor); } -/** X-Lock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_x_lock(const dict_index_t* index) -{ - rw_lock_x_lock(btr_get_search_latch(index)); -} - -/** X-Unlock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_x_unlock(const dict_index_t* index) -{ - rw_lock_x_unlock(btr_get_search_latch(index)); -} - /** Lock all search latches in exclusive mode. */ UNIV_INLINE void @@ -125,24 +106,6 @@ btr_search_x_unlock_all() } } -/** S-Lock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_s_lock(const dict_index_t* index) -{ - rw_lock_s_lock(btr_get_search_latch(index)); -} - -/** S-Unlock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_s_unlock(const dict_index_t* index) -{ - rw_lock_s_unlock(btr_get_search_latch(index)); -} - /** Lock all search latches in shared mode. */ UNIV_INLINE void diff --git a/storage/innobase/include/buf0lru.h b/storage/innobase/include/buf0lru.h index 7b739fc0332..f6a7695a2b5 100644 --- a/storage/innobase/include/buf0lru.h +++ b/storage/innobase/include/buf0lru.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -53,13 +53,15 @@ These are low-level functions /** Empty the flush list for all pages belonging to a tablespace. @param[in] id tablespace identifier @param[in,out] observer flush observer, - or NULL if nothing is to be written -@param[in] drop_ahi whether to drop the adaptive hash index */ + or NULL if nothing is to be written */ void buf_LRU_flush_or_remove_pages( ulint id, - FlushObserver* observer, - bool drop_ahi = false); + FlushObserver* observer +#ifdef BTR_CUR_HASH_ADAPT + , bool drop_ahi = false /*!< whether to drop the adaptive hash index */ +#endif /* BTR_CUR_HASH_ADAPT */ + ); #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG /********************************************************************//** diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index febf47c1d08..724deb4e6bc 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -946,10 +946,14 @@ fil_table_accessible(const dict_table_t* table) /** Delete a tablespace and associated .ibd file. @param[in] id tablespace identifier -@param[in] drop_ahi whether to drop the adaptive hash index @return DB_SUCCESS or error */ dberr_t -fil_delete_tablespace(ulint id, bool drop_ahi = false); +fil_delete_tablespace( + ulint id +#ifdef BTR_CUR_HASH_ADAPT + , bool drop_ahi = false /*!< whether to drop the adaptive hash index */ +#endif /* BTR_CUR_HASH_ADAPT */ + ); /** Truncate the tablespace to needed size. @param[in] space_id id of tablespace to truncate diff --git a/storage/innobase/include/ha0ha.h b/storage/innobase/include/ha0ha.h index ca4cb0a5f8f..f5be654f490 100644 --- a/storage/innobase/include/ha0ha.h +++ b/storage/innobase/include/ha0ha.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -210,7 +211,7 @@ struct ha_node_t { }; #endif /* BTR_CUR_HASH_ADAPT */ -#ifdef UNIV_DEBUG +#if defined UNIV_DEBUG && defined BTR_CUR_HASH_ADAPT /********************************************************************//** Assert that the synchronization object in a hash operation involving possible change in the hash table is held. diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 6a13d1d9640..779d4d238a0 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -2,7 +2,7 @@ Copyright (c) 1995, 2017, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2009, Google Inc. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -514,9 +514,11 @@ or the MySQL version that created the redo log file. */ #define LOG_HEADER_FORMAT_3_23 0 /** The MySQL 5.7.9/MariaDB 10.2.2 log format */ #define LOG_HEADER_FORMAT_10_2 1 +/** The MariaDB 10.3.2 log format */ +#define LOG_HEADER_FORMAT_10_3 103 /** The redo log format identifier corresponding to the current format version. Stored in LOG_HEADER_FORMAT. */ -#define LOG_HEADER_FORMAT_CURRENT 103 +#define LOG_HEADER_FORMAT_CURRENT LOG_HEADER_FORMAT_10_3 /** Encrypted MariaDB redo log */ #define LOG_HEADER_FORMAT_ENCRYPTED (1U<<31) diff --git a/storage/innobase/include/row0vers.h b/storage/innobase/include/row0vers.h index 576b53358f8..645f11faaad 100644 --- a/storage/innobase/include/row0vers.h +++ b/storage/innobase/include/row0vers.h @@ -41,13 +41,15 @@ class ReadView; /** Determine if an active transaction has inserted or modified a secondary index record. +@param[in,out] caller_trx trx of current thread @param[in] rec secondary index record @param[in] index secondary index @param[in] offsets rec_get_offsets(rec, index) -@return the active transaction; trx_release_reference() must be invoked +@return the active transaction; trx->release_reference() must be invoked @retval NULL if the record was committed */ trx_t* row_vers_impl_x_locked( + trx_t* caller_trx, const rec_t* rec, dict_index_t* index, const ulint* offsets); @@ -126,6 +128,7 @@ which should be seen by a semi-consistent read. */ void row_vers_build_for_semi_consistent_read( /*====================================*/ + trx_t* caller_trx,/*!<in/out: trx of current thread */ const rec_t* rec, /*!< in: record in a clustered index; the caller must have a latch on the page; this latch locks the top of the stack of versions diff --git a/storage/innobase/include/sync0sync.h b/storage/innobase/include/sync0sync.h index aed8f769716..319db158e98 100644 --- a/storage/innobase/include/sync0sync.h +++ b/storage/innobase/include/sync0sync.h @@ -112,6 +112,7 @@ extern mysql_pfs_key_t sync_array_mutex_key; extern mysql_pfs_key_t thread_mutex_key; extern mysql_pfs_key_t zip_pad_mutex_key; extern mysql_pfs_key_t row_drop_list_mutex_key; +extern mysql_pfs_key_t rw_trx_hash_element_mutex_key; #endif /* UNIV_PFS_MUTEX */ #ifdef UNIV_PFS_RWLOCK diff --git a/storage/innobase/include/sync0types.h b/storage/innobase/include/sync0types.h index 136626b65d5..d300b78cf60 100644 --- a/storage/innobase/include/sync0types.h +++ b/storage/innobase/include/sync0types.h @@ -233,6 +233,7 @@ enum latch_level_t { SYNC_REC_LOCK, SYNC_THREADS, SYNC_TRX, + SYNC_RW_TRX_HASH_ELEMENT, SYNC_TRX_SYS, SYNC_LOCK_SYS, SYNC_LOCK_WAIT_SYS, @@ -383,6 +384,7 @@ enum latch_id_t { LATCH_ID_FIL_CRYPT_STAT_MUTEX, LATCH_ID_FIL_CRYPT_DATA_MUTEX, LATCH_ID_FIL_CRYPT_THREADS_MUTEX, + LATCH_ID_RW_TRX_HASH_ELEMENT, LATCH_ID_TEST_MUTEX, LATCH_ID_MAX = LATCH_ID_TEST_MUTEX }; diff --git a/storage/innobase/include/trx0sys.h b/storage/innobase/include/trx0sys.h index dd8929911c6..e87aa1bef6c 100644 --- a/storage/innobase/include/trx0sys.h +++ b/storage/innobase/include/trx0sys.h @@ -189,14 +189,6 @@ inline bool trx_id_check(const void* db_trx_id, trx_id_t trx_id) #endif /****************************************************************//** -Looks for the trx instance with the given id in the rw trx_list. -@return the trx handle or NULL if not found */ -UNIV_INLINE -trx_t* -trx_get_rw_trx_by_id( -/*=================*/ - trx_id_t trx_id);/*!< in: trx id to search for */ -/****************************************************************//** Returns the minimum trx id in rw trx list. This is the smallest id for which the trx can possibly be active. (But, you must look at the trx->state to find out if the minimum trx id transaction itself is active, or already @@ -206,41 +198,6 @@ UNIV_INLINE trx_id_t trx_rw_min_trx_id(void); /*===================*/ -/****************************************************************//** -Checks if a rw transaction with the given id is active. -@return transaction instance if active, or NULL */ -UNIV_INLINE -trx_t* -trx_rw_is_active_low( -/*=================*/ - trx_id_t trx_id, /*!< in: trx id of the transaction */ - ibool* corrupt); /*!< in: NULL or pointer to a flag - that will be set if corrupt */ -/****************************************************************//** -Checks if a rw transaction with the given id is active. If the caller is -not holding trx_sys->mutex, the transaction may already have been -committed. -@return transaction instance if active, or NULL; */ -UNIV_INLINE -trx_t* -trx_rw_is_active( -/*=============*/ - trx_id_t trx_id, /*!< in: trx id of the transaction */ - ibool* corrupt, /*!< in: NULL or pointer to a flag - that will be set if corrupt */ - bool do_ref_count); /*!< in: if true then increment the - trx_t::n_ref_count */ -#if defined UNIV_DEBUG || defined UNIV_BLOB_LIGHT_DEBUG -/***********************************************************//** -Assert that a transaction has been recovered. -@return TRUE */ -UNIV_INLINE -ibool -trx_assert_recovered( -/*=================*/ - trx_id_t trx_id) /*!< in: transaction identifier */ - MY_ATTRIBUTE((warn_unused_result)); -#endif /* UNIV_DEBUG || UNIV_BLOB_LIGHT_DEBUG */ /*****************************************************************//** Updates the offset information about the end of the MySQL binlog entry which corresponds to the transaction just being committed. In a MySQL @@ -302,13 +259,6 @@ ulint trx_sys_any_active_transactions(void); /*=================================*/ -/** -Add the transaction to the RW transaction set -@param trx transaction instance to add */ -UNIV_INLINE -void -trx_sys_rw_trx_add(trx_t* trx); - #ifdef UNIV_DEBUG /*************************************************************//** Validate the trx_sys_t::rw_trx_list. @@ -497,6 +447,281 @@ FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID. */ #define TRX_SYS_DOUBLEWRITE_BLOCK_SIZE FSP_EXTENT_SIZE /* @} */ +trx_t* current_trx(); + +struct rw_trx_hash_element_t +{ + rw_trx_hash_element_t(): trx(0) + { + mutex_create(LATCH_ID_RW_TRX_HASH_ELEMENT, &mutex); + } + + + ~rw_trx_hash_element_t() + { + mutex_free(&mutex); + } + + + trx_id_t id; /* lf_hash_init() relies on this to be first in the struct */ + trx_t *trx; + ib_mutex_t mutex; +}; + + +/** + Wrapper around LF_HASH to store set of in memory read-write transactions. +*/ + +class rw_trx_hash_t +{ + LF_HASH hash; + + + /** + Constructor callback for lock-free allocator. + + Object is just allocated and is not yet accessible via rw_trx_hash by + concurrent threads. Object can be reused multiple times before it is freed. + Every time object is being reused initializer() callback is called. + */ + + static void rw_trx_hash_constructor(uchar *arg) + { + new(arg + LF_HASH_OVERHEAD) rw_trx_hash_element_t(); + } + + + /** + Destructor callback for lock-free allocator. + + Object is about to be freed and is not accessible via rw_trx_hash by + concurrent threads. + */ + + static void rw_trx_hash_destructor(uchar *arg) + { + reinterpret_cast<rw_trx_hash_element_t*> + (arg + LF_HASH_OVERHEAD)->~rw_trx_hash_element_t(); + } + + + /** + Initializer callback for lock-free hash. + + Object is not yet accessible via rw_trx_hash by concurrent threads, but is + about to become such. Object id can be changed only by this callback and + remains the same until all pins to this object are released. + + Object trx can be changed to 0 by erase() under object mutex protection, + which indicates it is about to be removed from lock-free hash and become + not accessible by concurrent threads. + */ + + static void rw_trx_hash_initializer(LF_HASH *, + rw_trx_hash_element_t *element, + trx_t *trx) + { + element->trx= trx; + element->id= trx->id; + trx->rw_trx_hash_element= element; + } + + + /** + Gets LF_HASH pins. + + Pins are used to protect object from being destroyed or reused. They are + normally stored in trx object for quick access. If caller doesn't have trx + available, we try to get it using currnet_trx(). If caller doesn't have trx + at all, temporary pins are allocated. + */ + + LF_PINS *get_pins(trx_t *trx) + { + if (!trx->rw_trx_hash_pins) + { + trx->rw_trx_hash_pins= lf_hash_get_pins(&hash); + ut_a(trx->rw_trx_hash_pins); + } + return trx->rw_trx_hash_pins; + } + + +public: + void init() + { + lf_hash_init(&hash, sizeof(rw_trx_hash_element_t), LF_HASH_UNIQUE, 0, + sizeof(trx_id_t), 0, &my_charset_bin); + hash.alloc.constructor= rw_trx_hash_constructor; + hash.alloc.destructor= rw_trx_hash_destructor; + hash.initializer= + reinterpret_cast<lf_hash_initializer>(rw_trx_hash_initializer); + } + + + void destroy() + { + lf_hash_destroy(&hash); + } + + + /** + Releases LF_HASH pins. + + Must be called by thread that owns trx_t object when the latter is being + "detached" from thread (e.g. released to the pool by trx_free()). Can be + called earlier if thread is expected not to use rw_trx_hash. + + Since pins are not allowed to be transferred to another thread, + initialisation thread calls this for recovered transactions. + */ + + void put_pins(trx_t *trx) + { + if (trx->rw_trx_hash_pins) + { + lf_hash_put_pins(trx->rw_trx_hash_pins); + trx->rw_trx_hash_pins= 0; + } + } + + + /** + Finds trx object in lock-free hash with given id. + + Only ACTIVE or PREPARED trx objects may participate in hash. Nevertheless + the transaction may get committed before this method returns. + + With do_ref_count == false the caller may dereference returned trx pointer + only if lock_sys->mutex was acquired before calling find(). + + With do_ref_count == true caller may dereference trx even if it is not + holding lock_sys->mutex. Caller is responsible for calling + trx->release_reference() when it is done playing with trx. + + Ideally this method should get caller rw_trx_hash_pins along with trx + object as a parameter, similar to insert() and erase(). However most + callers lose trx early in their call chains and it is not that easy to pass + them through. + + So we take more expensive approach: get trx through current_thd()->ha_data. + Some threads don't have trx attached to THD, and at least server + initialisation thread, fts_optimize_thread, srv_master_thread, + dict_stats_thread, srv_monitor_thread, btr_defragment_thread don't even + have THD at all. For such cases we allocate pins only for duration of + search and free them immediately. + + This has negative performance impact and should be fixed eventually (by + passing caller_trx as a parameter). Still stream of DML is more or less Ok. + + @return + @retval 0 not found + @retval pointer to trx + */ + + trx_t *find(trx_t *caller_trx, trx_id_t trx_id, bool do_ref_count= false) + { + /* + In MariaDB 10.3, purge will reset DB_TRX_ID to 0 + when the history is lost. Read/write transactions will + always have a nonzero trx_t::id; there the value 0 is + reserved for transactions that did not write or lock + anything yet. + */ + if (!trx_id) + return NULL; + + trx_t *trx= 0; + LF_PINS *pins= caller_trx ? get_pins(caller_trx) : lf_hash_get_pins(&hash); + ut_a(pins); + + rw_trx_hash_element_t *element= reinterpret_cast<rw_trx_hash_element_t*> + (lf_hash_search(&hash, pins, reinterpret_cast<const void*>(&trx_id), + sizeof(trx_id_t))); + if (element) + { + mutex_enter(&element->mutex); + lf_hash_search_unpin(pins); + if ((trx= element->trx)) + { + if (do_ref_count) + trx->reference(); +#ifdef UNIV_DEBUG + mutex_enter(&trx->mutex); + ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || + trx_state_eq(trx, TRX_STATE_PREPARED)); + mutex_exit(&trx->mutex); +#endif + } + mutex_exit(&element->mutex); + } + if (!caller_trx) + lf_hash_put_pins(pins); + return trx; + } + + + trx_t *find(trx_id_t trx_id, bool do_ref_count= false) + { + return find(current_trx(), trx_id, do_ref_count); + } + + + /** + Inserts trx to lock-free hash. + + Object becomes accessible via rw_trx_hash. + */ + + void insert(trx_t *trx) + { + ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || + trx_state_eq(trx, TRX_STATE_PREPARED)); + int res= lf_hash_insert(&hash, get_pins(trx), + reinterpret_cast<void*>(trx)); + ut_a(res == 0); + } + + + /** + Removes trx from lock-free hash. + + Object becomes not accessible via rw_trx_hash. But it still can be pinned + by concurrent find(), which is supposed to release it immediately after + it sees object trx is 0. + */ + + void erase(trx_t *trx) + { + ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || + trx_state_eq(trx, TRX_STATE_PREPARED)); + mutex_enter(&trx->rw_trx_hash_element->mutex); + trx->rw_trx_hash_element->trx= 0; + mutex_exit(&trx->rw_trx_hash_element->mutex); + int res= lf_hash_delete(&hash, get_pins(trx), + reinterpret_cast<const void*>(&trx->id), + sizeof(trx_id_t)); + ut_a(res == 0); + } + + + /** + Returns the number of elements in the hash. + + The number is exact only if hash is protected against concurrent + modifications (e.g. single threaded startup or hash is protected + by some mutex). Otherwise the number may be used as a hint only, + because it may change even before this method returns. + */ + + int32_t size() + { + return my_atomic_load32_explicit(&hash.count, MY_MEMORY_ORDER_RELAXED); + } +}; + + /** The transaction system central memory data structure. */ struct trx_sys_t { @@ -569,8 +794,16 @@ struct trx_sys_t { transactions), protected by rseg->mutex */ - TrxIdSet rw_trx_set; /*!< Mapping from transaction id - to transaction instance */ + const char rw_trx_hash_pre_pad[CACHE_LINE_SIZE]; + + + /** + Lock-free hash of in memory read-write transactions. + Works faster when it is on it's own cache line (tested). + */ + + rw_trx_hash_t rw_trx_hash; + const char rw_trx_hash_post_pad[CACHE_LINE_SIZE]; ulint n_prepared_trx; /*!< Number of transactions currently in the XA PREPARED state */ diff --git a/storage/innobase/include/trx0sys.ic b/storage/innobase/include/trx0sys.ic index 020a46fb028..5c2a913554a 100644 --- a/storage/innobase/include/trx0sys.ic +++ b/storage/innobase/include/trx0sys.ic @@ -193,32 +193,6 @@ trx_write_trx_id( } /****************************************************************//** -Looks for the trx handle with the given id in rw_trx_list. -The caller must be holding trx_sys->mutex. -@return the trx handle or NULL if not found; -the pointer must not be dereferenced unless lock_sys->mutex was -acquired before calling this function and is still being held */ -UNIV_INLINE -trx_t* -trx_get_rw_trx_by_id( -/*=================*/ - trx_id_t trx_id) /*!< in: trx id to search for */ -{ - ut_ad(trx_id > 0); - ut_ad(trx_sys_mutex_own()); - - if (trx_sys->rw_trx_set.empty()) { - return(NULL); - } - - TrxIdSet::iterator it; - - it = trx_sys->rw_trx_set.find(TrxTrack(trx_id)); - - return(it == trx_sys->rw_trx_set.end() ? NULL : it->m_trx); -} - -/****************************************************************//** Returns the minimum trx id in trx list. This is the smallest id for which the trx can possibly be active. (But, you must look at the trx->state to find out if the minimum trx id transaction itself is active, or already @@ -245,29 +219,6 @@ trx_rw_min_trx_id_low(void) return(id); } -#if defined UNIV_DEBUG || defined UNIV_BLOB_LIGHT_DEBUG -/***********************************************************//** -Assert that a transaction has been recovered. -@return TRUE */ -UNIV_INLINE -ibool -trx_assert_recovered( -/*=================*/ - trx_id_t trx_id) /*!< in: transaction identifier */ -{ - const trx_t* trx; - - trx_sys_mutex_enter(); - - trx = trx_get_rw_trx_by_id(trx_id); - ut_a(trx->is_recovered); - - trx_sys_mutex_exit(); - - return(TRUE); -} -#endif /* UNIV_DEBUG || UNIV_BLOB_LIGHT_DEBUG */ - /****************************************************************//** Returns the minimum trx id in rw trx list. This is the smallest id for which the rw trx can possibly be active. (But, you must look at the trx->state @@ -288,86 +239,6 @@ trx_rw_min_trx_id(void) return(id); } -/****************************************************************//** -Checks if a rw transaction with the given id is active. If the caller is -not holding lock_sys->mutex, the transaction may already have been committed. -@return transaction instance if active, or NULL */ -UNIV_INLINE -trx_t* -trx_rw_is_active_low( -/*=================*/ - trx_id_t trx_id, /*!< in: trx id of the transaction */ - ibool* corrupt) /*!< in: NULL or pointer to a flag - that will be set if corrupt */ -{ - trx_t* trx; - - ut_ad(trx_sys_mutex_own()); - - if (trx_id < trx_rw_min_trx_id_low()) { - - trx = NULL; - } else if (trx_id >= trx_sys->max_trx_id) { - - /* There must be corruption: we let the caller handle the - diagnostic prints in this case. */ - - trx = NULL; - if (corrupt != NULL) { - *corrupt = TRUE; - } - } else { - trx = trx_get_rw_trx_by_id(trx_id); - - if (trx != NULL - && trx_state_eq(trx, TRX_STATE_COMMITTED_IN_MEMORY)) { - - trx = NULL; - } - } - - return(trx); -} - -/****************************************************************//** -Checks if a rw transaction with the given id is active. If the caller is -not holding lock_sys->mutex, the transaction may already have been -committed. -@return transaction instance if active, or NULL; */ -UNIV_INLINE -trx_t* -trx_rw_is_active( -/*=============*/ - trx_id_t trx_id, /*!< in: trx id of the transaction */ - ibool* corrupt, /*!< in: NULL or pointer to a flag - that will be set if corrupt */ - bool do_ref_count) /*!< in: if true then increment the - trx_t::n_ref_count */ -{ - if (!trx_id) { - /* In MariaDB 10.3, purge will reset DB_TRX_ID to 0 - when the history is lost. Read/write transactions will - always have a nonzero trx_t::id; there the value 0 is - reserved for transactions that did not write or lock - anything yet. */ - return NULL; - } - - trx_t* trx; - - trx_sys_mutex_enter(); - - trx = trx_rw_is_active_low(trx_id, corrupt); - - if (trx != 0) { - trx = trx_reference(trx, do_ref_count); - } - - trx_sys_mutex_exit(); - - return(trx); -} - /*****************************************************************//** Allocates a new transaction id. @return new, allocated trx id */ @@ -441,16 +312,3 @@ trx_sys_get_n_rw_trx(void) return(n_trx); } - -/** -Add the transaction to the RW transaction set -@param trx transaction instance to add */ -UNIV_INLINE -void -trx_sys_rw_trx_add(trx_t* trx) -{ - ut_ad(trx->id != 0); - - trx_sys->rw_trx_set.insert(TrxTrack(trx->id, trx)); - ut_d(trx->in_rw_trx_list = true); -} diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index 9f9369b5f67..28f6f2eff12 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -55,6 +55,8 @@ class ReadView; // Forward declaration class FlushObserver; +struct rw_trx_hash_element_t; + /** Dummy session used currently in MySQL interface */ extern sess_t* trx_dummy_sess; @@ -531,31 +533,6 @@ trx_set_rw_mode( trx_t* trx); /** -Increase the reference count. If the transaction is in state -TRX_STATE_COMMITTED_IN_MEMORY then the transaction is considered -committed and the reference count is not incremented. -@param trx Transaction that is being referenced -@param do_ref_count Increment the reference iff this is true -@return transaction instance if it is not committed */ -UNIV_INLINE -trx_t* -trx_reference( - trx_t* trx, - bool do_ref_count); - -/** -Release the transaction. Decrease the reference count. -@param trx Transaction that is being released */ -UNIV_INLINE -void -trx_release_reference( - trx_t* trx); - -/** -Check if the transaction is being referenced. */ -#define trx_is_referenced(t) ((t)->n_ref > 0) - -/** @param[in] requestor Transaction requesting the lock @param[in] holder Transaction holding the lock @return the transaction that will be rolled back, null don't care */ @@ -952,6 +929,19 @@ struct TrxVersion { typedef std::list<TrxVersion, ut_allocator<TrxVersion> > hit_list_t; struct trx_t { +private: + /** + Count of references. + + We can't release the locks nor commit the transaction until this reference + is 0. We can change the state to TRX_STATE_COMMITTED_IN_MEMORY to signify + that it is no longer "active". + */ + + int32_t n_ref; + + +public: TrxMutex mutex; /*!< Mutex protecting the fields state and lock (except some fields of lock, which are protected by @@ -1011,6 +1001,9 @@ struct trx_t { Recovered XA: * NOT_STARTED -> PREPARED -> COMMITTED -> (freed) + Recovered XA followed by XA ROLLBACK: + * NOT_STARTED -> PREPARED -> ACTIVE -> COMMITTED -> (freed) + XA (2PC) (shutdown or disconnect before ROLLBACK or COMMIT): * NOT_STARTED -> PREPARED -> (freed) @@ -1295,14 +1288,6 @@ struct trx_t { const char* start_file; /*!< Filename where it was started */ #endif /* UNIV_DEBUG */ - lint n_ref; /*!< Count of references, protected - by trx_t::mutex. We can't release the - locks nor commit the transaction until - this reference is 0. We can change - the state to COMMITTED_IN_MEMORY to - signify that it is no longer - "active". */ - /** Version of this instance. It is incremented each time the instance is re-used in trx_start_low(). It is used to track whether a transaction has been restarted since it was tagged @@ -1337,6 +1322,8 @@ struct trx_t { os_event_t wsrep_event; /* event waited for in srv_conc_slot */ #endif /* WITH_WSREP */ + rw_trx_hash_element_t *rw_trx_hash_element; + LF_PINS *rw_trx_hash_pins; ulint magic_n; /** @return whether any persistent undo log has been generated */ @@ -1369,6 +1356,33 @@ struct trx_t { return(assign_temp_rseg()); } + + bool is_referenced() + { + return my_atomic_load32_explicit(&n_ref, MY_MEMORY_ORDER_RELAXED) > 0; + } + + + void reference() + { +#ifdef UNIV_DEBUG + int32_t old_n_ref= +#endif + my_atomic_add32_explicit(&n_ref, 1, MY_MEMORY_ORDER_RELAXED); + ut_ad(old_n_ref >= 0); + } + + + void release_reference() + { +#ifdef UNIV_DEBUG + int32_t old_n_ref= +#endif + my_atomic_add32_explicit(&n_ref, -1, MY_MEMORY_ORDER_RELAXED); + ut_ad(old_n_ref > 0); + } + + private: /** Assign a rollback segment for modifying temporary tables. @return the assigned rollback segment */ diff --git a/storage/innobase/include/trx0trx.ic b/storage/innobase/include/trx0trx.ic index 6fa00c5333f..c288da8bf0b 100644 --- a/storage/innobase/include/trx0trx.ic +++ b/storage/innobase/include/trx0trx.ic @@ -214,52 +214,6 @@ ok: } /** -Increase the reference count. If the transaction is in state -TRX_STATE_COMMITTED_IN_MEMORY then the transaction is considered -committed and the reference count is not incremented. -@param trx Transaction that is being referenced -@param do_ref_count Increment the reference iff this is true -@return transaction instance if it is not committed */ -UNIV_INLINE -trx_t* -trx_reference( - trx_t* trx, - bool do_ref_count) -{ - trx_mutex_enter(trx); - - if (trx_state_eq(trx, TRX_STATE_COMMITTED_IN_MEMORY)) { - trx_mutex_exit(trx); - trx = NULL; - } else if (do_ref_count) { - ut_ad(trx->n_ref >= 0); - ++trx->n_ref; - trx_mutex_exit(trx); - } else { - trx_mutex_exit(trx); - } - - return(trx); -} - -/** -Release the transaction. Decrease the reference count. -@param trx Transaction that is being released */ -UNIV_INLINE -void -trx_release_reference( - trx_t* trx) -{ - trx_mutex_enter(trx); - - ut_ad(trx->n_ref > 0); - --trx->n_ref; - - trx_mutex_exit(trx); -} - - -/** @param trx Get the active view for this transaction, if one exists @return the transaction's read view or NULL if one not assigned. */ UNIV_INLINE diff --git a/storage/innobase/include/trx0types.h b/storage/innobase/include/trx0types.h index be8ef3398ad..fb023ff9cd1 100644 --- a/storage/innobase/include/trx0types.h +++ b/storage/innobase/include/trx0types.h @@ -31,12 +31,9 @@ Created 3/26/1996 Heikki Tuuri #include "ut0mutex.h" #include "ut0new.h" -#include <set> #include <queue> #include <vector> -//#include <unordered_set> - /** printf(3) format used for printing DB_TRX_ID and other system fields */ #define TRX_ID_FMT IB_ID_FMT @@ -170,51 +167,4 @@ typedef ib_mutex_t PQMutex; typedef ib_mutex_t TrxSysMutex; typedef std::vector<trx_id_t, ut_allocator<trx_id_t> > trx_ids_t; - -/** Mapping read-write transactions from id to transaction instance, for -creating read views and during trx id lookup for MVCC and locking. */ -struct TrxTrack { - explicit TrxTrack(trx_id_t id, trx_t* trx = NULL) - : - m_id(id), - m_trx(trx) - { - // Do nothing - } - - trx_id_t m_id; - trx_t* m_trx; -}; - -struct TrxTrackHash { - size_t operator()(const TrxTrack& key) const - { - return(size_t(key.m_id)); - } -}; - -/** -Comparator for TrxMap */ -struct TrxTrackHashCmp { - - bool operator() (const TrxTrack& lhs, const TrxTrack& rhs) const - { - return(lhs.m_id == rhs.m_id); - } -}; - -/** -Comparator for TrxMap */ -struct TrxTrackCmp { - - bool operator() (const TrxTrack& lhs, const TrxTrack& rhs) const - { - return(lhs.m_id < rhs.m_id); - } -}; - -//typedef std::unordered_set<TrxTrack, TrxTrackHash, TrxTrackHashCmp> TrxIdSet; -typedef std::set<TrxTrack, TrxTrackCmp, ut_allocator<TrxTrack> > - TrxIdSet; - #endif /* trx0types_h */ diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 2f66e178daf..09b900d1554 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2017, MariaDB Corporation. +Copyright (c) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -459,11 +459,7 @@ lock_sec_rec_cons_read_sees( /* NOTE that we might call this function while holding the search system latch. */ - if (recv_recovery_is_on()) { - - return(false); - - } else if (dict_table_is_temporary(index->table)) { + if (dict_table_is_temporary(index->table)) { /* Temp-tables are not shared across connections and multiple transactions from different connections cannot simultaneously @@ -1494,6 +1490,7 @@ static trx_t* lock_sec_rec_some_has_impl( /*=======================*/ + trx_t* caller_trx,/*!<in/out: trx of current thread */ const rec_t* rec, /*!< in: user record */ dict_index_t* index, /*!< in: secondary index */ const ulint* offsets)/*!< in: rec_get_offsets(rec, index) */ @@ -1517,7 +1514,7 @@ lock_sec_rec_some_has_impl( max trx id to the log, and therefore during recovery, this value for a page may be incorrect. */ - if (max_trx_id < trx_rw_min_trx_id() && !recv_recovery_is_on()) { + if (max_trx_id < trx_rw_min_trx_id()) { trx = 0; @@ -1530,7 +1527,7 @@ lock_sec_rec_some_has_impl( x-lock. We have to look in the clustered index. */ } else { - trx = row_vers_impl_x_locked(rec, index, offsets); + trx = row_vers_impl_x_locked(caller_trx, rec, index, offsets); } return(trx); @@ -1540,6 +1537,11 @@ lock_sec_rec_some_has_impl( /*********************************************************************//** Checks if some transaction, other than given trx_id, has an explicit lock on the given rec, in the given precise_mode. + +FIXME: if the current transaction holds implicit lock from INSERT, a +subsequent locking read should not convert it to explicit. See also +MDEV-11215. + @return the transaction, whose id is not equal to trx_id, that has an explicit lock on the given rec, in the given precise_mode or NULL.*/ static @@ -1558,31 +1560,28 @@ lock_rec_other_trx_holds_expl( ut_ad(!page_rec_is_default_row(rec)); trx_t* holds = NULL; + ulint heap_no = page_rec_get_heap_no(rec); lock_mutex_enter(); + mutex_enter(&trx_sys->mutex); - if (trx_t* impl_trx = trx_rw_is_active(trx->id, NULL, false)) { - ulint heap_no = page_rec_get_heap_no(rec); - mutex_enter(&trx_sys->mutex); - - for (trx_t* t = UT_LIST_GET_FIRST(trx_sys->rw_trx_list); - t != NULL; - t = UT_LIST_GET_NEXT(trx_list, t)) { + for (trx_t* t = UT_LIST_GET_FIRST(trx_sys->rw_trx_list); + t != NULL; + t = UT_LIST_GET_NEXT(trx_list, t)) { - lock_t* expl_lock = lock_rec_has_expl( - precise_mode, block, heap_no, t); + lock_t* expl_lock = lock_rec_has_expl( + precise_mode, block, heap_no, t); - if (expl_lock && expl_lock->trx != impl_trx) { - /* An explicit lock is held by trx other than - the trx holding the implicit lock. */ - holds = expl_lock->trx; - break; - } + if (expl_lock && expl_lock->trx != trx) { + /* An explicit lock is held by trx other than + the trx holding the implicit lock. */ + holds = expl_lock->trx; + break; } - - mutex_exit(&trx_sys->mutex); } + mutex_exit(&trx_sys->mutex); + lock_mutex_exit(); return(holds); @@ -6223,7 +6222,6 @@ lock_rec_queue_validate( const dict_index_t* index, /*!< in: index, or NULL if not known */ const ulint* offsets)/*!< in: rec_get_offsets(rec, index) */ { - const trx_t* impl_trx; const lock_t* lock; ulint heap_no; @@ -6269,13 +6267,11 @@ lock_rec_queue_validate( /* Nothing we can do */ } else if (dict_index_is_clust(index)) { - trx_id_t trx_id; - /* Unlike the non-debug code, this invariant can only succeed if the check and assertion are covered by the lock mutex. */ - trx_id = lock_clust_rec_some_has_impl(rec, index, offsets); - impl_trx = trx_rw_is_active_low(trx_id, NULL); + const trx_t *impl_trx = trx_sys->rw_trx_hash.find( + lock_clust_rec_some_has_impl(rec, index, offsets)); ut_ad(lock_mutex_own()); /* impl_trx cannot be committed until lock_mutex_exit() @@ -6813,7 +6809,7 @@ lock_rec_convert_impl_to_expl_for_trx( trx_t* trx, /*!< in/out: active transaction */ ulint heap_no)/*!< in: rec heap number to lock */ { - ut_ad(trx_is_referenced(trx)); + ut_ad(trx->is_referenced()); ut_ad(page_rec_is_leaf(rec)); ut_ad(!rec_is_default_row(rec, index)); @@ -6837,7 +6833,7 @@ lock_rec_convert_impl_to_expl_for_trx( lock_mutex_exit(); - trx_release_reference(trx); + trx->release_reference(); DEBUG_SYNC_C("after_lock_rec_convert_impl_to_expl_for_trx"); } @@ -6849,6 +6845,7 @@ static void lock_rec_convert_impl_to_expl( /*==========================*/ + trx_t* caller_trx,/*!<in/out: trx of current thread */ const buf_block_t* block, /*!< in: buffer block of rec */ const rec_t* rec, /*!< in: user record on page */ dict_index_t* index, /*!< in: index of record */ @@ -6868,11 +6865,12 @@ lock_rec_convert_impl_to_expl( trx_id = lock_clust_rec_some_has_impl(rec, index, offsets); - trx = trx_rw_is_active(trx_id, NULL, true); + trx = trx_sys->rw_trx_hash.find(caller_trx, trx_id, true); } else { ut_ad(!dict_index_is_online_ddl(index)); - trx = lock_sec_rec_some_has_impl(rec, index, offsets); + trx = lock_sec_rec_some_has_impl(caller_trx, rec, index, + offsets); ut_ad(!trx || !lock_rec_other_trx_holds_expl( LOCK_S | LOCK_REC_NOT_GAP, trx, rec, block)); @@ -6881,7 +6879,7 @@ lock_rec_convert_impl_to_expl( if (trx != 0) { ulint heap_no = page_rec_get_heap_no(rec); - ut_ad(trx_is_referenced(trx)); + ut_ad(trx->is_referenced()); /* If the transaction is still active and has no explicit x-lock set on the record, set one for it. @@ -6934,7 +6932,8 @@ lock_clust_rec_modify_check_and_lock( /* If a transaction has no explicit x-lock set on the record, set one for it */ - lock_rec_convert_impl_to_expl(block, rec, index, offsets); + lock_rec_convert_impl_to_expl(thr_get_trx(thr), block, rec, index, + offsets); lock_mutex_enter(); @@ -7094,11 +7093,11 @@ lock_sec_rec_read_check_and_lock( if the max trx id for the page >= min trx id for the trx list or a database recovery is running. */ - if ((page_get_max_trx_id(block->frame) >= trx_rw_min_trx_id() - || recv_recovery_is_on()) - && !page_rec_is_supremum(rec)) { + if (!page_rec_is_supremum(rec) + && page_get_max_trx_id(block->frame) >= trx_rw_min_trx_id()) { - lock_rec_convert_impl_to_expl(block, rec, index, offsets); + lock_rec_convert_impl_to_expl(thr_get_trx(thr), block, rec, + index, offsets); } lock_mutex_enter(); @@ -7173,7 +7172,8 @@ lock_clust_rec_read_check_and_lock( if (heap_no != PAGE_HEAP_NO_SUPREMUM) { - lock_rec_convert_impl_to_expl(block, rec, index, offsets); + lock_rec_convert_impl_to_expl(thr_get_trx(thr), block, rec, + index, offsets); } lock_mutex_enter(); @@ -7650,13 +7650,13 @@ lock_trx_release_locks( trx->state = TRX_STATE_COMMITTED_IN_MEMORY; /*--------------------------------------*/ - if (trx_is_referenced(trx)) { + if (trx->is_referenced()) { ut_a(release_lock); lock_mutex_exit(); - while (trx_is_referenced(trx)) { + while (trx->is_referenced()) { trx_mutex_exit(trx); @@ -7676,7 +7676,7 @@ lock_trx_release_locks( trx_mutex_enter(trx); } - ut_ad(!trx_is_referenced(trx)); + ut_ad(!trx->is_referenced()); /* If the background thread trx_rollback_or_clean_recovered() is still active then there is a chance that the rollback diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 92373afbeff..cacf823f596 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -842,7 +842,7 @@ recv_find_max_checkpoint_0(log_group_t** max_group, ulint* max_field) " This redo log was created before MariaDB 10.2.2," " and we did not find a valid checkpoint." " Please follow the instructions at" - " " REFMAN "upgrading.html"; + " https://mariadb.com/kb/en/library/upgrading/"; return(DB_ERROR); } @@ -1159,6 +1159,8 @@ parse_log: redo log been written with something older than InnoDB Plugin 1.0.4. */ ut_ad(0 + /* fil_crypt_rotate_page() writes this */ + || offs == FIL_PAGE_SPACE_ID || offs == IBUF_TREE_SEG_HEADER + IBUF_HEADER + FSEG_HDR_SPACE || offs == IBUF_TREE_SEG_HEADER diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc index 5ede7faf3ba..d9a04ba9393 100644 --- a/storage/innobase/page/page0cur.cc +++ b/storage/innobase/page/page0cur.cc @@ -2,6 +2,7 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. +Copyright (c) 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1279,7 +1280,7 @@ page_cur_insert_rec_low( == (ibool) !!page_is_comp(page)); ut_ad(fil_page_index_page_check(page)); ut_ad(mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID) == index->id - || recv_recovery_is_on() + || index->is_dummy || (mtr ? mtr->is_inside_ibuf() : dict_index_is_ibuf(index))); ut_ad(!page_rec_is_supremum(current_rec)); @@ -1506,8 +1507,8 @@ page_cur_insert_rec_zip( ut_ad(page_is_comp(page)); ut_ad(fil_page_index_page_check(page)); ut_ad(mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID) == index->id - || (mtr ? mtr->is_inside_ibuf() : dict_index_is_ibuf(index)) - || recv_recovery_is_on()); + || index->is_dummy + || (mtr ? mtr->is_inside_ibuf() : dict_index_is_ibuf(index))); ut_ad(!page_get_instant(page)); ut_ad(!page_cur_is_after_last(cursor)); #ifdef UNIV_ZIP_DEBUG @@ -1626,6 +1627,7 @@ page_cur_insert_rec_zip( because the MLOG_COMP_REC_INSERT should only be logged after a successful operation. */ ut_ad(!recv_recovery_is_on()); + ut_ad(!index->is_dummy); } else if (recv_recovery_is_on()) { /* This should be followed by MLOG_ZIP_PAGE_COMPRESS_NO_DATA, @@ -1967,6 +1969,8 @@ page_parse_copy_rec_list_to_created_page( page_t* page; page_zip_des_t* page_zip; + ut_ad(index->is_dummy); + if (ptr + 4 > end_ptr) { return(NULL); @@ -1992,8 +1996,7 @@ page_parse_copy_rec_list_to_created_page( page_copy_rec_list_end_to_created_page() which was logged by. page_copy_rec_list_to_created_page_write_log(). For other pages, this field must be zero-initialized. */ - ut_ad(!page_get_instant(block->frame) - || (page_is_root(block->frame) && index->is_dummy)); + ut_ad(!page_get_instant(block->frame) || page_is_root(block->frame)); while (ptr < rec_end) { ptr = page_cur_parse_insert_rec(TRUE, ptr, end_ptr, @@ -2320,8 +2323,8 @@ page_cur_delete_rec( ut_ad(!!page_is_comp(page) == dict_table_is_comp(index->table)); ut_ad(fil_page_index_page_check(page)); ut_ad(mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID) == index->id - || (mtr ? mtr->is_inside_ibuf() : dict_index_is_ibuf(index)) - || recv_recovery_is_on()); + || index->is_dummy + || (mtr ? mtr->is_inside_ibuf() : dict_index_is_ibuf(index))); ut_ad(mtr == NULL || mtr->is_named_space(index->space)); /* The record must not be the supremum or infimum record. */ diff --git a/storage/innobase/que/que0que.cc b/storage/innobase/que/que0que.cc index 5a3af9dfaeb..a45ac188f2f 100644 --- a/storage/innobase/que/que0que.cc +++ b/storage/innobase/que/que0que.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1030,6 +1030,7 @@ que_thr_step( } else if (type == QUE_NODE_SELECT) { thr = row_sel_step(thr); } else if (type == QUE_NODE_INSERT) { + trx_start_if_not_started_xa(thr_get_trx(thr), true); thr = row_ins_step(thr); } else if (type == QUE_NODE_UPDATE) { trx_start_if_not_started_xa(thr_get_trx(thr), true); diff --git a/storage/innobase/read/read0read.cc b/storage/innobase/read/read0read.cc index 2fb7083b0b2..8f4dd4f37b2 100644 --- a/storage/innobase/read/read0read.cc +++ b/storage/innobase/read/read0read.cc @@ -371,6 +371,7 @@ Copy the transaction ids from the source vector */ void ReadView::copy_trx_ids(const trx_ids_t& trx_ids) { + ut_ad(mutex_own(&trx_sys->mutex)); ulint size = trx_ids.size(); if (m_creator_trx_id > 0) { @@ -424,14 +425,24 @@ ReadView::copy_trx_ids(const trx_ids_t& trx_ids) } #ifdef UNIV_DEBUG - /* Assert that all transaction ids in list are active. */ + /* Original assertion was here to make sure that rw_trx_ids and + rw_trx_hash are in sync and they hold either ACTIVE or PREPARED + transaction. + + Now rw_trx_hash.find() does + ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || + trx_state_eq(trx, TRX_STATE_PREPARED)). + No need to repeat it here. We even can't repeat it here: it'll be race + condition because we need trx->element->mutex locked to perform this + check (see how it is done in find()). + + Now rw_trx_ids and rw_trx_hash may get out of sync for a short while: + when transaction is registered it first gets added into rw_trx_ids + under trx_sys->mutex protection and then to rw_trx_hash without mutex + protection. Thus we need repeat this lookup. */ for (trx_ids_t::const_iterator it = trx_ids.begin(); it != trx_ids.end(); ++it) { - - trx_t* trx = trx_get_rw_trx_by_id(*it); - ut_ad(trx != NULL); - ut_ad(trx->state == TRX_STATE_ACTIVE - || trx->state == TRX_STATE_PREPARED); + while (!trx_sys->rw_trx_hash.find(*it)); } #endif /* UNIV_DEBUG */ } diff --git a/storage/innobase/rem/rem0rec.cc b/storage/innobase/rem/rem0rec.cc index 38359e17b8f..e3bea55d8ff 100644 --- a/storage/innobase/rem/rem0rec.cc +++ b/storage/innobase/rem/rem0rec.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -2484,8 +2484,6 @@ rec_get_trx_id( const rec_t* rec, const dict_index_t* index) { - const page_t* page - = page_align(rec); ulint trx_id_col = dict_index_get_sys_col_pos(index, DATA_TRX_ID); const byte* trx_id; @@ -2496,11 +2494,7 @@ rec_get_trx_id( ulint* offsets = offsets_; ut_ad(trx_id_col <= MAX_REF_PARTS); - ut_ad(fil_page_index_page_check(page)); - ut_ad(mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID) - == index->id); ut_ad(dict_index_is_clust(index)); - ut_ad(page_rec_is_leaf(rec)); ut_ad(trx_id_col > 0); ut_ad(trx_id_col != ULINT_UNDEFINED); diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index e71629b568b..4586fe93cbd 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, 2017, MariaDB Corporation. +Copyright (c) 2016, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -3253,9 +3253,9 @@ row_ins_clust_index_entry( n_uniq = dict_index_is_unique(index) ? index->n_uniq : 0; - const ulint flags = index->table->is_temporary() - ? BTR_NO_LOCKING_FLAG - : index->table->no_rollback() ? BTR_NO_ROLLBACK : 0; + const ulint flags = index->table->no_rollback() ? BTR_NO_ROLLBACK + : dict_table_is_temporary(index->table) + ? BTR_NO_LOCKING_FLAG : 0; const ulint orig_n_fields = entry->n_fields; /* Try first optimistic descent to the B-tree */ @@ -3371,7 +3371,7 @@ row_ins_index_entry( dtuple_t* entry, /*!< in/out: index entry to insert */ que_thr_t* thr) /*!< in: query thread */ { - ut_ad(thr_get_trx(thr)->id != 0); + ut_ad(thr_get_trx(thr)->id || index->table->no_rollback()); DBUG_EXECUTE_IF("row_ins_index_entry_timeout", { DBUG_SET("-d,row_ins_index_entry_timeout"); @@ -3831,8 +3831,6 @@ row_ins_step( trx = thr_get_trx(thr); - trx_start_if_not_started_xa(trx, true); - node = static_cast<ins_node_t*>(thr->run_node); ut_ad(que_node_get_type(node) == QUE_NODE_INSERT); diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 0e7712f453c..15659431a7e 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1449,17 +1449,6 @@ row_insert_for_mysql( } else if (high_level_read_only) { return(DB_READ_ONLY); } - DBUG_EXECUTE_IF("mark_table_corrupted", { - /* Mark the table corrupted for the clustered index */ - dict_index_t* index = dict_table_get_first_index(table); - ut_ad(dict_index_is_clust(index)); - dict_set_corrupted(index, trx, "INSERT TABLE"); }); - - if (dict_table_is_corrupted(table)) { - - ib::error() << "Table " << table->name << " is corrupt."; - return(DB_TABLE_CORRUPT); - } DBUG_EXECUTE_IF("mark_table_corrupted", { /* Mark the table corrupted for the clustered index */ @@ -1477,7 +1466,9 @@ row_insert_for_mysql( row_mysql_delay_if_needed(); - trx_start_if_not_started_xa(trx, true); + if (!table->no_rollback()) { + trx_start_if_not_started_xa(trx, true); + } row_get_prebuilt_insert_row(prebuilt); node = prebuilt->ins_node; diff --git a/storage/innobase/row/row0row.cc b/storage/innobase/row/row0row.cc index 3611b8c3fc7..3f6c8b9f918 100644 --- a/storage/innobase/row/row0row.cc +++ b/storage/innobase/row/row0row.cc @@ -415,8 +415,8 @@ row_build_low( times, and the cursor restore can happen multiple times for single insert or update statement. */ ut_a(!rec_offs_any_null_extern(rec, offsets) - || trx_rw_is_active(row_get_rec_trx_id(rec, index, offsets), - NULL, false)); + || trx_sys->rw_trx_hash.find(row_get_rec_trx_id(rec, index, + offsets))); #endif /* UNIV_DEBUG || UNIV_BLOB_LIGHT_DEBUG */ if (type != ROW_COPY_POINTERS) { diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 78e5bbcd15b..9c0b953fd6c 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -2,7 +2,7 @@ Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -799,7 +799,7 @@ row_sel_build_committed_vers_for_mysql( rec_offs_size(*offsets)); } - row_vers_build_for_semi_consistent_read( + row_vers_build_for_semi_consistent_read(prebuilt->trx, rec, mtr, clust_index, offsets, offset_heap, prebuilt->old_vers_heap, old_vers, vrow); } @@ -1272,24 +1272,19 @@ static void row_sel_open_pcur( /*==============*/ - plan_t* plan, /*!< in: table plan */ - ibool search_latch_locked, - /*!< in: TRUE if the thread currently - has the search latch locked in - s-mode */ - mtr_t* mtr) /*!< in: mtr */ + plan_t* plan, /*!< in: table plan */ +#ifdef BTR_CUR_HASH_ADAPT + rw_lock_t* ahi_latch, + /*!< in: the adaptive hash index latch */ +#endif /* BTR_CUR_HASH_ADAPT */ + mtr_t* mtr) /*!< in/out: mini-transaction */ { dict_index_t* index; func_node_t* cond; que_node_t* exp; ulint n_fields; - ulint has_search_latch = 0; /* RW_S_LATCH or 0 */ ulint i; - if (search_latch_locked) { - has_search_latch = RW_S_LATCH; - } - index = plan->index; /* Calculate the value of the search tuple: the exact match columns @@ -1325,7 +1320,7 @@ row_sel_open_pcur( btr_pcur_open_with_no_init(index, plan->tuple, plan->mode, BTR_SEARCH_LEAF, &plan->pcur, - has_search_latch, mtr); + ahi_latch, mtr); } else { /* Open the cursor to the start or the end of the index (FALSE: no init) */ @@ -1462,45 +1457,27 @@ row_sel_try_search_shortcut( sel_node_t* node, /*!< in: select node for a consistent read */ plan_t* plan, /*!< in: plan for a unique search in clustered index */ - ibool search_latch_locked, - /*!< in: whether the search holds latch on - search system. */ mtr_t* mtr) /*!< in: mtr */ { - dict_index_t* index; - rec_t* rec; - mem_heap_t* heap = NULL; - ulint offsets_[REC_OFFS_NORMAL_SIZE]; - ulint* offsets = offsets_; - ulint ret; - rec_offs_init(offsets_); - - index = plan->index; + dict_index_t* index = plan->index; ut_ad(node->read_view); ut_ad(plan->unique_search); ut_ad(!plan->must_get_clust); - ut_ad(!search_latch_locked - || rw_lock_own(btr_get_search_latch(index), RW_LOCK_S)); - row_sel_open_pcur(plan, search_latch_locked, mtr); + rw_lock_t* ahi_latch = btr_get_search_latch(index); + rw_lock_s_lock(ahi_latch); - rec = btr_pcur_get_rec(&(plan->pcur)); + row_sel_open_pcur(plan, ahi_latch, mtr); - if (!page_rec_is_user_rec(rec)) { + const rec_t* rec = btr_pcur_get_rec(&(plan->pcur)); + if (!page_rec_is_user_rec(rec) || rec_is_default_row(rec, index)) { +retry: + rw_lock_s_unlock(ahi_latch); return(SEL_RETRY); } - if (rec_is_default_row(rec, index)) { - /* Skip the 'default row' pseudo-record. */ - if (!btr_pcur_move_to_next_user_rec(&plan->pcur, mtr)) { - return(SEL_RETRY); - } - - rec = btr_pcur_get_rec(&plan->pcur); - } - ut_ad(plan->mode == PAGE_CUR_GE); /* As the cursor is now placed on a user record after a search with @@ -1508,36 +1485,34 @@ row_sel_try_search_shortcut( fields in the user record matched to the search tuple */ if (btr_pcur_get_up_match(&(plan->pcur)) < plan->n_exact_match) { - +exhausted: + rw_lock_s_unlock(ahi_latch); return(SEL_EXHAUSTED); } /* This is a non-locking consistent read: if necessary, fetch a previous version of the record */ + mem_heap_t* heap = NULL; + ulint offsets_[REC_OFFS_NORMAL_SIZE]; + ulint* offsets = offsets_; + rec_offs_init(offsets_); offsets = rec_get_offsets(rec, index, offsets, true, ULINT_UNDEFINED, &heap); if (dict_index_is_clust(index)) { if (!lock_clust_rec_cons_read_sees(rec, index, offsets, node->read_view)) { - ret = SEL_RETRY; - goto func_exit; + goto retry; } } else if (!srv_read_only_mode && !lock_sec_rec_cons_read_sees( rec, index, node->read_view)) { - - ret = SEL_RETRY; - goto func_exit; + goto retry; } - /* Test the deleted flag. */ - if (rec_get_deleted_flag(rec, dict_table_is_comp(plan->table))) { - - ret = SEL_EXHAUSTED; - goto func_exit; + goto exhausted; } /* Fetch the columns needed in test conditions. The index @@ -1551,20 +1526,18 @@ row_sel_try_search_shortcut( /* Test the rest of search conditions */ if (!row_sel_test_other_conds(plan)) { - - ret = SEL_EXHAUSTED; - goto func_exit; + goto exhausted; } ut_ad(plan->pcur.latch_mode == BTR_SEARCH_LEAF); plan->n_rows_fetched++; - ret = SEL_FOUND; -func_exit: + rw_lock_s_unlock(ahi_latch); + if (UNIV_LIKELY_NULL(heap)) { mem_heap_free(heap); } - return(ret); + return(SEL_FOUND); } #endif /* BTR_CUR_HASH_ADAPT */ @@ -1611,12 +1584,6 @@ row_sel( ut_ad(thr->run_node == node); -#ifdef BTR_CUR_HASH_ADAPT - ibool search_latch_locked = FALSE; -#else /* BTR_CUR_HASH_ADAPT */ -# define search_latch_locked false -#endif /* BTR_CUR_HASH_ADAPT */ - if (node->read_view) { /* In consistent reads, we try to do with the hash index and not to use the buffer page get. This is to reduce memory bus @@ -1665,33 +1632,14 @@ table_loop: #ifdef BTR_CUR_HASH_ADAPT if (consistent_read && plan->unique_search && !plan->pcur_is_open && !plan->must_get_clust) { - if (!search_latch_locked) { - btr_search_s_lock(index); - - search_latch_locked = TRUE; - } else if (rw_lock_get_writer(btr_get_search_latch(index)) - == RW_LOCK_X_WAIT) { - - /* There is an x-latch request waiting: release the - s-latch for a moment; as an s-latch here is often - kept for some 10 searches before being released, - a waiting x-latch request would block other threads - from acquiring an s-latch for a long time, lowering - performance significantly in multiprocessors. */ - - btr_search_s_unlock(index); - btr_search_s_lock(index); - } - - switch (row_sel_try_search_shortcut(node, plan, - search_latch_locked, - &mtr)) { + switch (row_sel_try_search_shortcut(node, plan, &mtr)) { case SEL_FOUND: goto next_table; case SEL_EXHAUSTED: goto table_exhausted; default: ut_ad(0); + /* fall through */ case SEL_RETRY: break; } @@ -1701,19 +1649,16 @@ table_loop: mtr_commit(&mtr); mtr_start(&mtr); } - - if (search_latch_locked) { - btr_search_s_unlock(index); - - search_latch_locked = FALSE; - } #endif /* BTR_CUR_HASH_ADAPT */ if (!plan->pcur_is_open) { /* Evaluate the expressions to build the search tuple and open the cursor */ - - row_sel_open_pcur(plan, search_latch_locked, &mtr); + row_sel_open_pcur(plan, +#ifdef BTR_CUR_HASH_ADAPT + NULL, +#endif /* BTR_CUR_HASH_ADAPT */ + &mtr); cursor_just_opened = TRUE; @@ -2116,8 +2061,6 @@ skip_lock: } next_rec: - ut_ad(!search_latch_locked); - if (mtr_has_extra_clust_latch) { /* We must commit &mtr if we are moving to the next @@ -2155,8 +2098,6 @@ next_table: plan->cursor_at_end = TRUE; } else { - ut_ad(!search_latch_locked); - plan->stored_cursor_rec_processed = TRUE; btr_pcur_store_position(&(plan->pcur), &mtr); @@ -2247,8 +2188,6 @@ stop_for_a_while: inserted new records which should have appeared in the result set, which would result in the phantom problem. */ - ut_ad(!search_latch_locked); - plan->stored_cursor_rec_processed = FALSE; btr_pcur_store_position(&(plan->pcur), &mtr); @@ -2265,7 +2204,6 @@ commit_mtr_for_a_while: plan->stored_cursor_rec_processed = TRUE; - ut_ad(!search_latch_locked); btr_pcur_store_position(&(plan->pcur), &mtr); mtr_commit(&mtr); @@ -2279,7 +2217,6 @@ lock_wait_or_error: /* See the note at stop_for_a_while: the same holds for this case */ ut_ad(!btr_pcur_is_before_first_on_page(&plan->pcur) || !node->asc); - ut_ad(!search_latch_locked); plan->stored_cursor_rec_processed = FALSE; btr_pcur_store_position(&(plan->pcur), &mtr); @@ -2287,11 +2224,6 @@ lock_wait_or_error: mtr_commit(&mtr); func_exit: -#ifdef BTR_CUR_HASH_ADAPT - if (search_latch_locked) { - btr_search_s_unlock(index); - } -#endif /* BTR_CUR_HASH_ADAPT */ ut_ad(!sync_check_iterate(dict_sync_check())); if (heap != NULL) { @@ -3904,30 +3836,25 @@ row_sel_try_search_shortcut_for_mysql( ut_ad(dict_index_is_clust(index)); ut_ad(!prebuilt->templ_contains_blob); + rw_lock_t* ahi_latch = btr_get_search_latch(index); + rw_lock_s_lock(ahi_latch); btr_pcur_open_with_no_init(index, search_tuple, PAGE_CUR_GE, - BTR_SEARCH_LEAF, pcur, RW_S_LATCH, mtr); + BTR_SEARCH_LEAF, pcur, ahi_latch, mtr); rec = btr_pcur_get_rec(pcur); - if (!page_rec_is_user_rec(rec)) { - + if (!page_rec_is_user_rec(rec) || rec_is_default_row(rec, index)) { +retry: + rw_lock_s_unlock(ahi_latch); return(SEL_RETRY); } - if (rec_is_default_row(rec, index)) { - /* Skip the 'default row' pseudo-record. */ - if (!btr_pcur_move_to_next_user_rec(pcur, mtr)) { - return(SEL_RETRY); - } - - rec = btr_pcur_get_rec(pcur); - } - /* As the cursor is now placed on a user record after a search with the mode PAGE_CUR_GE, the up_match field in the cursor tells how many fields in the user record matched to the search tuple */ if (btr_pcur_get_up_match(pcur) < dtuple_get_n_fields(search_tuple)) { - +exhausted: + rw_lock_s_unlock(ahi_latch); return(SEL_EXHAUSTED); } @@ -3939,20 +3866,19 @@ row_sel_try_search_shortcut_for_mysql( if (!lock_clust_rec_cons_read_sees( rec, index, *offsets, trx_get_read_view(trx))) { - - return(SEL_RETRY); + goto retry; } if (rec_get_deleted_flag(rec, dict_table_is_comp(index->table))) { /* In delete-marked records, DB_TRX_ID must always refer to an existing undo log record. */ ut_ad(row_get_rec_trx_id(rec, index, *offsets)); - - return(SEL_EXHAUSTED); + goto exhausted; } *out_rec = rec; + rw_lock_s_unlock(ahi_latch); return(SEL_FOUND); } #endif /* BTR_CUR_HASH_ADAPT */ @@ -4341,24 +4267,14 @@ row_search_mvcc( mode = PAGE_CUR_GE; - if (trx->mysql_n_tables_locked == 0 - && prebuilt->select_lock_type == LOCK_NONE + if (prebuilt->select_lock_type == LOCK_NONE && trx->isolation_level > TRX_ISO_READ_UNCOMMITTED && MVCC::is_view_active(trx->read_view)) { /* This is a SELECT query done as a consistent read, and the read view has already been allocated: let us try a search shortcut through the hash - index. - NOTE that we must also test that - mysql_n_tables_locked == 0, because this might - also be INSERT INTO ... SELECT ... or - CREATE TABLE ... SELECT ... . Our algorithm is - NOT prepared to inserts interleaved with the SELECT, - and if we try that, we can deadlock on the adaptive - hash index semaphore! */ - - rw_lock_s_lock(btr_get_search_latch(index)); + index. */ switch (row_sel_try_search_shortcut_for_mysql( &rec, prebuilt, &offsets, &heap, @@ -4406,27 +4322,17 @@ row_search_mvcc( shortcut_match: mtr_commit(&mtr); - /* NOTE that we do NOT store the cursor position */ - err = DB_SUCCESS; - - rw_lock_s_unlock(btr_get_search_latch(index)); - goto func_exit; case SEL_EXHAUSTED: shortcut_mismatch: mtr_commit(&mtr); - - err = DB_RECORD_NOT_FOUND; - - rw_lock_s_unlock(btr_get_search_latch(index)); - /* NOTE that we do NOT store the cursor position */ - + err = DB_RECORD_NOT_FOUND; goto func_exit; case SEL_RETRY: @@ -4438,8 +4344,6 @@ row_search_mvcc( mtr_commit(&mtr); mtr_start(&mtr); - - rw_lock_s_unlock(btr_get_search_latch(index)); } } #endif /* BTR_CUR_HASH_ADAPT */ @@ -4972,17 +4876,17 @@ wrong_offs: /* In delete-marked records, DB_TRX_ID must always refer to an existing undo log record. */ ut_ad(trx_id); - if (!trx_rw_is_active(trx_id, NULL, false)) { + if (!trx_sys->rw_trx_hash.find(trx, trx_id)) { /* The clustered index record was delete-marked in a committed transaction. Ignore the record. */ goto locks_ok_del_marked; } - } else if (trx_t* trx = row_vers_impl_x_locked( - rec, index, offsets)) { + } else if (trx_t* t = row_vers_impl_x_locked( + trx, rec, index, offsets)) { /* The record belongs to an active transaction. We must acquire a lock. */ - trx_release_reference(trx); + t->release_reference(); } else { /* The secondary index record does not point to a delete-marked clustered index diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index 4c2fd94c1cd..e4d96dc62cd 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -715,7 +715,7 @@ row_upd_rec_in_place( ut_ad(index->is_instant()); break; case REC_STATUS_NODE_PTR: - if (recv_recovery_is_on() + if (index->is_dummy && fil_page_get_type(page_align(rec)) == FIL_PAGE_RTREE) { /* The function rtr_update_mbr_field_in_place() diff --git a/storage/innobase/row/row0vers.cc b/storage/innobase/row/row0vers.cc index 712c46cba35..fc8b1b2ec81 100644 --- a/storage/innobase/row/row0vers.cc +++ b/storage/innobase/row/row0vers.cc @@ -47,6 +47,7 @@ Created 2/6/1997 Heikki Tuuri /** Check whether all non-virtual columns in a virtual index match that of in the cluster index +@param[in,out] caller_trx trx of current thread @param[in] index the secondary index @param[in] row the cluster index row in dtuple form @param[in] ext externally stored column prefix or NULL @@ -65,17 +66,19 @@ row_vers_non_vc_match( ulint* n_non_v_col); /** Determine if an active transaction has inserted or modified a secondary index record. +@param[in,out] caller_trx trx of current thread @param[in] clust_rec clustered index record @param[in] clust_index clustered index @param[in] rec secondary index record @param[in] index secondary index @param[in] offsets rec_get_offsets(rec, index) @param[in,out] mtr mini-transaction -@return the active transaction; trx_release_reference() must be invoked +@return the active transaction; trx->release_reference() must be invoked @retval NULL if the record was committed */ UNIV_INLINE trx_t* row_vers_impl_x_locked_low( + trx_t* caller_trx, const rec_t* clust_rec, dict_index_t* clust_index, const rec_t* rec, @@ -84,7 +87,6 @@ row_vers_impl_x_locked_low( mtr_t* mtr) { trx_id_t trx_id; - ibool corrupt; ulint comp; ulint rec_del; const rec_t* version; @@ -118,13 +120,15 @@ row_vers_impl_x_locked_low( mem_heap_free(heap); DBUG_RETURN(0); } - corrupt = FALSE; - trx_t* trx = trx_rw_is_active(trx_id, &corrupt, true); + trx_t* trx = trx_sys->rw_trx_hash.find(caller_trx, trx_id, true); if (trx == 0) { /* The transaction that modified or inserted clust_rec is no longer active, or it is corrupt: no implicit lock on rec */ + trx_sys_mutex_enter(); + bool corrupt = trx_id >= trx_sys->max_trx_id; + trx_sys_mutex_exit(); if (corrupt) { lock_report_trx_id_insanity( trx_id, clust_rec, clust_index, clust_offsets, @@ -189,7 +193,7 @@ row_vers_impl_x_locked_low( inserting a delete-marked record. */ ut_ad(prev_version || !rec_get_deleted_flag(version, comp) - || !trx_rw_is_active(trx_id, NULL, false)); + || !trx_sys->rw_trx_hash.find(caller_trx, trx_id)); /* Free version and clust_offsets. */ mem_heap_free(old_heap); @@ -213,7 +217,7 @@ row_vers_impl_x_locked_low( or updated, the leaf page record always is created with a clear delete-mark flag. (We never insert a delete-marked record.) */ - trx_release_reference(trx); + trx->release_reference(); trx = 0; } @@ -324,7 +328,7 @@ result_check: /* prev_version was the first version modified by the trx_id transaction: no implicit x-lock */ - trx_release_reference(trx); + trx->release_reference(); trx = 0; break; } @@ -342,13 +346,15 @@ result_check: /** Determine if an active transaction has inserted or modified a secondary index record. +@param[in,out] caller_trx trx of current thread @param[in] rec secondary index record @param[in] index secondary index @param[in] offsets rec_get_offsets(rec, index) -@return the active transaction; trx_release_reference() must be invoked +@return the active transaction; trx->release_reference() must be invoked @retval NULL if the record was committed */ trx_t* row_vers_impl_x_locked( + trx_t* caller_trx, const rec_t* rec, dict_index_t* index, const ulint* offsets) @@ -389,9 +395,10 @@ row_vers_impl_x_locked( trx = 0; } else { trx = row_vers_impl_x_locked_low( - clust_rec, clust_index, rec, index, offsets, &mtr); + caller_trx, clust_rec, clust_index, rec, index, + offsets, &mtr); - ut_ad(trx == 0 || trx_is_referenced(trx)); + ut_ad(trx == 0 || trx->is_referenced()); } mtr_commit(&mtr); @@ -1234,6 +1241,7 @@ which should be seen by a semi-consistent read. */ void row_vers_build_for_semi_consistent_read( /*====================================*/ + trx_t* caller_trx,/*!<in/out: trx of current thread */ const rec_t* rec, /*!< in: record in a clustered index; the caller must have a latch on the page; this latch locks the top of the stack of versions @@ -1270,7 +1278,6 @@ row_vers_build_for_semi_consistent_read( ut_ad(!vrow || !(*vrow)); for (;;) { - const trx_t* version_trx; mem_heap_t* heap2; rec_t* prev_version; trx_id_t version_trx_id; @@ -1280,24 +1287,7 @@ row_vers_build_for_semi_consistent_read( rec_trx_id = version_trx_id; } - if (!version_trx_id) { - goto committed_version_trx; - } - - trx_sys_mutex_enter(); - version_trx = trx_get_rw_trx_by_id(version_trx_id); - /* Because version_trx is a read-write transaction, - its state cannot change from or to NOT_STARTED while - we are holding the trx_sys->mutex. It may change from - ACTIVE to PREPARED or COMMITTED. */ - if (version_trx - && trx_state_eq(version_trx, - TRX_STATE_COMMITTED_IN_MEMORY)) { - version_trx = NULL; - } - trx_sys_mutex_exit(); - - if (!version_trx) { + if (!trx_sys->rw_trx_hash.find(caller_trx, version_trx_id)) { committed_version_trx: /* We found a version that belongs to a committed transaction: return it. */ diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 4f508c149e9..9297dbefbae 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -3,7 +3,7 @@ Copyright (c) 1996, 2017, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described diff --git a/storage/innobase/sync/sync0debug.cc b/storage/innobase/sync/sync0debug.cc index 76796f96913..e61c01c0443 100644 --- a/storage/innobase/sync/sync0debug.cc +++ b/storage/innobase/sync/sync0debug.cc @@ -480,6 +480,7 @@ LatchDebug::LatchDebug() LEVEL_MAP_INSERT(SYNC_REC_LOCK); LEVEL_MAP_INSERT(SYNC_THREADS); LEVEL_MAP_INSERT(SYNC_TRX); + LEVEL_MAP_INSERT(SYNC_RW_TRX_HASH_ELEMENT); LEVEL_MAP_INSERT(SYNC_TRX_SYS); LEVEL_MAP_INSERT(SYNC_LOCK_SYS); LEVEL_MAP_INSERT(SYNC_LOCK_WAIT_SYS); @@ -761,6 +762,7 @@ LatchDebug::check_order( case SYNC_THREADS: case SYNC_LOCK_SYS: case SYNC_LOCK_WAIT_SYS: + case SYNC_RW_TRX_HASH_ELEMENT: case SYNC_TRX_SYS: case SYNC_IBUF_BITMAP_MUTEX: case SYNC_REDO_RSEG: @@ -1521,6 +1523,8 @@ sync_latch_meta_init() PFS_NOT_INSTRUMENTED); LATCH_ADD_MUTEX(FIL_CRYPT_THREADS_MUTEX, SYNC_NO_ORDER_CHECK, PFS_NOT_INSTRUMENTED); + LATCH_ADD_MUTEX(RW_TRX_HASH_ELEMENT, SYNC_RW_TRX_HASH_ELEMENT, + rw_trx_hash_element_mutex_key); latch_id_t id = LATCH_ID_NONE; diff --git a/storage/innobase/sync/sync0sync.cc b/storage/innobase/sync/sync0sync.cc index 5d98e49fd6d..e78b2a033ac 100644 --- a/storage/innobase/sync/sync0sync.cc +++ b/storage/innobase/sync/sync0sync.cc @@ -96,6 +96,7 @@ mysql_pfs_key_t sync_array_mutex_key; mysql_pfs_key_t thread_mutex_key; mysql_pfs_key_t zip_pad_mutex_key; mysql_pfs_key_t row_drop_list_mutex_key; +mysql_pfs_key_t rw_trx_hash_element_mutex_key; #endif /* UNIV_PFS_MUTEX */ #ifdef UNIV_PFS_RWLOCK mysql_pfs_key_t btr_search_latch_key; diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 27a081c809f..699db7a38a8 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -1151,28 +1151,6 @@ trx_purge_rseg_get_next_history_log( mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); - - trx_sys_mutex_enter(); - - /* Add debug code to track history list corruption reported - on the MySQL mailing list on Nov 9, 2004. The fut0lst.cc - file-based list was corrupt. The prev node pointer was - FIL_NULL, even though the list length was over 8 million nodes! - We assume that purge truncates the history list in large - size pieces, and if we here reach the head of the list, the - list cannot be longer than 2000 000 undo logs now. */ - - if (trx_sys->rseg_history_len > 2000000) { - ib::warn() << "Purge reached the head of the history" - " list, but its length is still reported as " - << trx_sys->rseg_history_len << "! Make" - " a detailed bug report, and submit it to" - " https://jira.mariadb.org/"; - ut_ad(0); - } - - trx_sys_mutex_exit(); - return; } diff --git a/storage/innobase/trx/trx0sys.cc b/storage/innobase/trx/trx0sys.cc index c2e15fea953..ef69bd3102b 100644 --- a/storage/innobase/trx/trx0sys.cc +++ b/storage/innobase/trx/trx0sys.cc @@ -429,8 +429,6 @@ void trx_sys_init_at_db_start() { trx_sysf_t* sys_header; - ib_uint64_t rows_to_undo = 0; - const char* unit = ""; /* VERY important: after the database is started, max_trx_id value is divisible by TRX_SYS_TRX_ID_WRITE_MARGIN, and the 'if' in @@ -455,43 +453,6 @@ trx_sys_init_at_db_start() trx_dummy_sess = sess_open(); trx_lists_init_at_db_start(); - - /* This mutex is not strictly required, it is here only to satisfy - the debug code (assertions). We are still running in single threaded - bootstrap mode. */ - - trx_sys_mutex_enter(); - - if (UT_LIST_GET_LEN(trx_sys->rw_trx_list) > 0) { - const trx_t* trx; - - for (trx = UT_LIST_GET_FIRST(trx_sys->rw_trx_list); - trx != NULL; - trx = UT_LIST_GET_NEXT(trx_list, trx)) { - - ut_ad(trx->is_recovered); - assert_trx_in_rw_list(trx); - - if (trx_state_eq(trx, TRX_STATE_ACTIVE)) { - rows_to_undo += trx->undo_no; - } - } - - if (rows_to_undo > 1000000000) { - unit = "M"; - rows_to_undo = rows_to_undo / 1000000; - } - - ib::info() << UT_LIST_GET_LEN(trx_sys->rw_trx_list) - << " transaction(s) which must be rolled back or" - " cleaned up in total " << rows_to_undo << unit - << " row operations to undo"; - - ib::info() << "Trx id counter is " << trx_sys->max_trx_id; - } - - trx_sys_mutex_exit(); - trx_sys->mvcc->clone_oldest_view(&purge_sys->view); } @@ -515,8 +476,7 @@ trx_sys_create(void) new(&trx_sys->rw_trx_ids) trx_ids_t(ut_allocator<trx_id_t>( mem_key_trx_sys_t_rw_trx_ids)); - - new(&trx_sys->rw_trx_set) TrxIdSet(); + trx_sys->rw_trx_hash.init(); } /*****************************************************************//** @@ -669,8 +629,7 @@ trx_sys_close(void) trx_sys->rw_trx_ids.~trx_ids_t(); - trx_sys->rw_trx_set.~TrxIdSet(); - + trx_sys->rw_trx_hash.destroy(); ut_free(trx_sys); trx_sys = NULL; diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 882fd6f90a4..490f3d6f26b 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -247,6 +247,7 @@ struct TrxFactory { new(&trx->hit_list) hit_list_t(); + trx->rw_trx_hash_pins = 0; trx_init(trx); DBUG_LOG("trx", "Init: " << trx); @@ -457,6 +458,7 @@ trx_create_low() /* We just got trx from pool, it should be non locking */ ut_ad(trx->will_lock == 0); + ut_ad(!trx->rw_trx_hash_pins); /* Background trx should not be forced to rollback, we will unset the flag for user trx. */ @@ -494,6 +496,7 @@ trx_free(trx_t*& trx) { assert_trx_is_free(trx); + trx_sys->rw_trx_hash.put_pins(trx); trx->mysql_thd = 0; trx->mysql_log_file_name = 0; @@ -742,7 +745,9 @@ trx_resurrect_table_locks( trx_undo_rec_t* undo_rec; table_id_set tables; - if (trx_state_eq(trx, TRX_STATE_COMMITTED_IN_MEMORY) || undo->empty) { + ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || + trx_state_eq(trx, TRX_STATE_PREPARED)); + if (undo->empty) { return; } @@ -971,10 +976,65 @@ trx_resurrect_update( } } +/** Mapping read-write transactions from id to transaction instance, for +creating read views and during trx id lookup for MVCC and locking. */ +struct TrxTrack { + explicit TrxTrack(trx_id_t id, trx_t* trx = NULL) + : + m_id(id), + m_trx(trx) + { + // Do nothing + } + + trx_id_t m_id; + trx_t* m_trx; +}; + +/** +Comparator for TrxMap */ +struct TrxTrackCmp { + + bool operator() (const TrxTrack& lhs, const TrxTrack& rhs) const + { + return(lhs.m_id < rhs.m_id); + } +}; + +typedef std::set<TrxTrack, TrxTrackCmp, ut_allocator<TrxTrack> > + TrxIdSet; + +static inline void trx_sys_add_trx_at_init(trx_t *trx, trx_undo_t *undo, + uint64_t *rows_to_undo, + TrxIdSet *set) +{ + ut_ad(trx->id != 0); + ut_ad(trx->is_recovered); + + set->insert(TrxTrack(trx->id, trx)); + if (trx_state_eq(trx, TRX_STATE_ACTIVE) || + trx_state_eq(trx, TRX_STATE_PREPARED)) + { + trx_sys->rw_trx_hash.insert(trx); + trx_sys->rw_trx_hash.put_pins(trx); + trx_sys->rw_trx_ids.push_back(trx->id); + trx_resurrect_table_locks(trx, undo); + if (trx_state_eq(trx, TRX_STATE_ACTIVE)) + *rows_to_undo+= trx->undo_no; + } +#ifdef UNIV_DEBUG + trx->in_rw_trx_list= true; + if (trx->id > trx_sys->rw_max_trx_id) + trx_sys->rw_max_trx_id= trx->id; +#endif +} + /** Initialize (resurrect) transactions at startup. */ void trx_lists_init_at_db_start() { + TrxIdSet set; + uint64_t rows_to_undo = 0; ut_a(srv_is_being_started); ut_ad(!srv_was_started); ut_ad(!purge_sys); @@ -1004,15 +1064,10 @@ trx_lists_init_at_db_start() for (undo = UT_LIST_GET_FIRST(rseg->old_insert_list); undo != NULL; undo = UT_LIST_GET_NEXT(undo_list, undo)) { - - trx_t* trx; - - trx = trx_resurrect_insert(undo, rseg); + trx_t* trx = trx_resurrect_insert(undo, rseg); trx->start_time = start_time; - - trx_sys_rw_trx_add(trx); - - trx_resurrect_table_locks(trx, undo); + trx_sys_add_trx_at_init(trx, undo, &rows_to_undo, + &set); } /* Ressurrect other transactions. */ @@ -1020,12 +1075,10 @@ trx_lists_init_at_db_start() undo != NULL; undo = UT_LIST_GET_NEXT(undo_list, undo)) { - /* Check the trx_sys->rw_trx_set first. */ - trx_sys_mutex_enter(); - - trx_t* trx = trx_get_rw_trx_by_id(undo->trx_id); - - trx_sys_mutex_exit(); + /* Check if trx_id was already registered first. */ + TrxIdSet::iterator it = + set.find(TrxTrack(undo->trx_id)); + trx_t *trx= it == set.end() ? 0 : it->m_trx; if (trx == NULL) { trx = trx_allocate_for_background(); @@ -1036,34 +1089,30 @@ trx_lists_init_at_db_start() } trx_resurrect_update(trx, undo, rseg); - - trx_sys_rw_trx_add(trx); - - trx_resurrect_table_locks(trx, undo); + trx_sys_add_trx_at_init(trx, undo, &rows_to_undo, + &set); } } - TrxIdSet::iterator end = trx_sys->rw_trx_set.end(); + if (set.size()) { - for (TrxIdSet::iterator it = trx_sys->rw_trx_set.begin(); - it != end; - ++it) { + ib::info() << set.size() + << " transaction(s) which must be rolled back or" + " cleaned up in total " << rows_to_undo + << " row operations to undo"; - ut_ad(it->m_trx->in_rw_trx_list); -#ifdef UNIV_DEBUG - if (it->m_trx->id > trx_sys->rw_max_trx_id) { - trx_sys->rw_max_trx_id = it->m_trx->id; - } -#endif /* UNIV_DEBUG */ + ib::info() << "Trx id counter is " << trx_sys->max_trx_id; + } - if (it->m_trx->state == TRX_STATE_ACTIVE - || it->m_trx->state == TRX_STATE_PREPARED) { + TrxIdSet::iterator end = set.end(); - trx_sys->rw_trx_ids.push_back(it->m_id); - } + for (TrxIdSet::iterator it = set.begin(); + it != end; + ++it) { UT_LIST_ADD_FIRST(trx_sys->rw_trx_list, it->m_trx); } + std::sort(trx_sys->rw_trx_ids.begin(), trx_sys->rw_trx_ids.end()); } /** Assign a persistent rollback segment in a round-robin fashion, @@ -1182,8 +1231,8 @@ trx_t::assign_temp_rseg() mutex_enter(&trx_sys->mutex); id = trx_sys_get_new_trx_id(); trx_sys->rw_trx_ids.push_back(id); - trx_sys->rw_trx_set.insert(TrxTrack(id, this)); mutex_exit(&trx_sys->mutex); + trx_sys->rw_trx_hash.insert(this); } ut_ad(!rseg->is_persistent()); @@ -1248,10 +1297,14 @@ trx_start_low( ut_ad(!trx->in_rw_trx_list); - /* We tend to over assert and that complicates the code somewhat. - e.g., the transaction state can be set earlier but we are forced to - set it under the protection of the trx_sys_t::mutex because some - trx list assertions are triggered unnecessarily. */ + /* No other thread can access this trx object through rw_trx_hash, thus + we don't need trx_sys->mutex protection for that purpose. Still this + trx can be found through trx_sys->mysql_trx_list, which means state + change must be protected by e.g. trx->mutex. + + For now we update it without mutex protection, because original code + did it this way. It has to be reviewed and fixed properly. */ + trx->state = TRX_STATE_ACTIVE; /* By default all transactions are in the read-only list unless they are non-locking auto-commit read only transactions or background @@ -1273,8 +1326,6 @@ trx_start_low( trx_sys->rw_trx_ids.push_back(trx->id); - trx_sys_rw_trx_add(trx); - ut_ad(trx->rsegs.m_redo.rseg != 0 || srv_read_only_mode || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); @@ -1288,11 +1339,10 @@ trx_start_low( } #endif /* UNIV_DEBUG */ - trx->state = TRX_STATE_ACTIVE; - ut_ad(trx_sys_validate_trx_list()); trx_sys_mutex_exit(); + trx_sys->rw_trx_hash.insert(trx); } else { trx->id = 0; @@ -1313,17 +1363,11 @@ trx_start_low( trx_sys->rw_trx_ids.push_back(trx->id); - trx_sys->rw_trx_set.insert( - TrxTrack(trx->id, trx)); - trx_sys_mutex_exit(); + trx_sys->rw_trx_hash.insert(trx); } - - trx->state = TRX_STATE_ACTIVE; - } else { ut_ad(!read_write); - trx->state = TRX_STATE_ACTIVE; } } @@ -1653,24 +1697,14 @@ trx_erase_lists( bool serialised) { ut_ad(trx->id > 0); - trx_sys_mutex_enter(); - - if (serialised) { - UT_LIST_REMOVE(trx_sys->serialisation_list, trx); - } - - trx_ids_t::iterator it = std::lower_bound( - trx_sys->rw_trx_ids.begin(), - trx_sys->rw_trx_ids.end(), - trx->id); - ut_ad(*it == trx->id); - trx_sys->rw_trx_ids.erase(it); if (trx->read_only || trx->rsegs.m_redo.rseg == NULL) { + trx_sys_mutex_enter(); ut_ad(!trx->in_rw_trx_list); } else { + trx_sys_mutex_enter(); UT_LIST_REMOVE(trx_sys->rw_trx_list, trx); ut_d(trx->in_rw_trx_list = false); ut_ad(trx_sys_validate_trx_list()); @@ -1680,9 +1714,18 @@ trx_erase_lists( } } - trx_sys->rw_trx_set.erase(TrxTrack(trx->id)); + if (serialised) { + UT_LIST_REMOVE(trx_sys->serialisation_list, trx); + } + trx_ids_t::iterator it = std::lower_bound( + trx_sys->rw_trx_ids.begin(), + trx_sys->rw_trx_ids.end(), + trx->id); + ut_ad(*it == trx->id); + trx_sys->rw_trx_ids.erase(it); trx_sys_mutex_exit(); + trx_sys->rw_trx_hash.erase(trx); } /****************************************************************//** @@ -3047,6 +3090,7 @@ trx_set_rw_mode( ut_ad(!trx->in_rw_trx_list); ut_ad(!trx_is_autocommit_non_locking(trx)); ut_ad(!trx->read_only); + ut_ad(trx->id == 0); if (high_level_read_only) { return; @@ -3064,14 +3108,10 @@ trx_set_rw_mode( ut_ad(trx->rsegs.m_redo.rseg != 0); mutex_enter(&trx_sys->mutex); - - ut_ad(trx->id == 0); trx->id = trx_sys_get_new_trx_id(); trx_sys->rw_trx_ids.push_back(trx->id); - trx_sys->rw_trx_set.insert(TrxTrack(trx->id, trx)); - /* So that we can see our own changes. */ if (MVCC::is_view_active(trx->read_view)) { MVCC::set_view_creator_trx_id(trx->read_view, trx->id); @@ -3088,6 +3128,7 @@ trx_set_rw_mode( ut_d(trx->in_rw_trx_list = true); mutex_exit(&trx_sys->mutex); + trx_sys->rw_trx_hash.insert(trx); } /** diff --git a/storage/innobase/ut/ut0dbg.cc b/storage/innobase/ut/ut0dbg.cc index 9e596dcda81..7df189ac560 100644 --- a/storage/innobase/ut/ut0dbg.cc +++ b/storage/innobase/ut/ut0dbg.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -53,7 +53,7 @@ ut_dbg_assertion_failed( " or crashes, even\n" "InnoDB: immediately after the mysqld startup, there may be\n" "InnoDB: corruption in the InnoDB tablespace. Please refer to\n" - "InnoDB: " REFMAN "forcing-innodb-recovery.html\n" + "InnoDB: https://mariadb.com/kb/en/library/xtradbinnodb-recovery-modes/\n" "InnoDB: about forcing recovery.\n", stderr); fflush(stderr); diff --git a/storage/perfschema/pfs.cc b/storage/perfschema/pfs.cc index a690bdf911d..fd415d8bc21 100644 --- a/storage/perfschema/pfs.cc +++ b/storage/perfschema/pfs.cc @@ -41,7 +41,6 @@ #include "sp_head.h" #include "pfs_digest.h" -using std::min; /** @page PAGE_PERFORMANCE_SCHEMA The Performance Schema main page MySQL PERFORMANCE_SCHEMA implementation. @@ -2022,7 +2021,7 @@ static void set_thread_account_v1(const char *user, int user_len, DBUG_ASSERT((host != NULL) || (host_len == 0)); DBUG_ASSERT(host_len >= 0); - host_len= min<size_t>(host_len, sizeof(pfs->m_hostname)); + host_len= MY_MIN(host_len, static_cast<int>(sizeof(pfs->m_hostname))); if (unlikely(pfs == NULL)) return; diff --git a/storage/perfschema/table_threads.cc b/storage/perfschema/table_threads.cc index 7be51b2bb20..1458c0b11d5 100644 --- a/storage/perfschema/table_threads.cc +++ b/storage/perfschema/table_threads.cc @@ -257,8 +257,8 @@ int table_threads::read_row_values(TABLE *table, changed to less than or equal to 64 characters. */ set_field_varchar_utf8(f, m_row.m_processlist_state_ptr, - std::min<uint>(m_row.m_processlist_state_length, - f->char_length())); + MY_MIN(m_row.m_processlist_state_length, + f->char_length())); } else f->set_null(); diff --git a/storage/rocksdb/ha_rocksdb.h b/storage/rocksdb/ha_rocksdb.h index 3730d983cea..ff715423f71 100644 --- a/storage/rocksdb/ha_rocksdb.h +++ b/storage/rocksdb/ha_rocksdb.h @@ -1412,6 +1412,6 @@ private: Rdb_inplace_alter_ctx &operator=(const Rdb_inplace_alter_ctx &); }; -const int MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL= MariaDB_PLUGIN_MATURITY_ALPHA; +const int MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL= MariaDB_PLUGIN_MATURITY_BETA; } // namespace myrocks diff --git a/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result b/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result index 291effa832c..6850d8dff16 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result @@ -1,28 +1,30 @@ DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t2; DROP TABLE IF EXISTS t3; +SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK=1; +create table t1 (a int) engine=rocksdb; +drop table t1; select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; TYPE NAME VALUE MAX_INDEX_ID MAX_INDEX_ID max_index_id CF_FLAGS 0 default [0] CF_FLAGS 1 __system__ [0] +DDL_DROP_INDEX_ONGOING cf_id:0,index_id:max_index_id select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; count(*) -3 +4 +SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK=0; select VALUE into @keysIn from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CF_NAME = 'default' and LEVEL = 'Sum' and TYPE = 'KeyIn'; CREATE TABLE t1 (i1 INT, i2 INT, PRIMARY KEY (i1)) ENGINE = ROCKSDB; INSERT INTO t1 VALUES (1, 1), (2, 2), (3, 3); select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; TYPE NAME VALUE -BINLOG FILE master-bin.000001 -BINLOG POS 1066 -BINLOG GTID uuid:5 MAX_INDEX_ID MAX_INDEX_ID max_index_id CF_FLAGS 0 default [0] CF_FLAGS 1 __system__ [0] select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; count(*) -6 +3 set global rocksdb_force_flush_memtable_now = true; set global rocksdb_compact_cf='default'; select case when VALUE-@keysIn >= 3 then 'true' else 'false' end from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CF_NAME = 'default' and LEVEL = 'Sum' and TYPE = 'KeyIn'; @@ -66,7 +68,7 @@ SHOW GLOBAL VARIABLES LIKE 'ROCKSDB_PAUSE_BACKGROUND_WORK'; Variable_name Value rocksdb_pause_background_work ON DROP TABLE t3; -cf_id:0,index_id:267 +cf_id:0,index_id:264 SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK=0; SHOW GLOBAL VARIABLES LIKE 'ROCKSDB_PAUSE_BACKGROUND_WORK'; Variable_name Value diff --git a/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result b/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result index 9952314cd2c..021373be02a 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result @@ -69,15 +69,15 @@ set global rocksdb_strict_collation_check=@tmp_rscc; # select plugin_name, plugin_maturity from information_schema.plugins where plugin_name like '%rocksdb%'; plugin_name plugin_maturity -ROCKSDB Alpha -ROCKSDB_CFSTATS Alpha -ROCKSDB_DBSTATS Alpha -ROCKSDB_PERF_CONTEXT Alpha -ROCKSDB_PERF_CONTEXT_GLOBAL Alpha -ROCKSDB_CF_OPTIONS Alpha -ROCKSDB_COMPACTION_STATS Alpha -ROCKSDB_GLOBAL_INFO Alpha -ROCKSDB_DDL Alpha -ROCKSDB_INDEX_FILE_MAP Alpha -ROCKSDB_LOCKS Alpha -ROCKSDB_TRX Alpha +ROCKSDB Beta +ROCKSDB_CFSTATS Beta +ROCKSDB_DBSTATS Beta +ROCKSDB_PERF_CONTEXT Beta +ROCKSDB_PERF_CONTEXT_GLOBAL Beta +ROCKSDB_CF_OPTIONS Beta +ROCKSDB_COMPACTION_STATS Beta +ROCKSDB_GLOBAL_INFO Beta +ROCKSDB_DDL Beta +ROCKSDB_INDEX_FILE_MAP Beta +ROCKSDB_LOCKS Beta +ROCKSDB_TRX Beta diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rpl_row_triggers.result b/storage/rocksdb/mysql-test/rocksdb/r/rpl_row_triggers.result index 1d3cd7db641..69acc4a92e8 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/rpl_row_triggers.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rpl_row_triggers.result @@ -1,12 +1,12 @@ include/master-slave.inc -Warnings: -Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. -Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] # Test of row replication with triggers on the slave side +connection master; CREATE TABLE t1 (C1 CHAR(1) primary key, C2 CHAR(1)); SELECT * FROM t1; C1 C2 +connection slave; +connection slave; SET @old_slave_exec_mode= @@global.slave_exec_mode; SET @old_slave_run_triggers_for_rbr= @@global.slave_run_triggers_for_rbr; SET @@global.slave_exec_mode= IDEMPOTENT; @@ -38,8 +38,11 @@ i0 0 i1 0 u0 0 u1 0 +connection master; # INSERT triggers test insert into t1 values ('a','b'); +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 0 @@ -48,8 +51,11 @@ i0 1 a i1 1 a u0 0 u1 0 +connection master; # UPDATE triggers test update t1 set C1= 'd'; +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 0 @@ -58,8 +64,11 @@ i0 1 a i1 1 a u0 1 a d u1 1 a d +connection master; # DELETE triggers test delete from t1 where C1='d'; +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 1 d @@ -78,7 +87,10 @@ i0 2 0 i1 2 0 u0 1 a d u1 1 a d +connection master; insert into t1 values ('0','1'); +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 1 d @@ -90,32 +102,44 @@ u1 2 0 0 # INSERT triggers which cause also DELETE test # (insert duplicate row in table referenced by foreign key) insert into t1 values ('1','1'); +connection master; drop table if exists t1; +connection slave; +connection slave; SET @@global.slave_exec_mode= @old_slave_exec_mode; SET @@global.slave_run_triggers_for_rbr= @old_slave_run_triggers_for_rbr; drop table t2; +connection master; CREATE TABLE t1 (i INT); CREATE TABLE t2 (i INT); +connection slave; SET @old_slave_run_triggers_for_rbr= @@global.slave_run_triggers_for_rbr; SET GLOBAL slave_run_triggers_for_rbr=YES; CREATE TRIGGER tr AFTER INSERT ON t1 FOR EACH ROW INSERT INTO t2 VALUES (new.i); +connection master; BEGIN; INSERT INTO t1 VALUES (1); INSERT INTO t1 VALUES (2); COMMIT; +connection slave; select * from t2; i 1 2 SET @@global.slave_run_triggers_for_rbr= @old_slave_run_triggers_for_rbr; +connection master; drop tables t2,t1; +connection slave; # Triggers on slave do not work if master has some +connection master; CREATE TABLE t1 (C1 CHAR(1) primary key, C2 CHAR(1)); SELECT * FROM t1; C1 C2 create trigger t1_dummy before delete on t1 for each row set @dummy= 1; +connection slave; +connection slave; SET @old_slave_exec_mode= @@global.slave_exec_mode; SET @old_slave_run_triggers_for_rbr= @@global.slave_run_triggers_for_rbr; SET @@global.slave_exec_mode= IDEMPOTENT; @@ -145,8 +169,11 @@ i0 0 i1 0 u0 0 u1 0 +connection master; # INSERT triggers test insert into t1 values ('a','b'); +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 0 @@ -155,8 +182,11 @@ i0 0 i1 0 u0 0 u1 0 +connection master; # UPDATE triggers test update t1 set C1= 'd'; +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 0 @@ -165,8 +195,11 @@ i0 0 i1 0 u0 0 u1 0 +connection master; # DELETE triggers test delete from t1 where C1='d'; +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 0 @@ -185,7 +218,10 @@ i0 1 0 i1 1 0 u0 0 u1 0 +connection master; insert into t1 values ('0','1'); +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 0 @@ -197,22 +233,30 @@ u1 0 # INSERT triggers which cause also DELETE test # (insert duplicate row in table referenced by foreign key) insert into t1 values ('1','1'); +connection master; drop table if exists t1; +connection slave; +connection slave; SET @@global.slave_exec_mode= @old_slave_exec_mode; SET @@global.slave_run_triggers_for_rbr= @old_slave_run_triggers_for_rbr; drop table t2; # # MDEV-5513: Trigger is applied to the rows after first one # +connection master; create table t1 (a int, b int); create table tlog (a int auto_increment primary key); set sql_log_bin=0; create trigger tr1 after insert on t1 for each row insert into tlog values (null); set sql_log_bin=1; +connection slave; +connection slave; set @slave_run_triggers_for_rbr.saved = @@slave_run_triggers_for_rbr; set global slave_run_triggers_for_rbr=1; create trigger tr2 before insert on t1 for each row set new.b = new.a; +connection master; insert into t1 values (1,10),(2,20),(3,30); +connection slave; select * from t1; a b 1 10 diff --git a/storage/rocksdb/mysql-test/rocksdb/t/disabled.def b/storage/rocksdb/mysql-test/rocksdb/t/disabled.def index 63c3d259d4c..8d0b5693a5b 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/disabled.def +++ b/storage/rocksdb/mysql-test/rocksdb/t/disabled.def @@ -69,11 +69,12 @@ compact_deletes: MDEV-12663 : rocksdb.compact_deletes times out and causes other blind_delete_without_tx_api: MDEV-12286: rocksdb.blind_delete_without_tx_api test fails unique_check: wrong error number autoinc_vars_thread: debug sync point wait timed out +information_schema: MDEV-14372: unstable testcase ## ## Tests that fail for some other reason ## -information_schema : MariaRocks: requires GTIDs mysqlbinlog_gtid_skip_empty_trans_rocksdb : MariaRocks: requires GTIDs rpl_row_triggers : MariaRocks: requires GTIDs + diff --git a/storage/rocksdb/mysql-test/rocksdb/t/information_schema-master.opt b/storage/rocksdb/mysql-test/rocksdb/t/information_schema-master.opt index 40b14167e17..86379847638 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/information_schema-master.opt +++ b/storage/rocksdb/mysql-test/rocksdb/t/information_schema-master.opt @@ -1 +1 @@ ---binlog_format=row --gtid_mode=ON --enforce_gtid_consistency --log_slave_updates +--binlog_format=row --log_slave_updates diff --git a/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test b/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test index c8d9b538529..2ffc186dd8f 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test @@ -9,19 +9,29 @@ DROP TABLE IF EXISTS t2; DROP TABLE IF EXISTS t3; --enable_warnings +# MariaDB: the following is for handling the case where the tests +# is started on a totally empty datadir, where no MyRocks table has +# ever been created). In that case, there is no MAX_INDEX_ID. +# Create/drop a table so that we do have MAX_INDEX_ID. +SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK=1; +create table t1 (a int) engine=rocksdb; +drop table t1; + --let $max_index_id = query_get_value(SELECT * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO where type = 'MAX_INDEX_ID', VALUE, 1) --replace_result $max_index_id max_index_id select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; +SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK=0; select VALUE into @keysIn from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CF_NAME = 'default' and LEVEL = 'Sum' and TYPE = 'KeyIn'; CREATE TABLE t1 (i1 INT, i2 INT, PRIMARY KEY (i1)) ENGINE = ROCKSDB; INSERT INTO t1 VALUES (1, 1), (2, 2), (3, 3); ---let $MASTER_UUID = query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1) +# No binlog coordinates in MariaDB: --let $MASTER_UUID = query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1) --let $max_index_id = query_get_value(SELECT * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO where type = 'MAX_INDEX_ID', VALUE, 1) ---replace_result $MASTER_UUID uuid $max_index_id max_index_id +# No binlog coordinates in MariaDB: --replace_result $MASTER_UUID uuid $max_index_id max_index_id +--replace_result $max_index_id max_index_id select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/include/rocksdb_sys_var.inc b/storage/rocksdb/mysql-test/rocksdb_sys_vars/include/rocksdb_sys_var.inc index 6ba93026674..db0abc57358 100644 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/include/rocksdb_sys_var.inc +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/include/rocksdb_sys_var.inc @@ -10,6 +10,7 @@ --eval SET @start_global_value = @@global.$sys_var if (!$suppress_default_value) { + --replace_regex /[a-f0-9]{40}/#/ SELECT @start_global_value; if ($session) { diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_git_hash_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_git_hash_basic.result new file mode 100644 index 00000000000..bbcfa1417eb --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_git_hash_basic.result @@ -0,0 +1,7 @@ +SET @start_global_value = @@global.ROCKSDB_GIT_HASH; +SELECT @start_global_value; +@start_global_value +# +"Trying to set variable @@global.ROCKSDB_GIT_HASH to 444. It should fail because it is readonly." +SET @@global.ROCKSDB_GIT_HASH = 444; +ERROR HY000: Variable 'rocksdb_git_hash' is a read only variable diff --git a/storage/rocksdb/rdb_datadic.cc b/storage/rocksdb/rdb_datadic.cc index ff8a4faee55..a6d9a358b35 100644 --- a/storage/rocksdb/rdb_datadic.cc +++ b/storage/rocksdb/rdb_datadic.cc @@ -4261,7 +4261,7 @@ bool Rdb_binlog_manager::read(char *const binlog_name, std::string value; rocksdb::Status status = m_dict->get_value(m_key_slice, &value); if (status.ok()) { - if (!unpack_value((const uchar *)value.c_str(), binlog_name, binlog_pos, + if (!unpack_value((const uchar *)value.c_str(), value.size(), binlog_name, binlog_pos, binlog_gtid)) ret = true; } @@ -4331,35 +4331,60 @@ Rdb_binlog_manager::pack_value(uchar *const buf, const char *const binlog_name, @return true on error */ bool Rdb_binlog_manager::unpack_value(const uchar *const value, + size_t value_size_arg, char *const binlog_name, my_off_t *const binlog_pos, char *const binlog_gtid) const { uint pack_len = 0; + intmax_t value_size= value_size_arg; DBUG_ASSERT(binlog_pos != nullptr); + if ((value_size -= Rdb_key_def::VERSION_SIZE) < 0) + return true; // read version const uint16_t version = rdb_netbuf_to_uint16(value); + pack_len += Rdb_key_def::VERSION_SIZE; if (version != Rdb_key_def::BINLOG_INFO_INDEX_NUMBER_VERSION) return true; + if ((value_size -= sizeof(uint16)) < 0) + return true; + // read binlog file name length const uint16_t binlog_name_len = rdb_netbuf_to_uint16(value + pack_len); pack_len += sizeof(uint16); + + if (binlog_name_len >= (FN_REFLEN+1)) + return true; + + if ((value_size -= binlog_name_len) < 0) + return true; + if (binlog_name_len) { // read and set binlog name memcpy(binlog_name, value + pack_len, binlog_name_len); binlog_name[binlog_name_len] = '\0'; pack_len += binlog_name_len; + if ((value_size -= sizeof(uint32)) < 0) + return true; // read and set binlog pos *binlog_pos = rdb_netbuf_to_uint32(value + pack_len); pack_len += sizeof(uint32); + if ((value_size -= sizeof(uint16)) < 0) + return true; // read gtid length const uint16_t binlog_gtid_len = rdb_netbuf_to_uint16(value + pack_len); pack_len += sizeof(uint16); + + if (binlog_gtid_len >= GTID_BUF_LEN) + return true; + if ((value_size -= binlog_gtid_len) < 0) + return true; + if (binlog_gtid && binlog_gtid_len > 0) { // read and set gtid memcpy(binlog_gtid, value + pack_len, binlog_gtid_len); diff --git a/storage/rocksdb/rdb_datadic.h b/storage/rocksdb/rdb_datadic.h index 85846e8f968..fa0c0fd09e5 100644 --- a/storage/rocksdb/rdb_datadic.h +++ b/storage/rocksdb/rdb_datadic.h @@ -46,6 +46,8 @@ class Rdb_field_packing; class Rdb_cf_manager; class Rdb_ddl_manager; +const uint32_t GTID_BUF_LEN = 60; + /* @brief Field packing context. @@ -1158,7 +1160,8 @@ private: rocksdb::Slice pack_value(uchar *const buf, const char *const binlog_name, const my_off_t &binlog_pos, const char *const binlog_gtid) const; - bool unpack_value(const uchar *const value, char *const binlog_name, + bool unpack_value(const uchar *const value, size_t value_size, + char *const binlog_name, my_off_t *const binlog_pos, char *const binlog_gtid) const; std::atomic<Rdb_tbl_def *> m_slave_gtid_info_tbl; diff --git a/storage/rocksdb/rdb_i_s.cc b/storage/rocksdb/rdb_i_s.cc index 39f328444bf..8407f280d5c 100644 --- a/storage/rocksdb/rdb_i_s.cc +++ b/storage/rocksdb/rdb_i_s.cc @@ -729,7 +729,6 @@ static int rdb_i_s_global_info_fill_table( DBUG_ASSERT(tables != nullptr); static const uint32_t INT_BUF_LEN = 21; - static const uint32_t GTID_BUF_LEN = 60; static const uint32_t CF_ID_INDEX_BUF_LEN = 60; int ret = 0; diff --git a/strings/CMakeLists.txt b/strings/CMakeLists.txt index 52dc2ed2b0f..1f1e4635f31 100644 --- a/strings/CMakeLists.txt +++ b/strings/CMakeLists.txt @@ -34,4 +34,5 @@ ADD_DEFINITIONS(-DDISABLE_MYSQL_THREAD_H) ADD_CONVENIENCE_LIBRARY(strings ${STRINGS_SOURCES}) ADD_EXECUTABLE(conf_to_src EXCLUDE_FROM_ALL conf_to_src.c) -TARGET_LINK_LIBRARIES(conf_to_src strings) +SET_TARGET_PROPERTIES(conf_to_src PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD TRUE) +TARGET_LINK_LIBRARIES(conf_to_src mysys strings) diff --git a/strings/ctype-uca.c b/strings/ctype-uca.c index e3ed531df93..4386b6aa432 100644 --- a/strings/ctype-uca.c +++ b/strings/ctype-uca.c @@ -34034,7 +34034,7 @@ apply_one_rule(MY_CHARSET_LOADER *loader, /** Check if collation rules are valid, - i.e. characters are not outside of the collation suported range. + i.e. characters are not outside of the collation supported range. */ static int check_rules(MY_CHARSET_LOADER *loader, @@ -34170,7 +34170,7 @@ init_weight_level(MY_CHARSET_LOADER *loader, MY_COLL_RULES *rules, memcpy(dst->weights, src->weights, npages * sizeof(uint16 *)); /* - Calculate maximum lenghts for the pages which will be overwritten. + Calculate maximum lengths for the pages which will be overwritten. Mark pages that will be otherwriten as NULL. We'll allocate their own memory. */ @@ -34188,7 +34188,7 @@ init_weight_level(MY_CHARSET_LOADER *loader, MY_COLL_RULES *rules, { /* Not an expansion and not a contraction. - The page correspoding to r->curr[0] in "dst" + The page corresponding to r->curr[0] in "dst" will need at least the same amount of weights that r->base[0] has in "src". */ diff --git a/strings/decimal.c b/strings/decimal.c index 7db5111fc84..d7fd9a5722b 100644 --- a/strings/decimal.c +++ b/strings/decimal.c @@ -1694,7 +1694,7 @@ done: scale increment for '/' NOTE - returned valued may be larger than the actual buffer requred + returned valued may be larger than the actual buffer required in the operation, as decimal_result_size, by design, operates on precision/scale values only and not on the actual decimal number diff --git a/win/create_def_file.js b/win/create_def_file.js index 25bbbb4eb3d..9a178510ca8 100644 --- a/win/create_def_file.js +++ b/win/create_def_file.js @@ -257,4 +257,9 @@ function addToResponseFile(filename, responseFile) responseFile.WriteLine("\""+fso.GetFile(filename).Path+"\""); } } + else + { + echo("file " + filename + " not found. Can't generate symbols file"); + WScript.Quit (1); + } } diff --git a/win/packaging/ca/CustomAction.cpp b/win/packaging/ca/CustomAction.cpp index c0062ddcdd1..9803b0b07a2 100644 --- a/win/packaging/ca/CustomAction.cpp +++ b/win/packaging/ca/CustomAction.cpp @@ -524,7 +524,7 @@ LExit: or 4GB(x64 OS). Fragmentation due to loaded modules, heap and stack - limit maximum size of continous memory block further, + limit maximum size of continuous memory block further, so that limit for 32 bit process is about 1200 on 32 bit OS or 2000 MB on 64 bit OS(found experimentally). */ |