diff options
author | Marko Mäkelä <marko.makela@mariadb.com> | 2021-08-18 12:26:58 +0300 |
---|---|---|
committer | Marko Mäkelä <marko.makela@mariadb.com> | 2021-08-18 12:26:58 +0300 |
commit | cd65845a0eee830b8c783d2ebecae9d305e255c3 (patch) | |
tree | 72ef5515315361568e9734e6253be80213d4a5bf | |
parent | 2d259187a2f585ef6bc5bd66997975ebc4264c66 (diff) | |
parent | f73eea4984b632c1955211b9ea5b54be9dd56975 (diff) | |
download | mariadb-git-cd65845a0eee830b8c783d2ebecae9d305e255c3.tar.gz |
Merge 10.2 into 10.3
MDEV-18734 FIXME: vcol.partition triggers ASAN heap-use-after-free
39 files changed, 909 insertions, 301 deletions
diff --git a/cmake/plugin.cmake b/cmake/plugin.cmake index ff911008939..e7e9d2a3642 100644 --- a/cmake/plugin.cmake +++ b/cmake/plugin.cmake @@ -26,13 +26,13 @@ INCLUDE(CMakeParseArguments) # [STATIC_OUTPUT_NAME static_name] # [RECOMPILE_FOR_EMBEDDED] # [LINK_LIBRARIES lib1...libN] -# [DEPENDENCIES target1...targetN] +# [DEPENDS target1...targetN] MACRO(MYSQL_ADD_PLUGIN) CMAKE_PARSE_ARGUMENTS(ARG "STORAGE_ENGINE;STATIC_ONLY;MODULE_ONLY;MANDATORY;DEFAULT;DISABLED;RECOMPILE_FOR_EMBEDDED;CLIENT" "MODULE_OUTPUT_NAME;STATIC_OUTPUT_NAME;COMPONENT;CONFIG" - "LINK_LIBRARIES;DEPENDENCIES" + "LINK_LIBRARIES;DEPENDS" ${ARGN} ) IF(NOT WITHOUT_SERVER OR ARG_CLIENT) @@ -111,8 +111,8 @@ MACRO(MYSQL_ADD_PLUGIN) ENDIF() UNSET(${with_var} CACHE) - IF(NOT ARG_DEPENDENCIES) - SET(ARG_DEPENDENCIES) + IF(NOT ARG_DEPENDS) + SET(ARG_DEPENDS) ENDIF() IF(NOT ARG_MODULE_OUTPUT_NAME) @@ -138,7 +138,7 @@ MACRO(MYSQL_ADD_PLUGIN) ADD_LIBRARY(${target} STATIC ${SOURCES}) DTRACE_INSTRUMENT(${target}) - ADD_DEPENDENCIES(${target} GenError ${ARG_DEPENDENCIES}) + ADD_DEPENDENCIES(${target} GenError ${ARG_DEPENDS}) RESTRICT_SYMBOL_EXPORTS(${target}) IF(WITH_EMBEDDED_SERVER) # Embedded library should contain PIC code and be linkable @@ -152,7 +152,7 @@ MACRO(MYSQL_ADD_PLUGIN) SET_TARGET_PROPERTIES(${target}_embedded PROPERTIES COMPILE_DEFINITIONS "EMBEDDED_LIBRARY") ENDIF() - ADD_DEPENDENCIES(${target}_embedded GenError) + ADD_DEPENDENCIES(${target}_embedded GenError ${ARG_DEPENDS}) ENDIF() ENDIF() @@ -213,7 +213,7 @@ MACRO(MYSQL_ADD_PLUGIN) TARGET_LINK_LIBRARIES (${target} "-Wl,--no-undefined") ENDIF() - ADD_DEPENDENCIES(${target} GenError ${ARG_DEPENDENCIES}) + ADD_DEPENDENCIES(${target} GenError ${ARG_DEPENDS}) SET_TARGET_PROPERTIES(${target} PROPERTIES OUTPUT_NAME "${ARG_MODULE_OUTPUT_NAME}") @@ -256,15 +256,20 @@ MACRO(MYSQL_ADD_PLUGIN) INSTALL_MYSQL_TEST("${CMAKE_CURRENT_SOURCE_DIR}/mysql-test/" "plugin/${subpath}") ENDIF() - GET_TARGET_PROPERTY(plugin_type ${target} TYPE) - STRING(REGEX REPLACE "_LIBRARY$" "" plugin_type ${plugin_type}) - STRING(REGEX REPLACE "^NO$" "" plugin_type ${plugin_type}) + IF(TARGET ${target}) + GET_TARGET_PROPERTY(plugin_type ${target} TYPE) + STRING(REPLACE "_LIBRARY" "" plugin_type ${plugin_type}) + SET(have_target 1) + ELSE() + SET(plugin_type) + SET(have_target 0) + ENDIF() IF(ARG_STORAGE_ENGINE) - ADD_FEATURE_INFO(${plugin} PLUGIN_${plugin} "Storage Engine ${plugin_type}") + ADD_FEATURE_INFO(${plugin} ${have_target} "Storage Engine ${plugin_type}") ELSEIF(ARG_CLIENT) - ADD_FEATURE_INFO(${plugin} PLUGIN_${plugin} "Client plugin ${plugin_type}") + ADD_FEATURE_INFO(${plugin} ${have_target} "Client plugin ${plugin_type}") ELSE() - ADD_FEATURE_INFO(${plugin} PLUGIN_${plugin} "Server plugin ${plugin_type}") + ADD_FEATURE_INFO(${plugin} ${have_target} "Server plugin ${plugin_type}") ENDIF() ENDIF(NOT WITHOUT_SERVER OR ARG_CLIENT) ENDMACRO() diff --git a/mysql-test/suite/encryption/r/innodb_import.result b/mysql-test/suite/encryption/r/innodb_import.result new file mode 100644 index 00000000000..54b95ab26d4 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb_import.result @@ -0,0 +1,22 @@ +# +# MDEV-26131 SEGV in ha_innobase::discard_or_import_tablespace +# +CREATE TABLE t1(f1 int,f2 text)ENGINE=InnoDB; +INSERT INTO t1 VALUES(1, "InnoDB"); +CREATE TABLE t2 LIKE t1; +ALTER TABLE t2 ADD KEY idx (f2(13)); +ALTER TABLE t2 DISCARD TABLESPACE; +FLUSH TABLES t1 FOR EXPORT; +UNLOCK TABLES; +ALTER TABLE t2 IMPORT TABLESPACE; +ERROR HY000: Internal error: Drop all secondary indexes before importing table test/t2 when .cfg file is missing. +ALTER TABLE t2 DROP KEY idx; +Warnings: +Warning 1814 Tablespace has been discarded for table `t2` +ALTER TABLE t2 IMPORT TABLESPACE; +Warnings: +Warning 1810 IO Read error: (2, No such file or directory) Error opening './test/t2.cfg', will attempt to import without schema verification +SELECT * FROM t2; +f1 f2 +1 InnoDB +DROP TABLE t1, t2; diff --git a/mysql-test/suite/encryption/t/innodb_import.combinations b/mysql-test/suite/encryption/t/innodb_import.combinations new file mode 100644 index 00000000000..75458949582 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb_import.combinations @@ -0,0 +1,7 @@ +[page_compressed] +innodb-compression-default=1 +[encryption] +innodb-encrypt-tables=1 +[page_compressed_encryption] +innodb-compression-default=1 +innodb-encrypt-tables=1 diff --git a/mysql-test/suite/encryption/t/innodb_import.test b/mysql-test/suite/encryption/t/innodb_import.test new file mode 100644 index 00000000000..791a1757878 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb_import.test @@ -0,0 +1,22 @@ +--source include/have_innodb.inc +-- source include/have_example_key_management_plugin.inc +--echo # +--echo # MDEV-26131 SEGV in ha_innobase::discard_or_import_tablespace +--echo # +let $MYSQLD_DATADIR = `SELECT @@datadir`; +CREATE TABLE t1(f1 int,f2 text)ENGINE=InnoDB; +INSERT INTO t1 VALUES(1, "InnoDB"); +CREATE TABLE t2 LIKE t1; +ALTER TABLE t2 ADD KEY idx (f2(13)); +ALTER TABLE t2 DISCARD TABLESPACE; +FLUSH TABLES t1 FOR EXPORT; +--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_DATADIR/test/t2.ibd +UNLOCK TABLES; +--error ER_INTERNAL_ERROR +ALTER TABLE t2 IMPORT TABLESPACE; + +ALTER TABLE t2 DROP KEY idx; +--replace_regex /opening '.*\/test\//opening '.\/test\// +ALTER TABLE t2 IMPORT TABLESPACE; +SELECT * FROM t2; +DROP TABLE t1, t2; diff --git a/mysql-test/suite/federated/federated_partition.result b/mysql-test/suite/federated/federated_partition.result index c8a61d825b6..374dba515c7 100644 --- a/mysql-test/suite/federated/federated_partition.result +++ b/mysql-test/suite/federated/federated_partition.result @@ -47,6 +47,42 @@ connection slave; drop table federated.t1_1; drop table federated.t1_2; End of 5.1 tests +# +# MDEV-18734 ASAN heap-use-after-free upon sorting by blob column from partitioned table +# +connection slave; +use federated; +create table t1_1 (x int, b text, key(x)); +create table t1_2 (x int, b text, key(x)); +connection master; +create table t1 (x int, b text, key(x)) engine=federated +partition by range columns (x) ( +partition p1 values less than (40) connection='mysql://root@127.0.0.1:SLAVE_PORT/federated/t1_1', +partition pn values less than (maxvalue) connection='mysql://root@127.0.0.1:SLAVE_PORT/federated/t1_2' +); +insert t1 values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, 7), (8, 8); +insert t1 select x + 8, x + 8 from t1; +insert t1 select x + 16, x + 16 from t1; +insert t1 select x + 49, repeat(x + 49, 100) from t1; +flush tables; +# This produces wrong result before MDEV-17573 +select x, left(b, 10) from t1 where x > 30 and x < 60 order by b; +x left(b, 10) +31 31 +32 32 +50 5050505050 +51 5151515151 +52 5252525252 +53 5353535353 +54 5454545454 +55 5555555555 +56 5656565656 +57 5757575757 +58 5858585858 +59 5959595959 +drop table t1; +connection slave; +drop table t1_1, t1_2; connection master; DROP TABLE IF EXISTS federated.t1; DROP DATABASE IF EXISTS federated; diff --git a/mysql-test/suite/federated/federated_partition.test b/mysql-test/suite/federated/federated_partition.test index 47110b5eebf..42f56134279 100644 --- a/mysql-test/suite/federated/federated_partition.test +++ b/mysql-test/suite/federated/federated_partition.test @@ -51,4 +51,29 @@ drop table federated.t1_2; --echo End of 5.1 tests +--echo # +--echo # MDEV-18734 ASAN heap-use-after-free upon sorting by blob column from partitioned table +--echo # +connection slave; +use federated; +create table t1_1 (x int, b text, key(x)); +create table t1_2 (x int, b text, key(x)); +connection master; +--replace_result $SLAVE_MYPORT SLAVE_PORT +eval create table t1 (x int, b text, key(x)) engine=federated + partition by range columns (x) ( + partition p1 values less than (40) connection='mysql://root@127.0.0.1:$SLAVE_MYPORT/federated/t1_1', + partition pn values less than (maxvalue) connection='mysql://root@127.0.0.1:$SLAVE_MYPORT/federated/t1_2' +); +insert t1 values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5), (6, 6), (7, 7), (8, 8); +insert t1 select x + 8, x + 8 from t1; +insert t1 select x + 16, x + 16 from t1; +insert t1 select x + 49, repeat(x + 49, 100) from t1; +flush tables; +--echo # This produces wrong result before MDEV-17573 +select x, left(b, 10) from t1 where x > 30 and x < 60 order by b; +drop table t1; +connection slave; +drop table t1_1, t1_2; + source include/federated_cleanup.inc; diff --git a/mysql-test/suite/innodb/r/import_corrupted.result b/mysql-test/suite/innodb/r/import_corrupted.result new file mode 100644 index 00000000000..c0474ebbb1d --- /dev/null +++ b/mysql-test/suite/innodb/r/import_corrupted.result @@ -0,0 +1,30 @@ +call mtr.add_suppression("Table `test`.`t2` should have 2 indexes but the tablespace has 1 indexes"); +call mtr.add_suppression("Index for table 't2' is corrupt; try to repair it"); +call mtr.add_suppression("Trying to read page number 23 in space .*, space name test/t2, which is outside the tablespace bounds. Byte offset 0, len 16384"); +CREATE TABLE t1 ( +id INT AUTO_INCREMENT PRIMARY KEY, +not_id INT, +data CHAR(255), +data2 BLOB +) ENGINE=INNODB; +ALTER TABLE t1 MODIFY not_id INT UNIQUE KEY; +connect purge_control,localhost,root,,; +START TRANSACTION WITH CONSISTENT SNAPSHOT; +connection default; +DELETE FROM t1 WHERE id % 2 = 1; +FLUSH TABLES t1 FOR EXPORT; +UNLOCK TABLES; +connection purge_control; +COMMIT; +connection default; +DROP TABLE t1; +CREATE TABLE t2 ( +id INT AUTO_INCREMENT PRIMARY KEY, +not_id INT UNIQUE KEY, +data CHAR(255), +data2 BLOB +) ENGINE=INNODB; +ALTER TABLE t2 DISCARD TABLESPACE; +ALTER TABLE t2 IMPORT TABLESPACE; +ERROR HY000: Index for table 't2' is corrupt; try to repair it +DROP TABLE t2; diff --git a/mysql-test/suite/innodb/t/import_corrupted.test b/mysql-test/suite/innodb/t/import_corrupted.test new file mode 100644 index 00000000000..b4701949f33 --- /dev/null +++ b/mysql-test/suite/innodb/t/import_corrupted.test @@ -0,0 +1,67 @@ +--source include/have_innodb.inc + +call mtr.add_suppression("Table `test`.`t2` should have 2 indexes but the tablespace has 1 indexes"); +call mtr.add_suppression("Index for table 't2' is corrupt; try to repair it"); +call mtr.add_suppression("Trying to read page number 23 in space .*, space name test/t2, which is outside the tablespace bounds. Byte offset 0, len 16384"); + +let MYSQLD_DATADIR = `SELECT @@datadir`; + +CREATE TABLE t1 ( + id INT AUTO_INCREMENT PRIMARY KEY, + not_id INT, + data CHAR(255), + data2 BLOB +) ENGINE=INNODB; + +--disable_query_log +--let i = 0 +while ($i != 1000) { + eval INSERT INTO t1 VALUES (DEFAULT, $i, REPEAT('b', 255), REPEAT('a', 5000)); + --inc $i +} +--enable_query_log + +ALTER TABLE t1 MODIFY not_id INT UNIQUE KEY; + +connect (purge_control,localhost,root,,); +START TRANSACTION WITH CONSISTENT SNAPSHOT; +connection default; + +DELETE FROM t1 WHERE id % 2 = 1; + +FLUSH TABLES t1 FOR EXPORT; + +--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_DATADIR/test/tmp.ibd +--copy_file $MYSQLD_DATADIR/test/t1.cfg $MYSQLD_DATADIR/test/tmp.cfg + +perl; +use strict; +die unless open(FILE, "+<$ENV{MYSQLD_DATADIR}/test/tmp.ibd"); +die unless truncate(FILE, 16384*23); +close(FILE); +EOF + +UNLOCK TABLES; +connection purge_control; +COMMIT; +connection default; +DROP TABLE t1; + +CREATE TABLE t2 ( + id INT AUTO_INCREMENT PRIMARY KEY, + not_id INT UNIQUE KEY, + data CHAR(255), + data2 BLOB +) ENGINE=INNODB; + +ALTER TABLE t2 DISCARD TABLESPACE; + +--copy_file $MYSQLD_DATADIR/test/tmp.ibd $MYSQLD_DATADIR/test/t2.ibd +--copy_file $MYSQLD_DATADIR/test/tmp.cfg $MYSQLD_DATADIR/test/t2.cfg + +--error ER_NOT_KEYFILE +ALTER TABLE t2 IMPORT TABLESPACE; + +DROP TABLE t2; + +--remove_file $MYSQLD_DATADIR/test/tmp.ibd diff --git a/mysql-test/suite/innodb_gis/r/alter_spatial_index.result b/mysql-test/suite/innodb_gis/r/alter_spatial_index.result index 9bf4ae6cd06..6730aa89486 100644 --- a/mysql-test/suite/innodb_gis/r/alter_spatial_index.result +++ b/mysql-test/suite/innodb_gis/r/alter_spatial_index.result @@ -252,6 +252,16 @@ UNLOCK TABLES; ALTER TABLE tab DISCARD TABLESPACE; SELECT c1,ST_Astext(c2),ST_Astext(c4) FROM tab; ERROR HY000: Tablespace has been discarded for table `tab` +ERROR HY000: Internal error: Drop all secondary indexes before importing table test/tab when .cfg file is missing. +Table Create Table +tab CREATE TABLE `tab` ( + `c1` int(11) NOT NULL, + `c2` point NOT NULL, + `c3` linestring NOT NULL, + `c4` polygon NOT NULL, + `c5` geometry NOT NULL, + PRIMARY KEY (`c2`(25)) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 CHECK TABLE tab; Table Op Msg_type Msg_text test.tab check status OK @@ -282,9 +292,6 @@ INSERT INTO tab SELECT * FROM tab1; ALTER TABLE tab DROP PRIMARY KEY; affected rows: 1 info: Records: 1 Duplicates: 0 Warnings: 0 -ALTER TABLE tab DROP INDEX idx2; -affected rows: 0 -info: Records: 0 Duplicates: 0 Warnings: 0 SET STATEMENT sql_mode = 'NO_ENGINE_SUBSTITUTION' FOR CREATE TEMPORARY TABLE temp_tab AS SELECT * FROM tab where c1 = c2; ERROR HY000: Illegal parameter data types int and geometry for operation '=' @@ -325,18 +332,10 @@ tab CREATE TABLE `tab` ( `c2` point NOT NULL, `c3` linestring NOT NULL, `c4` polygon NOT NULL, - `c5` geometry NOT NULL, - SPATIAL KEY `idx3` (`c3`), - SPATIAL KEY `idx4` (`c4`) COMMENT 'testing spatial index on Polygon', - SPATIAL KEY `idx5` (`c5`) COMMENT 'testing spatial index on Geometry', - KEY `idx6` (`c4`(10)) USING BTREE + `c5` geometry NOT NULL ) ENGINE=InnoDB DEFAULT CHARSET=latin1 SHOW INDEX FROM tab; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment -tab 1 idx3 1 c3 A # 32 NULL SPATIAL -tab 1 idx4 1 c4 A # 32 NULL SPATIAL testing spatial index on Polygon -tab 1 idx5 1 c5 A # 32 NULL SPATIAL testing spatial index on Geometry -tab 1 idx6 1 c4 A # 10 NULL BTREE DELETE FROM tab; ALTER TABLE tab ADD PRIMARY KEY(c2); affected rows: 0 @@ -357,20 +356,12 @@ tab CREATE TABLE `tab` ( `c5` geometry NOT NULL, PRIMARY KEY (`c2`(25)), UNIQUE KEY `const_1` (`c2`(25)), - SPATIAL KEY `idx3` (`c3`), - SPATIAL KEY `idx4` (`c4`) COMMENT 'testing spatial index on Polygon', - SPATIAL KEY `idx5` (`c5`) COMMENT 'testing spatial index on Geometry', - KEY `idx6` (`c4`(10)) USING BTREE, SPATIAL KEY `idx2` (`c2`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 SHOW INDEX FROM tab; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tab 0 PRIMARY 1 c2 A # 25 NULL BTREE tab 0 const_1 1 c2 A # 25 NULL BTREE -tab 1 idx3 1 c3 A # 32 NULL SPATIAL -tab 1 idx4 1 c4 A # 32 NULL SPATIAL testing spatial index on Polygon -tab 1 idx5 1 c5 A # 32 NULL SPATIAL testing spatial index on Geometry -tab 1 idx6 1 c4 A # 10 NULL BTREE tab 1 idx2 1 c2 A # 32 NULL SPATIAL INSERT INTO tab(c1,c2,c3,c4,c5) VALUES(1,ST_GeomFromText('POINT(10 10)'),ST_GeomFromText('LINESTRING(5 5,20 20,30 30)'), @@ -399,20 +390,12 @@ tab CREATE TABLE `tab` ( `c5` geometry NOT NULL, PRIMARY KEY (`c5`(10)), UNIQUE KEY `const_1` (`c5`(10)), - SPATIAL KEY `idx3` (`c3`), - SPATIAL KEY `idx4` (`c4`) COMMENT 'testing spatial index on Polygon', - SPATIAL KEY `idx5` (`c5`) COMMENT 'testing spatial index on Geometry', - KEY `idx6` (`c4`(10)) USING BTREE, SPATIAL KEY `idx2` (`c2`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1 SHOW INDEX FROM tab; Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment tab 0 PRIMARY 1 c5 A # 10 NULL BTREE tab 0 const_1 1 c5 A # 10 NULL BTREE -tab 1 idx3 1 c3 A # 32 NULL SPATIAL -tab 1 idx4 1 c4 A # 32 NULL SPATIAL testing spatial index on Polygon -tab 1 idx5 1 c5 A # 32 NULL SPATIAL testing spatial index on Geometry -tab 1 idx6 1 c4 A # 10 NULL BTREE tab 1 idx2 1 c2 A # 32 NULL SPATIAL INSERT INTO tab(c1,c2,c3,c4,c5) VALUES(1,ST_GeomFromText('POINT(10 10)'),ST_GeomFromText('LINESTRING(5 5,20 20,30 30)'), diff --git a/mysql-test/suite/innodb_gis/t/alter_spatial_index.test b/mysql-test/suite/innodb_gis/t/alter_spatial_index.test index 653e250017a..2039e2d1c39 100644 --- a/mysql-test/suite/innodb_gis/t/alter_spatial_index.test +++ b/mysql-test/suite/innodb_gis/t/alter_spatial_index.test @@ -277,8 +277,17 @@ SELECT c1,ST_Astext(c2),ST_Astext(c4) FROM tab; --disable_query_log +--error ER_INTERNAL_ERROR ALTER TABLE tab IMPORT TABLESPACE; +ALTER TABLE tab DROP INDEX idx2; +ALTER TABLE tab DROP INDEX idx3; +ALTER TABLE tab DROP INDEX idx4; +ALTER TABLE tab DROP INDEX idx5; +ALTER TABLE tab DROP INDEX idx6; + +SHOW CREATE TABLE tab; +ALTER TABLE tab IMPORT TABLESPACE; --enable_query_log CHECK TABLE tab; @@ -308,7 +317,6 @@ INSERT INTO tab SELECT * FROM tab1; --enable_info ALTER TABLE tab DROP PRIMARY KEY; -ALTER TABLE tab DROP INDEX idx2; --disable_info # Check spatial index on temp tables diff --git a/mysql-test/suite/vcol/r/partition.result b/mysql-test/suite/vcol/r/partition.result index bd1353fa145..d7c5052b72a 100644 --- a/mysql-test/suite/vcol/r/partition.result +++ b/mysql-test/suite/vcol/r/partition.result @@ -28,3 +28,76 @@ set statement sql_mode= '' for update t1 set i= 1, v= 2; Warnings: Warning 1906 The value specified for generated column 'v' in table 't1' has been ignored drop table t1; +# +# MDEV-18734 ASAN heap-use-after-free in my_strnxfrm_simple_internal upon update on versioned partitioned table +# +# Cover queue_fix() in ha_partition::handle_ordered_index_scan() +create or replace table t1 ( +x int auto_increment primary key, +b text, v mediumtext as (b) virtual, +index (v(10)) +) partition by range columns (x) ( +partition p1 values less than (3), +partition p2 values less than (6), +partition p3 values less than (9), +partition p4 values less than (12), +partition p5 values less than (15), +partition p6 values less than (17), +partition p7 values less than (19), +partition p8 values less than (21), +partition p9 values less than (23), +partition p10 values less than (25), +partition p11 values less than (27), +partition p12 values less than (29), +partition p13 values less than (31), +partition p14 values less than (33), +partition p15 values less than (35), +partition pn values less than (maxvalue)); +insert into t1 (b) values +(repeat('q', 8192)), (repeat('z', 8192)), (repeat('a', 8192)), (repeat('b', 8192)), +(repeat('x', 8192)), (repeat('y', 8192)); +insert t1 (b) select b from t1; +insert t1 (b) select b from t1; +insert t1 (b) select b from t1; +insert t1 (b) select b from t1; +select x, left(b, 10), left(v, 10) from t1 where x > 30 and x < 60 order by v; +x left(b, 10) left(v, 10) +33 aaaaaaaaaa aaaaaaaaaa +39 aaaaaaaaaa aaaaaaaaaa +45 aaaaaaaaaa aaaaaaaaaa +51 aaaaaaaaaa aaaaaaaaaa +57 aaaaaaaaaa aaaaaaaaaa +34 bbbbbbbbbb bbbbbbbbbb +40 bbbbbbbbbb bbbbbbbbbb +46 bbbbbbbbbb bbbbbbbbbb +52 bbbbbbbbbb bbbbbbbbbb +58 bbbbbbbbbb bbbbbbbbbb +31 qqqqqqqqqq qqqqqqqqqq +37 qqqqqqqqqq qqqqqqqqqq +43 qqqqqqqqqq qqqqqqqqqq +49 qqqqqqqqqq qqqqqqqqqq +55 qqqqqqqqqq qqqqqqqqqq +35 xxxxxxxxxx xxxxxxxxxx +41 xxxxxxxxxx xxxxxxxxxx +47 xxxxxxxxxx xxxxxxxxxx +53 xxxxxxxxxx xxxxxxxxxx +59 xxxxxxxxxx xxxxxxxxxx +36 yyyyyyyyyy yyyyyyyyyy +42 yyyyyyyyyy yyyyyyyyyy +48 yyyyyyyyyy yyyyyyyyyy +54 yyyyyyyyyy yyyyyyyyyy +32 zzzzzzzzzz zzzzzzzzzz +38 zzzzzzzzzz zzzzzzzzzz +44 zzzzzzzzzz zzzzzzzzzz +50 zzzzzzzzzz zzzzzzzzzz +56 zzzzzzzzzz zzzzzzzzzz +update t1 set b= 'bar' where v > 'a' limit 20; +drop table t1; +# Cover return_top_record() in ha_partition::handle_ordered_index_scan() +create table t1 (x int primary key, b tinytext, v text as (b) virtual) +partition by range columns (x) ( +partition p1 values less than (4), +partition pn values less than (maxvalue)); +insert into t1 (x, b) values (1, ''), (2, ''), (3, 'a'), (4, 'b'); +update t1 set b= 'bar' where x > 0 order by v limit 2; +drop table t1; diff --git a/mysql-test/suite/vcol/t/partition.test b/mysql-test/suite/vcol/t/partition.test index 889724fb1c5..408990b20a6 100644 --- a/mysql-test/suite/vcol/t/partition.test +++ b/mysql-test/suite/vcol/t/partition.test @@ -30,3 +30,51 @@ subpartition by hash(v) subpartitions 3 ( insert t1 set i= 0; set statement sql_mode= '' for update t1 set i= 1, v= 2; drop table t1; + +--echo # +--echo # MDEV-18734 ASAN heap-use-after-free in my_strnxfrm_simple_internal upon update on versioned partitioned table +--echo # +--echo # Cover queue_fix() in ha_partition::handle_ordered_index_scan() +create or replace table t1 ( + x int auto_increment primary key, + b text, v mediumtext as (b) virtual, + index (v(10)) +) partition by range columns (x) ( + partition p1 values less than (3), + partition p2 values less than (6), + partition p3 values less than (9), + partition p4 values less than (12), + partition p5 values less than (15), + partition p6 values less than (17), + partition p7 values less than (19), + partition p8 values less than (21), + partition p9 values less than (23), + partition p10 values less than (25), + partition p11 values less than (27), + partition p12 values less than (29), + partition p13 values less than (31), + partition p14 values less than (33), + partition p15 values less than (35), + partition pn values less than (maxvalue)); +insert into t1 (b) values +(repeat('q', 8192)), (repeat('z', 8192)), (repeat('a', 8192)), (repeat('b', 8192)), +(repeat('x', 8192)), (repeat('y', 8192)); + +insert t1 (b) select b from t1; +insert t1 (b) select b from t1; +insert t1 (b) select b from t1; +insert t1 (b) select b from t1; + +select x, left(b, 10), left(v, 10) from t1 where x > 30 and x < 60 order by v; +update t1 set b= 'bar' where v > 'a' limit 20; + +drop table t1; + +--echo # Cover return_top_record() in ha_partition::handle_ordered_index_scan() +create table t1 (x int primary key, b tinytext, v text as (b) virtual) +partition by range columns (x) ( + partition p1 values less than (4), + partition pn values less than (maxvalue)); +insert into t1 (x, b) values (1, ''), (2, ''), (3, 'a'), (4, 'b'); +update t1 set b= 'bar' where x > 0 order by v limit 2; +drop table t1; diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 562f9dc3aac..67244a7c622 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -1010,7 +1010,13 @@ check_port() lsof -Pnl -i ":$port" 2>/dev/null | \ grep -q -E "^($utils)[^[:space:]]*[[:space:]]+$pid[[:space:]].*\\(LISTEN\\)" && rc=0 elif [ $sockstat_available -ne 0 ]; then - sockstat -p "$port" 2>/dev/null | \ + local opts='-p' + if [ "$OS" = 'FreeBSD' ]; then + # sockstat on FreeBSD requires the "-s" option + # to display the connection state: + opts='-sp' + fi + sockstat "$opts" "$port" 2>/dev/null | \ grep -q -E "[[:space:]]+($utils)[^[:space:]]*[[:space:]]+$pid[[:space:]].*[[:space:]]LISTEN" && rc=0 elif [ $ss_available -ne 0 ]; then ss -nlpH "( sport = :$port )" 2>/dev/null | \ diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 46804c9dce4..562b9b929f2 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -166,7 +166,8 @@ get_keys() fi if [ -z "$ekey" -a ! -r "$ekeyfile" ]; then - wsrep_log_error "FATAL: Either key or keyfile must be readable" + wsrep_log_error "FATAL: Either key must be specified " \ + "or keyfile must be readable" exit 3 fi @@ -448,9 +449,30 @@ encgroups='--mysqld|sst|xtrabackup' check_server_ssl_config() { - tcert=$(parse_cnf "$encgroups" 'ssl-ca') - tpem=$(parse_cnf "$encgroups" 'ssl-cert') - tkey=$(parse_cnf "$encgroups" 'ssl-key') + # backward-compatible behavior: + tcert=$(parse_cnf 'sst' 'tca') + tpem=$(parse_cnf 'sst' 'tcert') + tkey=$(parse_cnf 'sst' 'tkey') + # reading new ssl configuration options: + local tcert2=$(parse_cnf "$encgroups" 'ssl-ca') + local tpem2=$(parse_cnf "$encgroups" 'ssl-cert') + local tkey2=$(parse_cnf "$encgroups" 'ssl-key') + # if there are no old options, then we take new ones: + if [ -z "$tcert" -a -z "$tpem" -a -z "$tkey" ]; then + tcert="$tcert2" + tpem="$tpem2" + tkey="$tkey2" + # checking for presence of the new-style SSL configuration: + elif [ -n "$tcert2" -o -n "$tpem2" -o -n "$tkey2" ]; then + if [ "$tcert" != "$tcert2" -o \ + "$tpem" != "$tpem2" -o \ + "$tkey" != "$tkey2" ] + then + wsrep_log_info "new ssl configuration options (ssl-ca, ssl-cert " \ + "and ssl-key) are ignored by SST due to presence " \ + "of the tca, tcert and/or tkey in the [sst] section" + fi + fi } read_cnf() @@ -463,18 +485,10 @@ read_cnf() if [ $encrypt -eq 0 -o $encrypt -ge 2 ] then - if [ "$tmode" != 'DISABLED' -o $encrypt -ge 2 ] - then - tcert=$(parse_cnf 'sst' 'tca') - tpem=$(parse_cnf 'sst' 'tcert') - tkey=$(parse_cnf 'sst' 'tkey') + if [ "$tmode" != 'DISABLED' -o $encrypt -ge 2 ]; then + check_server_ssl_config fi if [ "$tmode" != 'DISABLED' ]; then - # backward-incompatible behavior - if [ -z "$tpem" -a -z "$tkey" -a -z "$tcert" ]; then - # no old-style SSL config in [sst] - check_server_ssl_config - fi if [ 0 -eq $encrypt -a -n "$tpem" -a -n "$tkey" ] then encrypt=3 # enable cert/key SSL encyption @@ -489,7 +503,11 @@ read_cnf() ealgo=$(parse_cnf "$encgroups" 'encrypt-algo') eformat=$(parse_cnf "$encgroups" 'encrypt-format' 'openssl') ekey=$(parse_cnf "$encgroups" 'encrypt-key') - ekeyfile=$(parse_cnf "$encgroups" 'encrypt-key-file') + # The keyfile should be read only when the key + # is not specified or empty: + if [ -z "$ekey" ]; then + ekeyfile=$(parse_cnf "$encgroups" 'encrypt-key-file') + fi fi wsrep_log_info "SSL configuration: CA='$tcert', CERT='$tpem'," \ diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index fc9f5017937..d90e87b68f2 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -93,7 +93,15 @@ check_pid_and_port() else local filter='([^[:space:]]+[[:space:]]+){4}[^[:space:]]+' if [ $sockstat_available -eq 1 ]; then - port_info=$(sockstat -p "$port" 2>/dev/null | \ + local opts='-p' + if [ "$OS" = 'FreeBSD' ]; then + # sockstat on FreeBSD requires the "-s" option + # to display the connection state: + opts='-sp' + # in addition, sockstat produces an additional column: + filter='([^[:space:]]+[[:space:]]+){5}[^[:space:]]+' + fi + port_info=$(sockstat "$opts" "$port" 2>/dev/null | \ grep -E '[[:space:]]LISTEN' | grep -o -E "$filter") else port_info=$(ss -nlpH "( sport = :$port )" 2>/dev/null | \ @@ -388,7 +396,7 @@ EOF # Use deltaxfer only for WAN inv=$(basename "$0") WHOLE_FILE_OPT="" - if [ "${inv%wsrep_sst_rsync_wan*}" != "$inv" ]; then + if [ "${inv%wsrep_sst_rsync_wan*}" = "$inv" ]; then WHOLE_FILE_OPT="--whole-file" fi diff --git a/sql/field.cc b/sql/field.cc index 383c2bb34aa..eaa829cd361 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -1,6 +1,6 @@ /* Copyright (c) 2000, 2017, Oracle and/or its affiliates. - Copyright (c) 2008, 2020, MariaDB + Copyright (c) 2008, 2021, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -8441,6 +8441,7 @@ int Field_blob::store(const char *from,size_t length,CHARSET_INFO *cs) rc= well_formed_copy_with_check((char*) value.ptr(), (uint) new_length, cs, from, length, length, true, ©_len); + value.length(copy_len); Field_blob::store_length(copy_len); bmove(ptr+packlength,(uchar*) &tmp,sizeof(char*)); diff --git a/sql/field.h b/sql/field.h index a3385736c0a..825a0dae7aa 100644 --- a/sql/field.h +++ b/sql/field.h @@ -3784,6 +3784,12 @@ public: uchar *new_ptr, uint32 length, uchar *new_null_ptr, uint new_null_bit); void sql_type(String &str) const; + /** + Copy blob buffer into internal storage "value" and update record pointer. + + @retval true Memory allocation error + @retval false Success + */ inline bool copy() { uchar *tmp= get_ptr(); @@ -3796,6 +3802,33 @@ public: memcpy(ptr+packlength, &tmp, sizeof(char*)); return 0; } + void swap(String &inout, bool set_read_value) + { + if (set_read_value) + read_value.swap(inout); + else + value.swap(inout); + } + /** + Return pointer to blob cache or NULL if not cached. + */ + String * cached(bool *set_read_value) + { + char *tmp= (char *) get_ptr(); + if (!value.is_empty() && tmp == value.ptr()) + { + *set_read_value= false; + return &value; + } + + if (!read_value.is_empty() && tmp == read_value.ptr()) + { + *set_read_value= true; + return &read_value; + } + + return NULL; + } /* store value for the duration of the current read record */ inline void swap_value_and_read_value() { diff --git a/sql/filesort.cc b/sql/filesort.cc index 823e98b3a8a..98d5bbcb14c 100644 --- a/sql/filesort.cc +++ b/sql/filesort.cc @@ -603,6 +603,15 @@ const char* dbug_print_table_row(TABLE *table) } +const char* dbug_print_row(TABLE *table, uchar *rec) +{ + table->move_fields(table->field, rec, table->record[0]); + const char* ret= dbug_print_table_row(table); + table->move_fields(table->field, table->record[0], rec); + return ret; +} + + /* Print a text, SQL-like record representation into dbug trace. diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index 9fa4d448639..bae15892ba3 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -5248,58 +5248,68 @@ bool ha_partition::init_record_priority_queue() /* Initialize the ordered record buffer. */ - if (!m_ordered_rec_buffer) - { - size_t alloc_len; - uint used_parts= bitmap_bits_set(&m_part_info->read_partitions); - - if (used_parts == 0) /* Do nothing since no records expected. */ - DBUG_RETURN(false); + size_t alloc_len; + uint used_parts= bitmap_bits_set(&m_part_info->read_partitions); - /* Allocate record buffer for each used partition. */ - m_priority_queue_rec_len= m_rec_length + PARTITION_BYTES_IN_POS; - if (!m_using_extended_keys) - m_priority_queue_rec_len += get_open_file_sample()->ref_length; - alloc_len= used_parts * m_priority_queue_rec_len; - /* Allocate a key for temporary use when setting up the scan. */ - alloc_len+= table_share->max_key_length; + if (used_parts == 0) /* Do nothing since no records expected. */ + DBUG_RETURN(false); - if (!(m_ordered_rec_buffer= (uchar*)my_malloc(alloc_len, MYF(MY_WME)))) - DBUG_RETURN(true); + /* Allocate record buffer for each used partition. */ + m_priority_queue_rec_len= m_rec_length + ORDERED_REC_OFFSET; + if (!m_using_extended_keys) + m_priority_queue_rec_len+= get_open_file_sample()->ref_length; + alloc_len= used_parts * m_priority_queue_rec_len; + /* Allocate a key for temporary use when setting up the scan. */ + alloc_len+= table_share->max_key_length; + Ordered_blob_storage **blob_storage; + Ordered_blob_storage *objs; + const size_t n_all= used_parts * table->s->blob_fields; + + if (!my_multi_malloc(MYF(MY_WME), &m_ordered_rec_buffer, alloc_len, + &blob_storage, n_all * sizeof *blob_storage, + &objs, n_all * sizeof *objs, NULL)) + DBUG_RETURN(true); - /* - We set-up one record per partition and each record has 2 bytes in - front where the partition id is written. This is used by ordered - index_read. - We also set-up a reference to the first record for temporary use in - setting up the scan. - */ - char *ptr= (char*) m_ordered_rec_buffer; - uint i; - for (i= bitmap_get_first_set(&m_part_info->read_partitions); - i < m_tot_parts; - i= bitmap_get_next_set(&m_part_info->read_partitions, i)) + /* + We set-up one record per partition and each record has 2 bytes in + front where the partition id is written. This is used by ordered + index_read. + We also set-up a reference to the first record for temporary use in + setting up the scan. + */ + char *ptr= (char*) m_ordered_rec_buffer; + uint i; + for (i= bitmap_get_first_set(&m_part_info->read_partitions); + i < m_tot_parts; + i= bitmap_get_next_set(&m_part_info->read_partitions, i)) + { + DBUG_PRINT("info", ("init rec-buf for part %u", i)); + if (table->s->blob_fields) { - DBUG_PRINT("info", ("init rec-buf for part %u", i)); - int2store(ptr, i); - ptr+= m_priority_queue_rec_len; + for (uint j= 0; j < table->s->blob_fields; ++j, ++objs) + blob_storage[j]= new (objs) Ordered_blob_storage; + *((Ordered_blob_storage ***) ptr)= blob_storage; + blob_storage+= table->s->blob_fields; } - m_start_key.key= (const uchar*)ptr; + int2store(ptr + sizeof(String **), i); + ptr+= m_priority_queue_rec_len; + } + m_start_key.key= (const uchar*)ptr; - /* Initialize priority queue, initialized to reading forward. */ - int (*cmp_func)(void *, uchar *, uchar *); - void *cmp_arg= (void*) this; - if (!m_using_extended_keys && !(table_flags() & HA_CMP_REF_IS_EXPENSIVE)) - cmp_func= cmp_key_rowid_part_id; - else - cmp_func= cmp_key_part_id; - DBUG_PRINT("info", ("partition queue_init(1) used_parts: %u", used_parts)); - if (init_queue(&m_queue, used_parts, 0, 0, cmp_func, cmp_arg, 0, 0)) - { - my_free(m_ordered_rec_buffer); - m_ordered_rec_buffer= NULL; - DBUG_RETURN(true); - } + /* Initialize priority queue, initialized to reading forward. */ + int (*cmp_func)(void *, uchar *, uchar *); + void *cmp_arg= (void*) this; + if (!m_using_extended_keys && !(table_flags() & HA_CMP_REF_IS_EXPENSIVE)) + cmp_func= cmp_key_rowid_part_id; + else + cmp_func= cmp_key_part_id; + DBUG_PRINT("info", ("partition queue_init(1) used_parts: %u", used_parts)); + if (init_queue(&m_queue, used_parts, ORDERED_PART_NUM_OFFSET, + 0, cmp_func, cmp_arg, 0, 0)) + { + my_free(m_ordered_rec_buffer); + m_ordered_rec_buffer= NULL; + DBUG_RETURN(true); } DBUG_RETURN(false); } @@ -5314,6 +5324,20 @@ void ha_partition::destroy_record_priority_queue() DBUG_ENTER("ha_partition::destroy_record_priority_queue"); if (m_ordered_rec_buffer) { + if (table->s->blob_fields) + { + char *ptr= (char *) m_ordered_rec_buffer; + for (uint i= bitmap_get_first_set(&m_part_info->read_partitions); + i < m_tot_parts; + i= bitmap_get_next_set(&m_part_info->read_partitions, i)) + { + Ordered_blob_storage **blob_storage= *((Ordered_blob_storage ***) ptr); + for (uint b= 0; b < table->s->blob_fields; ++b) + blob_storage[b]->blob.free(); + ptr+= m_priority_queue_rec_len; + } + } + delete_queue(&m_queue); my_free(m_ordered_rec_buffer); m_ordered_rec_buffer= NULL; @@ -5541,12 +5565,10 @@ static int cmp_part_ids(uchar *ref1, uchar *ref2) extern "C" int cmp_key_part_id(void *ptr, uchar *ref1, uchar *ref2) { ha_partition *file= (ha_partition*)ptr; - int res; - if ((res= key_rec_cmp(file->m_curr_key_info, ref1 + PARTITION_BYTES_IN_POS, - ref2 + PARTITION_BYTES_IN_POS))) - { + if (int res= key_rec_cmp(file->m_curr_key_info, + ref1 + PARTITION_BYTES_IN_POS, + ref2 + PARTITION_BYTES_IN_POS)) return res; - } return cmp_part_ids(ref1, ref2); } @@ -6801,6 +6823,48 @@ int ha_partition::pre_ft_end() } +void ha_partition::swap_blobs(uchar * rec_buf, Ordered_blob_storage ** storage, bool restore) +{ + uint *ptr, *end; + uint blob_n= 0; + table->move_fields(table->field, rec_buf, table->record[0]); + for (ptr= table->s->blob_field, end= ptr + table->s->blob_fields; + ptr != end; ++ptr, ++blob_n) + { + DBUG_ASSERT(*ptr < table->s->fields); + Field_blob *blob= (Field_blob*) table->field[*ptr]; + DBUG_ASSERT(blob->flags & BLOB_FLAG); + DBUG_ASSERT(blob->field_index == *ptr); + if (!bitmap_is_set(table->read_set, *ptr) || blob->is_null()) + continue; + + Ordered_blob_storage &s= *storage[blob_n]; + + if (restore) + { + /* + We protect only blob cache (value or read_value). If the cache was + empty that doesn't mean the blob was empty. Blobs allocated by a + storage engine should work just fine. + */ + if (!s.blob.is_empty()) + blob->swap(s.blob, s.set_read_value); + } + else + { + bool set_read_value; + String *cached= blob->cached(&set_read_value); + if (cached) + { + cached->swap(s.blob); + s.set_read_value= set_read_value; + } + } + } + table->move_fields(table->field, table->record[0], rec_buf); +} + + /** Initialize a full text search using the extended API. @@ -7514,8 +7578,8 @@ int ha_partition::handle_ordered_index_scan(uchar *buf, bool reverse_order) { DBUG_PRINT("info", ("reading from part %u (scan_type: %u)", i, m_index_scan_type)); - DBUG_ASSERT(i == uint2korr(part_rec_buf_ptr)); - uchar *rec_buf_ptr= part_rec_buf_ptr + PARTITION_BYTES_IN_POS; + DBUG_ASSERT(i == uint2korr(part_rec_buf_ptr + ORDERED_PART_NUM_OFFSET)); + uchar *rec_buf_ptr= part_rec_buf_ptr + ORDERED_REC_OFFSET; handler *file= m_file[i]; switch (m_index_scan_type) { @@ -7595,6 +7659,12 @@ int ha_partition::handle_ordered_index_scan(uchar *buf, bool reverse_order) Initialize queue without order first, simply insert */ queue_element(&m_queue, j++)= part_rec_buf_ptr; + if (table->s->blob_fields) + { + Ordered_blob_storage **storage= + *((Ordered_blob_storage ***) part_rec_buf_ptr); + swap_blobs(rec_buf_ptr, storage, false); + } } else if (error == HA_ERR_KEY_NOT_FOUND) { @@ -7637,7 +7707,7 @@ int ha_partition::handle_ordered_index_scan(uchar *buf, bool reverse_order) DBUG_PRINT("info", ("partition !bitmap_is_set(&m_mrr_used_partitions, i)")); continue; } - DBUG_ASSERT(i == uint2korr(part_rec_buf_ptr)); + DBUG_ASSERT(i == uint2korr(part_rec_buf_ptr + ORDERED_PART_NUM_OFFSET)); if (smallest_range_seq == m_stock_range_seq[i]) { m_stock_range_seq[i]= 0; @@ -7684,12 +7754,17 @@ void ha_partition::return_top_record(uchar *buf) { uint part_id; uchar *key_buffer= queue_top(&m_queue); - uchar *rec_buffer= key_buffer + PARTITION_BYTES_IN_POS; + uchar *rec_buffer= key_buffer + ORDERED_REC_OFFSET; DBUG_ENTER("ha_partition::return_top_record"); DBUG_PRINT("enter", ("partition this: %p", this)); - part_id= uint2korr(key_buffer); + part_id= uint2korr(key_buffer + ORDERED_PART_NUM_OFFSET); memcpy(buf, rec_buffer, m_rec_length); + if (table->s->blob_fields) + { + Ordered_blob_storage **storage= *((Ordered_blob_storage ***) key_buffer); + swap_blobs(buf, storage, true); + } m_last_part= part_id; DBUG_PRINT("info", ("partition m_last_part: %u", m_last_part)); m_top_entry= part_id; @@ -7741,7 +7816,7 @@ int ha_partition::handle_ordered_index_scan_key_not_found() This partition is used and did return HA_ERR_KEY_NOT_FOUND in index_read_map. */ - curr_rec_buf= part_buf + PARTITION_BYTES_IN_POS; + curr_rec_buf= part_buf + ORDERED_REC_OFFSET; error= m_file[i]->ha_index_next(curr_rec_buf); /* HA_ERR_KEY_NOT_FOUND is not allowed from index_next! */ DBUG_ASSERT(error != HA_ERR_KEY_NOT_FOUND); @@ -7792,7 +7867,8 @@ int ha_partition::handle_ordered_next(uchar *buf, bool is_next_same) DBUG_RETURN(HA_ERR_END_OF_FILE); uint part_id= m_top_entry; - uchar *rec_buf= queue_top(&m_queue) + PARTITION_BYTES_IN_POS; + uchar *part_rec_buf_ptr= queue_top(&m_queue); + uchar *rec_buf= part_rec_buf_ptr + ORDERED_REC_OFFSET; handler *file; if (m_key_not_found) @@ -7834,7 +7910,16 @@ int ha_partition::handle_ordered_next(uchar *buf, bool is_next_same) if (m_index_scan_type == partition_read_range) { error= file->read_range_next(); - memcpy(rec_buf, table->record[0], m_rec_length); + if (likely(!error)) + { + memcpy(rec_buf, table->record[0], m_rec_length); + if (table->s->blob_fields) + { + Ordered_blob_storage **storage= + *((Ordered_blob_storage ***) part_rec_buf_ptr); + swap_blobs(rec_buf, storage, false); + } + } } else if (m_index_scan_type == partition_read_multi_range) { @@ -7921,9 +8006,8 @@ int ha_partition::handle_ordered_next(uchar *buf, bool is_next_same) DBUG_PRINT("info",("partition !bitmap_is_set(&m_mrr_used_partitions, i)")); continue; } - DBUG_PRINT("info",("partition uint2korr: %u", - uint2korr(part_rec_buf_ptr))); - DBUG_ASSERT(i == uint2korr(part_rec_buf_ptr)); + DBUG_ASSERT(i == uint2korr(part_rec_buf_ptr + + ORDERED_PART_NUM_OFFSET)); DBUG_PRINT("info", ("partition m_stock_range_seq[%u]: %u", i, m_stock_range_seq[i])); if (smallest_range_seq == m_stock_range_seq[i]) @@ -8012,7 +8096,7 @@ int ha_partition::handle_ordered_prev(uchar *buf) DBUG_RETURN(HA_ERR_END_OF_FILE); uint part_id= m_top_entry; - uchar *rec_buf= queue_top(&m_queue) + PARTITION_BYTES_IN_POS; + uchar *rec_buf= queue_top(&m_queue) + ORDERED_REC_OFFSET; handler *file= m_file[part_id]; if (unlikely((error= file->ha_index_prev(rec_buf)))) diff --git a/sql/ha_partition.h b/sql/ha_partition.h index 3f6dc3254ab..48552301f55 100644 --- a/sql/ha_partition.h +++ b/sql/ha_partition.h @@ -21,7 +21,17 @@ #include "sql_partition.h" /* part_id_range, partition_element */ #include "queues.h" /* QUEUE */ +struct Ordered_blob_storage +{ + String blob; + bool set_read_value; + Ordered_blob_storage() : set_read_value(false) + {} +}; + #define PARTITION_BYTES_IN_POS 2 +#define ORDERED_PART_NUM_OFFSET sizeof(Ordered_blob_storage **) +#define ORDERED_REC_OFFSET (ORDERED_PART_NUM_OFFSET + PARTITION_BYTES_IN_POS) /** Struct used for partition_name_hash */ @@ -925,6 +935,7 @@ private: int handle_ordered_next(uchar * buf, bool next_same); int handle_ordered_prev(uchar * buf); void return_top_record(uchar * buf); + void swap_blobs(uchar* rec_buf, Ordered_blob_storage ** storage, bool restore); public: /* ------------------------------------------------------------------------- diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 10be3187072..70d4604b5d2 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright (c) 2006, 2017, Oracle and/or its affiliates. All rights reserved. -# Copyright (c) 2014, 2019, MariaDB Corporation. +# Copyright (c) 2014, 2021, MariaDB Corporation. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index 2842f9a6bc0..bc85a955f80 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -3167,7 +3167,7 @@ func_exit: @param[in,out] page page to remove @param[in] index index tree @param[in,out] mtr mini-transaction */ -void +dberr_t btr_level_list_remove_func( ulint space, const page_size_t& page_size, @@ -3210,6 +3210,10 @@ btr_level_list_remove_func( page_id_t(space, next_page_no), page_size, RW_X_LATCH, index, mtr); + if (!next_block) { + return DB_ERROR; + } + page_t* next_page = buf_block_get_frame(next_block); #ifdef UNIV_BTR_DEBUG @@ -3222,6 +3226,8 @@ btr_level_list_remove_func( buf_block_get_page_zip(next_block), prev_page_no, mtr); } + + return DB_SUCCESS; } /****************************************************************//** @@ -3712,8 +3718,11 @@ retry: btr_search_drop_page_hash_index(block); /* Remove the page from the level list */ - btr_level_list_remove(index->table->space_id, - page_size, page, index, mtr); + if (DB_SUCCESS != btr_level_list_remove(index->table->space_id, + page_size, page, index, + mtr)) { + goto err_exit; + } if (dict_index_is_spatial(index)) { rec_t* my_rec = father_cursor.page_cur.rec; @@ -3842,8 +3851,10 @@ retry: #endif /* UNIV_BTR_DEBUG */ /* Remove the page from the level list */ - btr_level_list_remove(index->table->space_id, - page_size, page, index, mtr); + if (DB_SUCCESS != btr_level_list_remove(index->table->space_id, + page_size, page, index, mtr)) { + goto err_exit; + } ut_ad(btr_node_ptr_get_child_page_no( btr_cur_get_rec(&father_cursor), offsets) @@ -4222,8 +4233,8 @@ btr_discard_page( } /* Remove the page from the level list */ - btr_level_list_remove(index->table->space_id, page_size, - page, index, mtr); + ut_a(DB_SUCCESS == btr_level_list_remove(index->table->space_id, + page_size, page, index, mtr)); #ifdef UNIV_ZIP_DEBUG { diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 66e680ae494..62611c7798a 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -334,10 +334,12 @@ btr_cur_latch_leaves( page_size, RW_X_LATCH, cursor->index, mtr); latch_leaves.blocks[2] = get_block; #ifdef UNIV_BTR_DEBUG - ut_a(page_is_comp(get_block->frame) - == page_is_comp(page)); - ut_a(btr_page_get_prev(get_block->frame) - == page_get_page_no(page)); + if (get_block) { + ut_a(page_is_comp(get_block->frame) + == page_is_comp(page)); + ut_a(btr_page_get_prev(get_block->frame) + == page_get_page_no(page)); + } #endif /* UNIV_BTR_DEBUG */ if (spatial) { cursor->rtr_info->tree_blocks[ diff --git a/storage/innobase/btr/btr0defragment.cc b/storage/innobase/btr/btr0defragment.cc index 99b15965281..08a110c55af 100644 --- a/storage/innobase/btr/btr0defragment.cc +++ b/storage/innobase/btr/btr0defragment.cc @@ -486,9 +486,10 @@ btr_defragment_merge_pages( lock_update_merge_left(to_block, orig_pred, from_block); btr_search_drop_page_hash_index(from_block); - btr_level_list_remove( + + ut_a(DB_SUCCESS == btr_level_list_remove( index->table->space_id, - page_size, from_page, index, mtr); + page_size, from_page, index, mtr)); btr_page_get_father(index, from_block, mtr, &parent); btr_cur_node_ptr_delete(&parent, mtr); /* btr_blob_dbg_remove(from_page, index, diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 647a5a96037..47382bf7b1d 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -4430,6 +4430,10 @@ loop: } } + if (local_err == DB_IO_ERROR) { + return NULL; + } + ib::fatal() << "Unable to read page " << page_id << " into the buffer pool after " << BUF_PAGE_READ_MAX_RETRIES diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index b0eb56972fc..56817b59c36 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -201,7 +201,8 @@ buf_read_page_low( } return(0); } else if (IORequest::ignore_missing(type) - || *err == DB_TABLESPACE_DELETED) { + || *err == DB_TABLESPACE_DELETED + || *err == DB_IO_ERROR) { buf_read_page_handle_error(bpage); return(0); } diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 9f9ed51a07f..16069351651 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -4114,27 +4114,30 @@ fil_node_complete_io(fil_node_t* node, const IORequest& type) } } -/** Report information about an invalid page access. */ -static -void -fil_report_invalid_page_access( - ulint block_offset, /*!< in: block offset */ - ulint space_id, /*!< in: space id */ - const char* space_name, /*!< in: space name */ - ulint byte_offset, /*!< in: byte offset */ - ulint len, /*!< in: I/O length */ - bool is_read) /*!< in: I/O type */ +/** Compose error message about an invalid page access. +@param[in] block_offset block offset +@param[in] space_id space id +@param[in] space_name space name +@param[in] byte_offset byte offset +@param[in] len I/O length +@param[in] is_read I/O type +@return std::string with error message */ +static std::string fil_invalid_page_access_msg(size_t block_offset, + size_t space_id, + const char *space_name, + size_t byte_offset, size_t len, + bool is_read) { - ib::fatal() - << "Trying to " << (is_read ? "read" : "write") - << " page number " << block_offset << " in" - " space " << space_id << ", space name " << space_name << "," - " which is outside the tablespace bounds. Byte offset " - << byte_offset << ", len " << len << - (space_id == 0 && !srv_was_started - ? "Please check that the configuration matches" - " the InnoDB system tablespace location (ibdata files)" - : ""); + std::stringstream ss; + ss << "Trying to " << (is_read ? "read" : "write") << " page number " + << block_offset << " in space " << space_id << ", space name " + << space_name << ", which is outside the tablespace bounds. Byte offset " + << byte_offset << ", len " << len + << (space_id == 0 && !srv_was_started + ? "Please check that the configuration matches" + " the InnoDB system tablespace location (ibdata files)" + : ""); + return ss.str(); } /** Reads or writes data. This operation could be asynchronous (aio). @@ -4269,7 +4272,17 @@ fil_io( return(DB_ERROR); } - fil_report_invalid_page_access( + if (space->purpose == FIL_TYPE_IMPORT) { + mutex_exit(&fil_system.mutex); + ib::error() << fil_invalid_page_access_msg( + page_id.page_no(), page_id.space(), + space->name, byte_offset, len, + req_type.is_read()); + + return DB_IO_ERROR; + } + + ib::fatal() << fil_invalid_page_access_msg( page_id.page_no(), page_id.space(), space->name, byte_offset, len, req_type.is_read()); @@ -4349,7 +4362,7 @@ fil_io( return(DB_ERROR); } - fil_report_invalid_page_access( + ib::fatal() << fil_invalid_page_access_msg( page_id.page_no(), page_id.space(), space->name, byte_offset, len, req_type.is_read()); } diff --git a/storage/innobase/fts/fts0config.cc b/storage/innobase/fts/fts0config.cc index 9f8e07285bf..1170519b543 100644 --- a/storage/innobase/fts/fts0config.cc +++ b/storage/innobase/fts/fts0config.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2020, MariaDB Corporation. +Copyright (c) 2017, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -97,7 +97,7 @@ fts_config_get_value( fts_table->suffix = "CONFIG"; fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, true, "table_name", table_name); + pars_info_bind_id(info, "table_name", table_name); graph = fts_parse_sql( fts_table, @@ -217,7 +217,7 @@ fts_config_set_value( fts_table->suffix = "CONFIG"; fts_get_table_name(fts_table, table_name, dict_locked); - pars_info_bind_id(info, true, "table_name", table_name); + pars_info_bind_id(info, "table_name", table_name); graph = fts_parse_sql( fts_table, info, @@ -245,7 +245,7 @@ fts_config_set_value( info, "value", value->f_str, value->f_len); fts_get_table_name(fts_table, table_name, dict_locked); - pars_info_bind_id(info, true, "table_name", table_name); + pars_info_bind_id(info, "table_name", table_name); graph = fts_parse_sql( fts_table, info, diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index ec304df8fc1..cfbc257ef90 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -483,7 +483,7 @@ cleanup: pars_info_t* info = pars_info_create(); - pars_info_bind_id(info, TRUE, "table_stopword", stopword_table_name); + pars_info_bind_id(info, "table_stopword", stopword_table_name); pars_info_bind_function(info, "my_func", fts_read_stopword, stopword_info); @@ -1912,7 +1912,7 @@ fts_create_common_tables( fts_table.suffix = "CONFIG"; fts_get_table_name(&fts_table, fts_name, true); - pars_info_bind_id(info, true, "config_table", fts_name); + pars_info_bind_id(info, "config_table", fts_name); graph = fts_parse_sql_no_dict_lock( info, fts_config_table_insert_values_sql); @@ -2640,7 +2640,7 @@ retry: info, "my_func", fts_fetch_store_doc_id, doc_id); fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, true, "config_table", table_name); + pars_info_bind_id(info, "config_table", table_name); graph = fts_parse_sql( &fts_table, info, @@ -2768,7 +2768,7 @@ fts_update_sync_doc_id( fts_get_table_name(&fts_table, fts_name, table->fts->dict_locked); - pars_info_bind_id(info, true, "table_name", fts_name); + pars_info_bind_id(info, "table_name", fts_name); graph = fts_parse_sql( &fts_table, info, @@ -2911,7 +2911,7 @@ fts_delete( fts_table.suffix = "DELETED"; fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, true, "deleted", table_name); + pars_info_bind_id(info, "deleted", table_name); graph = fts_parse_sql( &fts_table, @@ -3743,7 +3743,7 @@ fts_doc_fetch_by_doc_id( pars_info_bind_function(info, "my_func", callback, arg); select_str = fts_get_select_columns_str(index, info, info->heap); - pars_info_bind_id(info, TRUE, "table_name", index->table->name.m_name); + pars_info_bind_id(info, "table_name", index->table->name.m_name); if (!get_doc || !get_doc->get_document_graph) { if (option == FTS_FETCH_DOC_BY_ID_EQUAL) { @@ -3850,7 +3850,7 @@ fts_write_node( info = pars_info_create(); fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, true, "index_table_name", table_name); + pars_info_bind_id(info, "index_table_name", table_name); } pars_info_bind_varchar_literal(info, "token", word->f_str, word->f_len); @@ -3925,7 +3925,7 @@ fts_sync_add_deleted_cache( &fts_table, "DELETED_CACHE", FTS_COMMON_TABLE, sync->table); fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, true, "table_name", table_name); + pars_info_bind_id(info, "table_name", table_name); graph = fts_parse_sql( &fts_table, @@ -4930,7 +4930,7 @@ fts_get_rows_count( pars_info_bind_function(info, "my_func", fts_read_ulint, &count); fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, true, "table_name", table_name); + pars_info_bind_id(info, "table_name", table_name); graph = fts_parse_sql( fts_table, diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index abb46eb7f4b..893a98aa11e 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2018, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, 2020, MariaDB Corporation. +Copyright (c) 2016, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -492,7 +492,7 @@ fts_index_fetch_nodes( fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, true, "table_name", table_name); + pars_info_bind_id(info, "table_name", table_name); } pars_info_bind_function(info, "my_func", fetch->read_record, fetch); @@ -821,7 +821,7 @@ fts_index_fetch_words( info, "word", word->f_str, word->f_len); fts_get_table_name(&optim->fts_index_table, table_name); - pars_info_bind_id(info, true, "table_name", table_name); + pars_info_bind_id(info, "table_name", table_name); graph = fts_parse_sql( &optim->fts_index_table, @@ -977,7 +977,7 @@ fts_table_fetch_doc_ids( pars_info_bind_function(info, "my_func", fts_fetch_doc_ids, doc_ids); fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, true, "table_name", table_name); + pars_info_bind_id(info, "table_name", table_name); graph = fts_parse_sql( fts_table, @@ -1441,7 +1441,7 @@ fts_optimize_write_word( fts_table->suffix = fts_get_suffix(selected); fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, true, "table_name", table_name); + pars_info_bind_id(info, "table_name", table_name); graph = fts_parse_sql( fts_table, @@ -2033,11 +2033,11 @@ fts_optimize_purge_deleted_doc_ids( used in the fts_delete_doc_ids_sql */ optim->fts_common_table.suffix = fts_common_tables[3]; fts_get_table_name(&optim->fts_common_table, deleted); - pars_info_bind_id(info, true, fts_common_tables[3], deleted); + pars_info_bind_id(info, fts_common_tables[3], deleted); optim->fts_common_table.suffix = fts_common_tables[4]; fts_get_table_name(&optim->fts_common_table, deleted_cache); - pars_info_bind_id(info, true, fts_common_tables[4], deleted_cache); + pars_info_bind_id(info, fts_common_tables[4], deleted_cache); graph = fts_parse_sql(NULL, info, fts_delete_doc_ids_sql); @@ -2090,12 +2090,11 @@ fts_optimize_purge_deleted_doc_id_snapshot( used in the fts_end_delete_sql */ optim->fts_common_table.suffix = fts_common_tables[0]; fts_get_table_name(&optim->fts_common_table, being_deleted); - pars_info_bind_id(info, true, fts_common_tables[0], being_deleted); + pars_info_bind_id(info, fts_common_tables[0], being_deleted); optim->fts_common_table.suffix = fts_common_tables[1]; fts_get_table_name(&optim->fts_common_table, being_deleted_cache); - pars_info_bind_id(info, true, fts_common_tables[1], - being_deleted_cache); + pars_info_bind_id(info, fts_common_tables[1], being_deleted_cache); /* Delete the doc ids that were copied to delete pending state at the start of optimize. */ @@ -2151,20 +2150,19 @@ fts_optimize_create_deleted_doc_id_snapshot( used in the fts_init_delete_sql */ optim->fts_common_table.suffix = fts_common_tables[0]; fts_get_table_name(&optim->fts_common_table, being_deleted); - pars_info_bind_id(info, true, fts_common_tables[0], being_deleted); + pars_info_bind_id(info, fts_common_tables[0], being_deleted); optim->fts_common_table.suffix = fts_common_tables[3]; fts_get_table_name(&optim->fts_common_table, deleted); - pars_info_bind_id(info, true, fts_common_tables[3], deleted); + pars_info_bind_id(info, fts_common_tables[3], deleted); optim->fts_common_table.suffix = fts_common_tables[1]; fts_get_table_name(&optim->fts_common_table, being_deleted_cache); - pars_info_bind_id(info, true, fts_common_tables[1], - being_deleted_cache); + pars_info_bind_id(info, fts_common_tables[1], being_deleted_cache); optim->fts_common_table.suffix = fts_common_tables[4]; fts_get_table_name(&optim->fts_common_table, deleted_cache); - pars_info_bind_id(info, true, fts_common_tables[4], deleted_cache); + pars_info_bind_id(info, fts_common_tables[4], deleted_cache); /* Move doc_ids that are to be deleted to state being deleted. */ graph = fts_parse_sql(NULL, info, fts_init_delete_sql); diff --git a/storage/innobase/fts/fts0que.cc b/storage/innobase/fts/fts0que.cc index b3ef795a463..5047e99f4d7 100644 --- a/storage/innobase/fts/fts0que.cc +++ b/storage/innobase/fts/fts0que.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2020, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2020, MariaDB Corporation. +Copyright (c) 2017, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -2146,7 +2146,7 @@ fts_query_find_term( query->fts_index_table.suffix = fts_get_suffix(selected); fts_get_table_name(&query->fts_index_table, table_name); - pars_info_bind_id(info, true, "index_table_name", table_name); + pars_info_bind_id(info, "index_table_name", table_name); } select.found = FALSE; @@ -2286,7 +2286,7 @@ fts_query_total_docs_containing_term( fts_get_table_name(&query->fts_index_table, table_name); - pars_info_bind_id(info, true, "index_table_name", table_name); + pars_info_bind_id(info, "index_table_name", table_name); graph = fts_parse_sql( &query->fts_index_table, @@ -2369,7 +2369,7 @@ fts_query_terms_in_document( fts_get_table_name(&query->fts_index_table, table_name); - pars_info_bind_id(info, true, "index_table_name", table_name); + pars_info_bind_id(info, "index_table_name", table_name); graph = fts_parse_sql( &query->fts_index_table, diff --git a/storage/innobase/fts/fts0sql.cc b/storage/innobase/fts/fts0sql.cc index f799a46d088..ff9c8ebc5e8 100644 --- a/storage/innobase/fts/fts0sql.cc +++ b/storage/innobase/fts/fts0sql.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2019, MariaDB Corporation. +Copyright (c) 2019, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -248,7 +248,7 @@ fts_get_select_columns_str( sel_str = mem_heap_printf(heap, "sel%lu", (ulong) i); /* Set copy_name to TRUE since it's dynamic. */ - pars_info_bind_id(info, TRUE, sel_str, field->name); + pars_info_bind_id(info, sel_str, field->name); str = mem_heap_printf( heap, "%s%s$%s", str, (*str) ? ", " : "", sel_str); diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 206756ec09b..87d57457c00 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2020, MariaDB Corporation. +Copyright (c) 2014, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -3435,7 +3435,7 @@ i_s_fts_index_table_fill_selected( FTS_INIT_INDEX_TABLE(&fts_table, fts_get_suffix(selected), FTS_INDEX_TABLE, index); fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, true, "table_name", table_name); + pars_info_bind_id(info, "table_name", table_name); graph = fts_parse_sql( &fts_table, info, diff --git a/storage/innobase/include/btr0btr.h b/storage/innobase/include/btr0btr.h index 07ec5357822..d4818ef241c 100644 --- a/storage/innobase/include/btr0btr.h +++ b/storage/innobase/include/btr0btr.h @@ -757,7 +757,7 @@ btr_validate_index( /*************************************************************//** Removes a page from the level list of pages. */ UNIV_INTERN -void +MY_ATTRIBUTE((warn_unused_result)) dberr_t btr_level_list_remove_func( /*=======================*/ ulint space, /*!< in: space where removed */ diff --git a/storage/innobase/include/pars0pars.h b/storage/innobase/include/pars0pars.h index 73a19dd24d8..475dd523d38 100644 --- a/storage/innobase/include/pars0pars.h +++ b/storage/innobase/include/pars0pars.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2019, MariaDB Corporation. +Copyright (c) 2017, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -491,7 +491,6 @@ void pars_info_bind_id( /*=============*/ pars_info_t* info, /*!< in: info struct */ - ibool copy_name,/* in: make a copy of name if TRUE */ const char* name, /*!< in: name */ const char* id); /*!< in: id */ /****************************************************************//** @@ -538,15 +537,6 @@ pars_info_bind_ull_literal( MY_ATTRIBUTE((nonnull)); /****************************************************************//** -Add bound id. */ -void -pars_info_add_id( -/*=============*/ - pars_info_t* info, /*!< in: info struct */ - const char* name, /*!< in: name */ - const char* id); /*!< in: id */ - -/****************************************************************//** Get bound literal with the given name. @return bound literal, or NULL if not found */ pars_bound_lit_t* diff --git a/storage/innobase/pars/pars0pars.cc b/storage/innobase/pars/pars0pars.cc index 6a187787c12..7ac42ec8f44 100644 --- a/storage/innobase/pars/pars0pars.cc +++ b/storage/innobase/pars/pars0pars.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2018, 2019, MariaDB Corporation. +Copyright (c) 2018, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -2356,7 +2356,6 @@ void pars_info_bind_id( /*==============*/ pars_info_t* info, /*!< in: info struct */ - ibool copy_name, /* in: copy name if TRUE */ const char* name, /*!< in: name */ const char* id) /*!< in: id */ { @@ -2379,8 +2378,7 @@ pars_info_bind_id( bid = static_cast<pars_bound_id_t*>( ib_vector_push(info->bound_ids, NULL)); - bid->name = (copy_name) - ? mem_heap_strdup(info->heap, name) : name; + bid->name = name; } bid->id = id; diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index d3ca5579d17..1652f162e23 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -222,6 +222,19 @@ struct row_import { found and was readable */ }; +struct fil_iterator_t { + pfs_os_file_t file; /*!< File handle */ + const char* filepath; /*!< File path name */ + os_offset_t start; /*!< From where to start */ + os_offset_t end; /*!< Where to stop */ + os_offset_t file_size; /*!< File size in bytes */ + ulint n_io_buffers; /*!< Number of pages to use + for IO */ + byte* io_buffer; /*!< Buffer to use for IO */ + fil_space_crypt_t *crypt_data; /*!< Crypt data (if encrypted) */ + byte* crypt_io_buffer; /*!< IO buffer when encrypted */ +}; + /** Use the page cursor to iterate over records in a block. */ class RecIterator { public: @@ -431,6 +444,10 @@ public: ? block->page.zip.data : block->frame; } + /** Invoke the functionality for the callback */ + virtual dberr_t run(const fil_iterator_t& iter, + buf_block_t* block) UNIV_NOTHROW = 0; + protected: /** Get the physical offset of the extent descriptor within the page. @param page_no page number of the extent descriptor @@ -592,6 +609,24 @@ AbstractCallback::init( } /** +TODO: This can be made parallel trivially by chunking up the file +and creating a callback per thread.. Main benefit will be to use +multiple CPUs for checksums and compressed tables. We have to do +compressed tables block by block right now. Secondly we need to +decompress/compress and copy too much of data. These are +CPU intensive. + +Iterate over all the pages in the tablespace. +@param iter - Tablespace iterator +@param block - block to use for IO +@param callback - Callback to inspect and update page contents +@retval DB_SUCCESS or error code */ +static dberr_t fil_iterate( + const fil_iterator_t& iter, + buf_block_t* block, + AbstractCallback& callback); + +/** Try and determine the index root pages by checking if the next/prev pointers are both FIL_NULL. We need to ensure that skip deleted pages. */ struct FetchIndexRootPages : public AbstractCallback { @@ -608,19 +643,24 @@ struct FetchIndexRootPages : public AbstractCallback { ulint m_page_no; /*!< Root page number */ }; - typedef std::vector<Index, ut_allocator<Index> > Indexes; - /** Constructor @param trx covering (user) transaction @param table table definition in server .*/ FetchIndexRootPages(const dict_table_t* table, trx_t* trx) : AbstractCallback(trx, ULINT_UNDEFINED), - m_table(table) UNIV_NOTHROW { } + m_table(table), m_index(0, 0) UNIV_NOTHROW { } /** Destructor */ virtual ~FetchIndexRootPages() UNIV_NOTHROW { } + /** Fetch the clustered index root page in the tablespace + @param iter Tablespace iterator + @param block Block to use for IO + @retval DB_SUCCESS or error code */ + dberr_t run(const fil_iterator_t& iter, + buf_block_t* block) UNIV_NOTHROW; + /** Called for each block as it is read from the file. @param block block to convert, it is not from the buffer pool. @retval DB_SUCCESS or error code. */ @@ -634,7 +674,7 @@ struct FetchIndexRootPages : public AbstractCallback { const dict_table_t* m_table; /** Index information */ - Indexes m_indexes; + Index m_index; }; /** Called for each block as it is read from the file. Check index pages to @@ -649,31 +689,21 @@ dberr_t FetchIndexRootPages::operator()(buf_block_t* block) UNIV_NOTHROW const page_t* page = get_frame(block); - ulint page_type = fil_page_get_type(page); - - if (page_type == FIL_PAGE_TYPE_XDES) { - return set_current_xdes(block->page.id.page_no(), page); - } else if (fil_page_index_page_check(page) - && !is_free(block->page.id.page_no()) - && !page_has_siblings(page)) { - - index_id_t id = btr_page_get_index_id(page); + index_id_t id = btr_page_get_index_id(page); - m_indexes.push_back(Index(id, block->page.id.page_no())); + m_index.m_id = id; + m_index.m_page_no = block->page.id.page_no(); - if (m_indexes.size() == 1) { - /* Check that the tablespace flags match the table flags. */ - ulint expected = dict_tf_to_fsp_flags(m_table->flags); - if (!fsp_flags_match(expected, m_space_flags)) { - ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, - ER_TABLE_SCHEMA_MISMATCH, - "Expected FSP_SPACE_FLAGS=0x%x, .ibd " - "file contains 0x%x.", - unsigned(expected), - unsigned(m_space_flags)); - return(DB_CORRUPTION); - } - } + /* Check that the tablespace flags match the table flags. */ + ulint expected = dict_tf_to_fsp_flags(m_table->flags); + if (!fsp_flags_match(expected, m_space_flags)) { + ib_errf(m_trx->mysql_thd, IB_LOG_LEVEL_ERROR, + ER_TABLE_SCHEMA_MISMATCH, + "Expected FSP_SPACE_FLAGS=0x%x, .ibd " + "file contains 0x%x.", + unsigned(expected), + unsigned(m_space_flags)); + return(DB_CORRUPTION); } return DB_SUCCESS; @@ -685,11 +715,9 @@ Update the import configuration that will be used to import the tablespace. dberr_t FetchIndexRootPages::build_row_import(row_import* cfg) const UNIV_NOTHROW { - Indexes::const_iterator end = m_indexes.end(); - ut_a(cfg->m_table == m_table); cfg->m_page_size.copy_from(m_page_size); - cfg->m_n_indexes = m_indexes.size(); + cfg->m_n_indexes = 1; if (cfg->m_n_indexes == 0) { @@ -715,37 +743,32 @@ FetchIndexRootPages::build_row_import(row_import* cfg) const UNIV_NOTHROW row_index_t* cfg_index = cfg->m_indexes; - for (Indexes::const_iterator it = m_indexes.begin(); - it != end; - ++it, ++cfg_index) { + char name[BUFSIZ]; - char name[BUFSIZ]; + snprintf(name, sizeof(name), "index" IB_ID_FMT, m_index.m_id); - snprintf(name, sizeof(name), "index" IB_ID_FMT, it->m_id); + ulint len = strlen(name) + 1; - ulint len = strlen(name) + 1; + cfg_index->m_name = UT_NEW_ARRAY_NOKEY(byte, len); - cfg_index->m_name = UT_NEW_ARRAY_NOKEY(byte, len); - - /* Trigger OOM */ - DBUG_EXECUTE_IF( - "ib_import_OOM_12", - UT_DELETE_ARRAY(cfg_index->m_name); - cfg_index->m_name = NULL; - ); + /* Trigger OOM */ + DBUG_EXECUTE_IF( + "ib_import_OOM_12", + UT_DELETE_ARRAY(cfg_index->m_name); + cfg_index->m_name = NULL; + ); - if (cfg_index->m_name == NULL) { - return(DB_OUT_OF_MEMORY); - } + if (cfg_index->m_name == NULL) { + return(DB_OUT_OF_MEMORY); + } - memcpy(cfg_index->m_name, name, len); + memcpy(cfg_index->m_name, name, len); - cfg_index->m_id = it->m_id; + cfg_index->m_id = m_index.m_id; - cfg_index->m_space = m_space; + cfg_index->m_space = m_space; - cfg_index->m_page_no = it->m_page_no; - } + cfg_index->m_page_no = m_index.m_page_no; return(DB_SUCCESS); } @@ -804,6 +827,11 @@ public: } } + dberr_t run(const fil_iterator_t& iter, buf_block_t* block) UNIV_NOTHROW + { + return fil_iterate(iter, block, *this); + } + /** Called for each block as it is read from the file. @param block block to convert, it is not from the buffer pool. @retval DB_SUCCESS or error code. */ @@ -1858,7 +1886,7 @@ PageConverter::update_index_page( if (is_free(block->page.id.page_no())) { return(DB_SUCCESS); - } else if ((id = btr_page_get_index_id(page)) != m_index->m_id) { + } else if ((id = btr_page_get_index_id(page)) != m_index->m_id && !m_cfg->m_missing) { row_index_t* index = find_index(id); @@ -3359,20 +3387,6 @@ dberr_t row_import_update_discarded_flag(trx_t* trx, table_id_t table_id, return(err); } -struct fil_iterator_t { - pfs_os_file_t file; /*!< File handle */ - const char* filepath; /*!< File path name */ - os_offset_t start; /*!< From where to start */ - os_offset_t end; /*!< Where to stop */ - os_offset_t file_size; /*!< File size in bytes */ - ulint n_io_buffers; /*!< Number of pages to use - for IO */ - byte* io_buffer; /*!< Buffer to use for IO */ - fil_space_crypt_t *crypt_data; /*!< Crypt data (if encrypted) */ - byte* crypt_io_buffer; /*!< IO buffer when encrypted */ -}; - - /** InnoDB writes page by page when there is page compressed tablespace involved. It does help to save the disk space when punch hole is enabled @@ -3423,22 +3437,91 @@ dberr_t fil_import_compress_fwrite(const fil_iterator_t &iter, return err; } -/********************************************************************//** -TODO: This can be made parallel trivially by chunking up the file and creating -a callback per thread. . Main benefit will be to use multiple CPUs for -checksums and compressed tables. We have to do compressed tables block by -block right now. Secondly we need to decompress/compress and copy too much -of data. These are CPU intensive. +dberr_t FetchIndexRootPages::run(const fil_iterator_t& iter, + buf_block_t* block) UNIV_NOTHROW +{ + const ulint size= get_page_size().physical(); + const ulint buf_size = srv_page_size +#ifdef HAVE_LZO + + LZO1X_1_15_MEM_COMPRESS +#elif defined HAVE_SNAPPY + + snappy_max_compressed_length(srv_page_size) +#endif + ; + byte* page_compress_buf = static_cast<byte*>(malloc(buf_size)); + ut_ad(!srv_read_only_mode); -Iterate over all the pages in the tablespace. -@param iter - Tablespace iterator -@param block - block to use for IO -@param callback - Callback to inspect and update page contents -@retval DB_SUCCESS or error code */ -static -dberr_t -fil_iterate( -/*========*/ + if (!page_compress_buf) + return DB_OUT_OF_MEMORY; + + const bool encrypted= iter.crypt_data != NULL && + iter.crypt_data->should_encrypt(); + byte* const readptr= iter.io_buffer; + block->frame= readptr; + + if (block->page.zip.data) + block->page.zip.data= readptr; + + IORequest read_request(IORequest::READ); + read_request.disable_partial_io_warnings(); + ulint page_no= 0; + bool page_compressed= false; + + dberr_t err= os_file_read_no_error_handling( + read_request, iter.file, readptr, 3 * size, size, 0); + if (err != DB_SUCCESS) + { + ib::error() << iter.filepath << ": os_file_read() failed"; + goto func_exit; + } + + block->page.id.set_page_no(3); + page_no= page_get_page_no(readptr); + + if (page_no != 3) + { +page_corrupted: + ib::warn() << filename() << ": Page 3 at offset " + << 3 * size << " looks corrupted."; + err= DB_CORRUPTION; + goto func_exit; + } + + page_compressed= fil_page_is_compressed_encrypted(readptr) || + fil_page_is_compressed(readptr); + + if (page_compressed && block->page.zip.data) + goto page_corrupted; + + if (encrypted) + { + if (!fil_space_verify_crypt_checksum(readptr, get_page_size())) + goto page_corrupted; + + if (!fil_space_decrypt(iter.crypt_data, readptr, + get_page_size(), readptr, &err) || + err != DB_SUCCESS) + goto func_exit; + } + + if (page_compressed) + { + ulint compress_length = fil_page_decompress(page_compress_buf, readptr); + ut_ad(compress_length != srv_page_size); + if (compress_length == 0) + goto page_corrupted; + } + else if (buf_page_is_corrupted( + false, readptr, get_page_size(), NULL)) + goto page_corrupted; + + err = this->operator()(block); +func_exit: + free(page_compress_buf); + return err; +} + +static dberr_t fil_iterate( const fil_iterator_t& iter, buf_block_t* block, AbstractCallback& callback) @@ -3875,7 +3958,7 @@ fil_tablespace_iterate( block->page.zip.data = block->frame + srv_page_size; } - err = fil_iterate(iter, block, callback); + err = callback.run(iter, block); if (iter.crypt_data) { fil_space_destroy_crypt_data(&iter.crypt_data); @@ -4020,6 +4103,16 @@ row_import_for_mysql( cfg.m_page_size.copy_from(univ_page_size); + if (UT_LIST_GET_LEN(table->indexes) > 1) { + ib_errf(trx->mysql_thd, IB_LOG_LEVEL_ERROR, + ER_INTERNAL_ERROR, + "Drop all secondary indexes before importing " + "table %s when .cfg file is missing.", + table->name.m_name); + err = DB_ERROR; + return row_import_error(prebuilt, trx, err); + } + FetchIndexRootPages fetchIndexRootPages(table, trx); err = fil_tablespace_iterate( diff --git a/storage/perfschema/CMakeLists.txt b/storage/perfschema/CMakeLists.txt index 56083f24223..7ed365ea175 100644 --- a/storage/perfschema/CMakeLists.txt +++ b/storage/perfschema/CMakeLists.txt @@ -187,9 +187,10 @@ table_session_connect_attrs.cc table_session_account_connect_attrs.cc ) -MYSQL_ADD_PLUGIN(perfschema ${PERFSCHEMA_SOURCES} STORAGE_ENGINE DEFAULT STATIC_ONLY) +MYSQL_ADD_PLUGIN(perfschema ${PERFSCHEMA_SOURCES} STORAGE_ENGINE DEFAULT + STATIC_ONLY DEPENDS GenServerSource) + IF (TARGET perfschema) - ADD_DEPENDENCIES(perfschema GenServerSource) IF(WITH_UNIT_TESTS) ADD_SUBDIRECTORY(unittest) ENDIF(WITH_UNIT_TESTS) diff --git a/tests/mysql_client_test.c b/tests/mysql_client_test.c index 554eec83644..668fa2133ef 100644 --- a/tests/mysql_client_test.c +++ b/tests/mysql_client_test.c @@ -18128,9 +18128,9 @@ static void test_bug40365(void) if (!opt_silent) fprintf(stdout, "\ntime[%d]: %02d-%02d-%02d ", i, tm[i].year, tm[i].month, tm[i].day); - DIE_UNLESS(tm[i].year == 0); - DIE_UNLESS(tm[i].month == 0); - DIE_UNLESS(tm[i].day == 0); + DIE_UNLESS(tm[i].year == 0); + DIE_UNLESS(tm[i].month == 0); + DIE_UNLESS(tm[i].day == 0); } mysql_stmt_close(stmt); rc= mysql_commit(mysql); |