From 4652260d656ea871fe1ff31176b6d85eeffde932 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Wed, 25 Jan 2023 11:46:28 -0800 Subject: MDEV-28616 Crash when using derived table over union with order by clause This bug manifested itself when the server processed a query containing a derived table over union whose ORDER BY clause included a subquery with unresolvable column reference. For such a query the server crashed when trying to resolve column references in the ORDER BY clause used by union. For any union with ORDER BY clause an extra SELECT_LEX structure is created and it is attached to SELECT_LEX_UNIT structure of the union via the field fake_select_lex. The outer context for fake_select_lex must be the same as for other selects of the union. If the union is used in the FROM list of a derived table then the outer context for fake_select_lex must be set to NULL in line with other selects of the union. It was not done and it caused a crash when searching for possible resolution of an unresolvable column reference occurred in a subquery used in the ORDER BY clause. Approved by Oleksandr Byelkin --- mysql-test/main/derived.result | 23 +++++++++++++++++++++++ mysql-test/main/derived.test | 30 ++++++++++++++++++++++++++++++ sql/sql_derived.cc | 3 +++ 3 files changed, 56 insertions(+) diff --git a/mysql-test/main/derived.result b/mysql-test/main/derived.result index 2761fdfa287..0cb029fa7a4 100644 --- a/mysql-test/main/derived.result +++ b/mysql-test/main/derived.result @@ -1327,5 +1327,28 @@ a b DROP VIEW v1; DROP TABLE t1; # +# MDEV-28616: derived table over union with order by clause that +# contains subquery with unresolvable column reference +# +SELECT 1 FROM ( +SELECT 1 UNION SELECT 2 ORDER BY (SELECT 1 FROM DUAL WHERE xxx = 0) +) dt; +ERROR 42S22: Unknown column 'xxx' in 'where clause' +create table t1 (a int, b int); +insert into t1 values (3,8), (7,2), (1,4), (5,9); +create table t2 (a int, b int); +insert into t2 values (9,1), (7,3), (2,6); +create table t3 (c int, d int); +insert into t3 values (7,8), (1,2), (3,8); +select * from +( +select a,b from t1 where t1.a > 3 +union +select a,b from t2 where t2.b < 6 +order by (a - b / (select a + max(c) from t3 where d = x)) +) dt; +ERROR 42S22: Unknown column 'x' in 'where clause' +drop table t1,t2,t3; +# # End of 10.3 tests # diff --git a/mysql-test/main/derived.test b/mysql-test/main/derived.test index 6a831000e57..dca7243febb 100644 --- a/mysql-test/main/derived.test +++ b/mysql-test/main/derived.test @@ -1137,6 +1137,36 @@ SELECT * FROM v1 WHERE b > 0; DROP VIEW v1; DROP TABLE t1; +--echo # +--echo # MDEV-28616: derived table over union with order by clause that +--echo # contains subquery with unresolvable column reference +--echo # + +--error ER_BAD_FIELD_ERROR +SELECT 1 FROM ( + SELECT 1 UNION SELECT 2 ORDER BY (SELECT 1 FROM DUAL WHERE xxx = 0) +) dt; + +create table t1 (a int, b int); +insert into t1 values (3,8), (7,2), (1,4), (5,9); + +create table t2 (a int, b int); +insert into t2 values (9,1), (7,3), (2,6); + +create table t3 (c int, d int); +insert into t3 values (7,8), (1,2), (3,8); + +--error ER_BAD_FIELD_ERROR +select * from +( + select a,b from t1 where t1.a > 3 + union + select a,b from t2 where t2.b < 6 + order by (a - b / (select a + max(c) from t3 where d = x)) +) dt; + +drop table t1,t2,t3; + --echo # --echo # End of 10.3 tests --echo # diff --git a/sql/sql_derived.cc b/sql/sql_derived.cc index 93dc62828ac..8177ee27943 100644 --- a/sql/sql_derived.cc +++ b/sql/sql_derived.cc @@ -771,6 +771,9 @@ bool mysql_derived_prepare(THD *thd, LEX *lex, TABLE_LIST *derived) cursor->outer_join|= JOIN_TYPE_OUTER; } } + // Prevent it for possible ORDER BY clause + if (unit->fake_select_lex) + unit->fake_select_lex->context.outer_context= 0; /* Above cascade call of prepare is important for PS protocol, but after it -- cgit v1.2.1 From b1043ea0ed01a7caa398d4a066b415d6eeebb08e Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Thu, 26 Jan 2023 10:57:01 +0400 Subject: Revert "MDEV-30151 parse error 1=2 not between/in" This reverts commit eba099184e1f6704894694ea41f97f216eae5f21. A different patch with less shift-reduce conflicts is coming. --- mysql-test/main/parser.result | 11 ----------- mysql-test/main/parser.test | 8 -------- sql/sql_yacc.yy | 6 +++--- sql/sql_yacc_ora.yy | 6 +++--- 4 files changed, 6 insertions(+), 25 deletions(-) diff --git a/mysql-test/main/parser.result b/mysql-test/main/parser.result index f44478727ae..0bb4e82c8b8 100644 --- a/mysql-test/main/parser.result +++ b/mysql-test/main/parser.result @@ -1866,15 +1866,4 @@ ERROR 42000: You have an error in your SQL syntax; check the manual that corresp EXECUTE IMMEDIATE 'CREATE PROCEDURE p() UPDATE t SET c=\'\'"abc'; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '"abc' at line 1 SET @@sql_mode=@save_sql_mode; -# -# MDEV-30151 parse error 1=2 not between/in -# -select 1=2 not in (3,4); -1=2 not in (3,4) -1 -select 1=2 not between 3 and 4; -1=2 not between 3 and 4 -1 -# # End of 10.3 tests -# diff --git a/mysql-test/main/parser.test b/mysql-test/main/parser.test index cfe4f9d6f53..9df18c50ee3 100644 --- a/mysql-test/main/parser.test +++ b/mysql-test/main/parser.test @@ -1673,12 +1673,4 @@ EXECUTE IMMEDIATE 'CREATE PROCEDURE p() UPDATE t SET c=\'\'"abc'; SET @@sql_mode=@save_sql_mode; ---echo # ---echo # MDEV-30151 parse error 1=2 not between/in ---echo # -select 1=2 not in (3,4); -select 1=2 not between 3 and 4; - ---echo # --echo # End of 10.3 tests ---echo # diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 3025d93de0f..7766049c104 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -899,7 +899,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); /* We should not introduce any further shift/reduce conflicts. */ -%expect 96 +%expect 85 /* Comments for TOKENS. @@ -1687,7 +1687,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); %left PREC_BELOW_NOT -%nonassoc LOW_PRIORITY_NOT +%nonassoc NOT_SYM %left '=' EQUAL_SYM GE '>' LE '<' NE %nonassoc IS %right BETWEEN_SYM @@ -9840,7 +9840,7 @@ expr: MYSQL_YYABORT; } } - | NOT_SYM expr %prec LOW_PRIORITY_NOT + | NOT_SYM expr %prec NOT_SYM { $$= negate_expression(thd, $2); if (unlikely($$ == NULL)) diff --git a/sql/sql_yacc_ora.yy b/sql/sql_yacc_ora.yy index df90ba6c634..a5ee1892e5e 100644 --- a/sql/sql_yacc_ora.yy +++ b/sql/sql_yacc_ora.yy @@ -293,7 +293,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); /* We should not introduce any further shift/reduce conflicts. */ -%expect 98 +%expect 87 /* Comments for TOKENS. @@ -1081,7 +1081,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); %left PREC_BELOW_NOT -%nonassoc LOW_PRIORITY_NOT +%nonassoc NOT_SYM %left '=' EQUAL_SYM GE '>' LE '<' NE %nonassoc IS %right BETWEEN_SYM @@ -9797,7 +9797,7 @@ expr: MYSQL_YYABORT; } } - | NOT_SYM expr %prec LOW_PRIORITY_NOT + | NOT_SYM expr %prec NOT_SYM { $$= negate_expression(thd, $2); if (unlikely($$ == NULL)) -- cgit v1.2.1 From 895673dae52d36c20caf900e6179694de1c7699b Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 12 Dec 2022 17:45:48 +0400 Subject: MDEV-30151 parse error 1=2 not between/in This patch fixes the problem by adding a new rule booleat_test. This makes the grammar clearer and less conflicting. Additionally, fixing %prec in this grammar branch: - | boolean_test IS NULL_SYM %prec PREC_BELOW_NOT + | boolean_test IS NULL_SYM %prec IS to have consistently "%prec IS" in all grammar branches starting with "boolean_test IS ...". It's not clear why these three rules needed different %prec before the fix: - boolean_test IS TRUE - boolean_test IS UNKNOWN - boolean_test IS NULL --- mysql-test/main/parser.result | 31 +++++++++++++++++++++++++ mysql-test/main/parser.test | 21 +++++++++++++++++ sql/sql_yacc.yy | 54 +++++++++++++++++++++++++++++++------------ sql/sql_yacc_ora.yy | 54 +++++++++++++++++++++++++++++++------------ 4 files changed, 130 insertions(+), 30 deletions(-) diff --git a/mysql-test/main/parser.result b/mysql-test/main/parser.result index 0bb4e82c8b8..a8ee4440b5e 100644 --- a/mysql-test/main/parser.result +++ b/mysql-test/main/parser.result @@ -1866,4 +1866,35 @@ ERROR 42000: You have an error in your SQL syntax; check the manual that corresp EXECUTE IMMEDIATE 'CREATE PROCEDURE p() UPDATE t SET c=\'\'"abc'; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '"abc' at line 1 SET @@sql_mode=@save_sql_mode; +# +# MDEV-30151 parse error 1=2 not between/in +# +SELECT 1=2 NOT IN (3,4); +1=2 NOT IN (3,4) +1 +SELECT 1=2 NOT BETWEEN 3 AND 4; +1=2 NOT BETWEEN 3 AND 4 +1 +CREATE TABLE t1 ( f INT AS ( 1 IN ( 2 NOT BETWEEN 3 AND 4 ) ) ); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `f` int(11) GENERATED ALWAYS AS (1 = 2 not between 3 and 4) VIRTUAL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE t1; +CREATE TABLE t1 ( f INT, CHECK ( 1 IN ( 2 NOT BETWEEN 3 AND 4 ) ) ); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `f` int(11) DEFAULT NULL, + CONSTRAINT `CONSTRAINT_1` CHECK (1 = 2 not between 3 and 4) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +DROP TABLE t1; +CREATE VIEW v1 AS SELECT 1 IN ( 2 NOT BETWEEN 3 AND 4 ); +SHOW CREATE VIEW v1; +View Create View character_set_client collation_connection +v1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select 1 = 2 not between 3 and 4 AS `1 IN ( 2 NOT BETWEEN 3 AND 4 )` latin1 latin1_swedish_ci +DROP VIEW v1; +# # End of 10.3 tests +# diff --git a/mysql-test/main/parser.test b/mysql-test/main/parser.test index 9df18c50ee3..9e46f859d5c 100644 --- a/mysql-test/main/parser.test +++ b/mysql-test/main/parser.test @@ -1673,4 +1673,25 @@ EXECUTE IMMEDIATE 'CREATE PROCEDURE p() UPDATE t SET c=\'\'"abc'; SET @@sql_mode=@save_sql_mode; +--echo # +--echo # MDEV-30151 parse error 1=2 not between/in +--echo # + +SELECT 1=2 NOT IN (3,4); +SELECT 1=2 NOT BETWEEN 3 AND 4; + +CREATE TABLE t1 ( f INT AS ( 1 IN ( 2 NOT BETWEEN 3 AND 4 ) ) ); +SHOW CREATE TABLE t1; +DROP TABLE t1; + +CREATE TABLE t1 ( f INT, CHECK ( 1 IN ( 2 NOT BETWEEN 3 AND 4 ) ) ); +SHOW CREATE TABLE t1; +DROP TABLE t1; + +CREATE VIEW v1 AS SELECT 1 IN ( 2 NOT BETWEEN 3 AND 4 ); +SHOW CREATE VIEW v1; +DROP VIEW v1; + +--echo # --echo # End of 10.3 tests +--echo # diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 7766049c104..e03bde31832 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -899,7 +899,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); /* We should not introduce any further shift/reduce conflicts. */ -%expect 85 +%expect 78 /* Comments for TOKENS. @@ -1687,7 +1687,8 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); %left PREC_BELOW_NOT -%nonassoc NOT_SYM +/* The precendence of boolean NOT is in fact here. See the comment below. */ + %left '=' EQUAL_SYM GE '>' LE '<' NE %nonassoc IS %right BETWEEN_SYM @@ -1699,6 +1700,24 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); %left '*' '/' '%' DIV_SYM MOD_SYM %left '^' %left MYSQL_CONCAT_SYM +/* + Boolean negation has a special branch in "expr" starting with NOT_SYM. + The precedence of logical negation is determined by the grammar itself + (without using Bison terminal symbol precedence) in this order + - Boolean factor (i.e. logical AND) + - Boolean NOT + - Boolean test (such as '=', IS NULL, IS TRUE) + + But we also need a precedence for NOT_SYM in other contexts, + to shift (without reduce) in these cases: + predicate NOT IN ... + predicate NOT BETWEEN ... + predicate NOT LIKE ... + predicate NOT REGEXP ... + If the precedence of NOT_SYM was low, it would reduce immediately + after scanning "predicate" and then produce a syntax error on "NOT". +*/ +%nonassoc NOT_SYM %nonassoc NEG '~' NOT2_SYM BINARY %nonassoc COLLATE_SYM @@ -1938,6 +1957,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); literal insert_ident order_ident temporal_literal simple_ident expr sum_expr in_sum_expr variable variable_aux + boolean_test predicate bit_expr parenthesized_expr table_wild simple_expr column_default_non_parenthesized_expr udf_expr primary_expr string_factor_expr mysql_concatenation_expr @@ -9840,79 +9860,83 @@ expr: MYSQL_YYABORT; } } - | NOT_SYM expr %prec NOT_SYM + | NOT_SYM expr { $$= negate_expression(thd, $2); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr IS TRUE_SYM %prec IS + | boolean_test %prec PREC_BELOW_NOT + ; + +boolean_test: + boolean_test IS TRUE_SYM %prec IS { $$= new (thd->mem_root) Item_func_istrue(thd, $1); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr IS not TRUE_SYM %prec IS + | boolean_test IS not TRUE_SYM %prec IS { $$= new (thd->mem_root) Item_func_isnottrue(thd, $1); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr IS FALSE_SYM %prec IS + | boolean_test IS FALSE_SYM %prec IS { $$= new (thd->mem_root) Item_func_isfalse(thd, $1); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr IS not FALSE_SYM %prec IS + | boolean_test IS not FALSE_SYM %prec IS { $$= new (thd->mem_root) Item_func_isnotfalse(thd, $1); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr IS UNKNOWN_SYM %prec IS + | boolean_test IS UNKNOWN_SYM %prec IS { $$= new (thd->mem_root) Item_func_isnull(thd, $1); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr IS not UNKNOWN_SYM %prec IS + | boolean_test IS not UNKNOWN_SYM %prec IS { $$= new (thd->mem_root) Item_func_isnotnull(thd, $1); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr IS NULL_SYM %prec PREC_BELOW_NOT + | boolean_test IS NULL_SYM %prec IS { $$= new (thd->mem_root) Item_func_isnull(thd, $1); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr IS not NULL_SYM %prec IS + | boolean_test IS not NULL_SYM %prec IS { $$= new (thd->mem_root) Item_func_isnotnull(thd, $1); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr EQUAL_SYM predicate %prec EQUAL_SYM + | boolean_test EQUAL_SYM predicate %prec EQUAL_SYM { $$= new (thd->mem_root) Item_func_equal(thd, $1, $3); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr comp_op predicate %prec '=' + | boolean_test comp_op predicate %prec '=' { $$= (*$2)(0)->create(thd, $1, $3); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr comp_op all_or_any '(' subselect ')' %prec '=' + | boolean_test comp_op all_or_any '(' subselect ')' %prec '=' { $$= all_any_subquery_creator(thd, $1, $2, $3, $5); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | predicate + | predicate %prec BETWEEN_SYM ; predicate: diff --git a/sql/sql_yacc_ora.yy b/sql/sql_yacc_ora.yy index a5ee1892e5e..89f7412ea89 100644 --- a/sql/sql_yacc_ora.yy +++ b/sql/sql_yacc_ora.yy @@ -293,7 +293,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); /* We should not introduce any further shift/reduce conflicts. */ -%expect 87 +%expect 80 /* Comments for TOKENS. @@ -1081,7 +1081,8 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); %left PREC_BELOW_NOT -%nonassoc NOT_SYM +/* The precendence of boolean NOT is in fact here. See the comment below. */ + %left '=' EQUAL_SYM GE '>' LE '<' NE %nonassoc IS %right BETWEEN_SYM @@ -1093,6 +1094,24 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); %left '*' '/' '%' DIV_SYM MOD_SYM %left '^' %left MYSQL_CONCAT_SYM +/* + Boolean negation has a special branch in "expr" starting with NOT_SYM. + The precedence of logical negation is determined by the grammar itself + (without using Bison terminal symbol precedence) in this order + - Boolean factor (i.e. logical AND) + - Boolean NOT + - Boolean test (such as '=', IS NULL, IS TRUE) + + But we also need a precedence for NOT_SYM in other contexts, + to shift (without reduce) in these cases: + predicate NOT IN ... + predicate NOT BETWEEN ... + predicate NOT LIKE ... + predicate NOT REGEXP ... + If the precedence of NOT_SYM was low, it would reduce immediately + after scanning "predicate" and then produce a syntax error on "NOT". +*/ +%nonassoc NOT_SYM %nonassoc NEG '~' NOT2_SYM BINARY %nonassoc COLLATE_SYM @@ -1339,6 +1358,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, size_t *yystacksize); literal insert_ident order_ident temporal_literal simple_ident expr sum_expr in_sum_expr variable variable_aux + boolean_test predicate bit_expr parenthesized_expr table_wild simple_expr column_default_non_parenthesized_expr udf_expr primary_expr string_factor_expr mysql_concatenation_expr @@ -9797,79 +9817,83 @@ expr: MYSQL_YYABORT; } } - | NOT_SYM expr %prec NOT_SYM + | NOT_SYM expr { $$= negate_expression(thd, $2); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr IS TRUE_SYM %prec IS + | boolean_test %prec PREC_BELOW_NOT + ; + +boolean_test: + boolean_test IS TRUE_SYM %prec IS { $$= new (thd->mem_root) Item_func_istrue(thd, $1); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr IS not TRUE_SYM %prec IS + | boolean_test IS not TRUE_SYM %prec IS { $$= new (thd->mem_root) Item_func_isnottrue(thd, $1); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr IS FALSE_SYM %prec IS + | boolean_test IS FALSE_SYM %prec IS { $$= new (thd->mem_root) Item_func_isfalse(thd, $1); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr IS not FALSE_SYM %prec IS + | boolean_test IS not FALSE_SYM %prec IS { $$= new (thd->mem_root) Item_func_isnotfalse(thd, $1); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr IS UNKNOWN_SYM %prec IS + | boolean_test IS UNKNOWN_SYM %prec IS { $$= new (thd->mem_root) Item_func_isnull(thd, $1); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr IS not UNKNOWN_SYM %prec IS + | boolean_test IS not UNKNOWN_SYM %prec IS { $$= new (thd->mem_root) Item_func_isnotnull(thd, $1); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr IS NULL_SYM %prec PREC_BELOW_NOT + | boolean_test IS NULL_SYM %prec IS { $$= new (thd->mem_root) Item_func_isnull(thd, $1); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr IS not NULL_SYM %prec IS + | boolean_test IS not NULL_SYM %prec IS { $$= new (thd->mem_root) Item_func_isnotnull(thd, $1); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr EQUAL_SYM predicate %prec EQUAL_SYM + | boolean_test EQUAL_SYM predicate %prec EQUAL_SYM { $$= new (thd->mem_root) Item_func_equal(thd, $1, $3); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr comp_op predicate %prec '=' + | boolean_test comp_op predicate %prec '=' { $$= (*$2)(0)->create(thd, $1, $3); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | expr comp_op all_or_any '(' subselect ')' %prec '=' + | boolean_test comp_op all_or_any '(' subselect ')' %prec '=' { $$= all_any_subquery_creator(thd, $1, $2, $3, $5); if (unlikely($$ == NULL)) MYSQL_YYABORT; } - | predicate + | predicate %prec BETWEEN_SYM ; predicate: -- cgit v1.2.1 From f812f8e1ab7acc5dd17daf5ab5f73495c7963af5 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 26 Jan 2023 12:22:38 +0100 Subject: MDEV-30475 Windows, mtr - Remove outdated instructions on how to install post-mortem debugger Also, use standard C:\symbols location for OS debugging symbols cache, rather than own invention C:\cdb_symbols. --- mysql-test/lib/My/CoreDump.pm | 33 +++------------------------------ 1 file changed, 3 insertions(+), 30 deletions(-) diff --git a/mysql-test/lib/My/CoreDump.pm b/mysql-test/lib/My/CoreDump.pm index 05b6edf1385..be6d21146d1 100644 --- a/mysql-test/lib/My/CoreDump.pm +++ b/mysql-test/lib/My/CoreDump.pm @@ -310,16 +310,8 @@ sub cdb_check { `cdb -? 2>&1`; if ($? >> 8) { - print "Cannot find cdb. Please Install Debugging tools for Windows\n"; - print "from http://www.microsoft.com/whdc/devtools/debugging/"; - if($ENV{'ProgramW6432'}) - { - print "install64bit.mspx (native x64 version)\n"; - } - else - { - print "installx86.mspx\n"; - } + print "Cannot find the cdb debugger. Please install Debugging tools for Windows\n"; + print "and set PATH environment variable to include location of cdb.exe"; } } @@ -328,25 +320,6 @@ sub _cdb { my ($core_name, $format)= @_; print "\nTrying 'cdb' to get a backtrace\n"; return unless -f $core_name; - - # Try to set environment for debugging tools for Windows - if ($ENV{'PATH'} !~ /Debugging Tools/) - { - if ($ENV{'ProgramW6432'}) - { - # On x64 computer - $ENV{'PATH'}.= ";".$ENV{'ProgramW6432'}."\\Debugging Tools For Windows (x64)"; - } - else - { - # On x86 computer. Newest versions of Debugging tools are installed in the - # directory with (x86) suffix, older versions did not have this suffix. - $ENV{'PATH'}.= ";".$ENV{'ProgramFiles'}."\\Debugging Tools For Windows (x86)"; - $ENV{'PATH'}.= ";".$ENV{'ProgramFiles'}."\\Debugging Tools For Windows"; - } - } - - # Read module list, find out the name of executable and # build symbol path (required by cdb if executable was built on # different machine) @@ -384,7 +357,7 @@ sub _cdb { if (!$ENV{'_NT_SYMBOL_PATH'}) { my $windir= $ENV{'windir'}; - my $symbol_cache= substr($windir ,0, index($windir,'\\'))."\\cdb_symbols"; + my $symbol_cache= substr($windir ,0, index($windir,'\\'))."\\symbols"; print "OS debug symbols will be downloaded and stored in $symbol_cache.\n"; print "You can control the location of symbol cache with _NT_SYMBOL_PATH\n"; -- cgit v1.2.1 From 2a78c3ef6fd6663d6731dd5cec2f462420b61123 Mon Sep 17 00:00:00 2001 From: Salman Mohammadi Date: Sun, 8 Jan 2023 20:14:58 +0100 Subject: MDEV-30509: mariadb-plugin-connect: introduce curl as recommends in order to be able to retrieve files using REST queries. Otherwise, `ERROR 1105 (HY000): Curl not installed.` will be thrown. --- debian/control | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/control b/debian/control index aed073e5c8c..09e0fd74d57 100644 --- a/debian/control +++ b/debian/control @@ -525,6 +525,7 @@ Depends: libxml2, unixodbc, ${misc:Depends}, ${shlibs:Depends} +Recommends: curl Breaks: mariadb-connect-engine-10.1, mariadb-connect-engine-10.2, mariadb-connect-engine-10.3 -- cgit v1.2.1 From 9b32e4b192303421ca26625153ae1190429e307f Mon Sep 17 00:00:00 2001 From: Nayuta Yanagisawa Date: Tue, 27 Sep 2022 15:22:57 +0900 Subject: MDEV-29644 a potential bug of null pointer dereference in spider_db_mbase::print_warnings() The function spider_db_mbase::print_warnings() can potentially result in a null pointer dereference. Remove the null pointer dereference by cleaning up the function. Some small changes to the original commit 422fb63a9bbee35c50b6c7be19d199afe0bc98fa. Co-Authored-By: Yuchen Pei --- .../mysql-test/spider/bugfix/r/mdev_29644.result | 41 ++++++++++ .../mysql-test/spider/bugfix/t/mdev_29644.cnf | 3 + .../mysql-test/spider/bugfix/t/mdev_29644.test | 56 ++++++++++++++ storage/spider/spd_db_mysql.cc | 88 +++++++++------------- storage/spider/spd_db_mysql.h | 4 +- 5 files changed, 136 insertions(+), 56 deletions(-) create mode 100644 storage/spider/mysql-test/spider/bugfix/r/mdev_29644.result create mode 100644 storage/spider/mysql-test/spider/bugfix/t/mdev_29644.cnf create mode 100644 storage/spider/mysql-test/spider/bugfix/t/mdev_29644.test diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_29644.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_29644.result new file mode 100644 index 00000000000..b52cecc5bb7 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_29644.result @@ -0,0 +1,41 @@ +# +# MDEV-29644 a potential bug of null pointer dereference in spider_db_mbase::print_warnings() +# +for master_1 +for child2 +child2_1 +child2_2 +child2_3 +for child3 +connection child2_1; +CREATE DATABASE auto_test_remote; +USE auto_test_remote; +CREATE TABLE tbl_a ( +a CHAR(5) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; +SET GLOBAL sql_mode=''; +connection master_1; +CREATE DATABASE auto_test_local; +USE auto_test_local; +CREATE TABLE tbl_a ( +a CHAR(255) +) ENGINE=Spider DEFAULT CHARSET=utf8 COMMENT='table "tbl_a", srv "s_2_1"'; +SET sql_mode=''; +INSERT INTO tbl_a VALUES ("this will be truncated"); +NOT FOUND /\[WARN SPIDER RESULT\].* Warning 1265 Data truncated for column 'a' at row 1.*/ in mysqld.1.1.err +SET GLOBAL spider_log_result_errors=4; +INSERT INTO tbl_a VALUES ("this will be truncated"); +FOUND 1 /\[WARN SPIDER RESULT\].* Warning 1265 Data truncated for column 'a' at row 1.*/ in mysqld.1.1.err +connection master_1; +SET GLOBAL spider_log_result_errors=DEFAULT; +SET sql_mode=DEFAULT; +DROP DATABASE IF EXISTS auto_test_local; +connection child2_1; +SET GLOBAL sql_mode=DEFAULT; +DROP DATABASE IF EXISTS auto_test_remote; +for master_1 +for child2 +child2_1 +child2_2 +child2_3 +for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_29644.cnf b/storage/spider/mysql-test/spider/bugfix/t/mdev_29644.cnf new file mode 100644 index 00000000000..05dfd8a0bce --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_29644.cnf @@ -0,0 +1,3 @@ +!include include/default_mysqld.cnf +!include ../my_1_1.cnf +!include ../my_2_1.cnf diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_29644.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_29644.test new file mode 100644 index 00000000000..3a8fbb251e1 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_29644.test @@ -0,0 +1,56 @@ +--echo # +--echo # MDEV-29644 a potential bug of null pointer dereference in spider_db_mbase::print_warnings() +--echo # + +# The test case below does not cause the potential null pointer dereference. +# It is just for checking spider_db_mbase::fetch_and_print_warnings() works. + +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log + +--connection child2_1 +CREATE DATABASE auto_test_remote; +USE auto_test_remote; +eval CREATE TABLE tbl_a ( + a CHAR(5) +) $CHILD2_1_ENGINE $CHILD2_1_CHARSET; + +SET GLOBAL sql_mode=''; + +--connection master_1 +CREATE DATABASE auto_test_local; +USE auto_test_local; +eval CREATE TABLE tbl_a ( + a CHAR(255) +) $MASTER_1_ENGINE $MASTER_1_CHARSET COMMENT='table "tbl_a", srv "s_2_1"'; + +SET sql_mode=''; + +let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.1.err; +let SEARCH_PATTERN= \[WARN SPIDER RESULT\].* Warning 1265 Data truncated for column 'a' at row 1.*; + +INSERT INTO tbl_a VALUES ("this will be truncated"); +--source include/search_pattern_in_file.inc # should not find + +SET GLOBAL spider_log_result_errors=4; + +INSERT INTO tbl_a VALUES ("this will be truncated"); +--source include/search_pattern_in_file.inc # should find + +--connection master_1 +SET GLOBAL spider_log_result_errors=DEFAULT; +SET sql_mode=DEFAULT; +DROP DATABASE IF EXISTS auto_test_local; + +--connection child2_1 +SET GLOBAL sql_mode=DEFAULT; +DROP DATABASE IF EXISTS auto_test_remote; + +--disable_query_log +--disable_result_log +--source ../t/test_deinit.inc +--enable_query_log +--enable_result_log diff --git a/storage/spider/spd_db_mysql.cc b/storage/spider/spd_db_mysql.cc index e942d1d9063..b1c222d193a 100644 --- a/storage/spider/spd_db_mysql.cc +++ b/storage/spider/spd_db_mysql.cc @@ -2090,7 +2090,7 @@ int spider_db_mbase::exec_query( db_conn->affected_rows, db_conn->insert_id, db_conn->server_status, db_conn->warning_count); if (spider_param_log_result_errors() >= 3) - print_warnings(l_time); + fetch_and_print_warnings(l_time); } else if (log_result_errors >= 4) { time_t cur_time = (time_t) time((time_t*) 0); @@ -2172,61 +2172,43 @@ bool spider_db_mbase::is_xa_nota_error( DBUG_RETURN(xa_nota); } -void spider_db_mbase::print_warnings( - struct tm *l_time -) { - DBUG_ENTER("spider_db_mbase::print_warnings"); - DBUG_PRINT("info",("spider this=%p", this)); - if (db_conn->status == MYSQL_STATUS_READY) +void spider_db_mbase::fetch_and_print_warnings(struct tm *l_time) +{ + DBUG_ENTER("spider_db_mbase::fetch_and_print_warnings"); + + if (spider_param_dry_access() || db_conn->status != MYSQL_STATUS_READY || + db_conn->server_status & SERVER_MORE_RESULTS_EXISTS) + DBUG_VOID_RETURN; + + if (mysql_real_query(db_conn, SPIDER_SQL_SHOW_WARNINGS_STR, + SPIDER_SQL_SHOW_WARNINGS_LEN)) + DBUG_VOID_RETURN; + + MYSQL_RES *res= mysql_store_result(db_conn); + if (!res) + DBUG_VOID_RETURN; + + uint num_fields= mysql_num_fields(res); + if (num_fields != 3) { -#if MYSQL_VERSION_ID < 50500 - if (!(db_conn->last_used_con->server_status & SERVER_MORE_RESULTS_EXISTS)) -#else - if (!(db_conn->server_status & SERVER_MORE_RESULTS_EXISTS)) -#endif - { - if ( - spider_param_dry_access() || - !mysql_real_query(db_conn, SPIDER_SQL_SHOW_WARNINGS_STR, - SPIDER_SQL_SHOW_WARNINGS_LEN) - ) { - MYSQL_RES *res = NULL; - MYSQL_ROW row = NULL; - uint num_fields; - if ( - spider_param_dry_access() || - !(res = mysql_store_result(db_conn)) || - !(row = mysql_fetch_row(res)) - ) { - if (mysql_errno(db_conn)) - { - if (res) - mysql_free_result(res); - DBUG_VOID_RETURN; - } - /* no record is ok */ - } - num_fields = mysql_num_fields(res); - if (num_fields != 3) - { - mysql_free_result(res); - DBUG_VOID_RETURN; - } - while (row) - { - fprintf(stderr, "%04d%02d%02d %02d:%02d:%02d [WARN SPIDER RESULT] " - "from [%s] %ld to %ld: %s %s %s\n", + mysql_free_result(res); + DBUG_VOID_RETURN; + } + + MYSQL_ROW row= mysql_fetch_row(res); + while (row) + { + fprintf(stderr, + "%04d%02d%02d %02d:%02d:%02d [WARN SPIDER RESULT] from [%s] %ld " + "to %ld: %s %s %s\n", l_time->tm_year + 1900, l_time->tm_mon + 1, l_time->tm_mday, - l_time->tm_hour, l_time->tm_min, l_time->tm_sec, - conn->tgt_host, (ulong) db_conn->thread_id, - (ulong) current_thd->thread_id, row[0], row[1], row[2]); - row = mysql_fetch_row(res); - } - if (res) - mysql_free_result(res); - } - } + l_time->tm_hour, l_time->tm_min, l_time->tm_sec, conn->tgt_host, + (ulong) db_conn->thread_id, (ulong) current_thd->thread_id, row[0], + row[1], row[2]); + row= mysql_fetch_row(res); } + mysql_free_result(res); + DBUG_VOID_RETURN; } diff --git a/storage/spider/spd_db_mysql.h b/storage/spider/spd_db_mysql.h index 4d5327b7533..576162b2b55 100644 --- a/storage/spider/spd_db_mysql.h +++ b/storage/spider/spd_db_mysql.h @@ -392,9 +392,7 @@ public: bool is_xa_nota_error( int error_num ); - void print_warnings( - struct tm *l_time - ); + void fetch_and_print_warnings(struct tm *l_time); spider_db_result *store_result( spider_db_result_buffer **spider_res_buf, st_spider_db_request_key *request_key, -- cgit v1.2.1 From e62947f38bb9c15f7fa8d3faa60b1852c4fecb80 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 9 Mar 2023 15:32:24 +0100 Subject: bump the VERSION --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 876e7c96e80..26166cc8151 100644 --- a/VERSION +++ b/VERSION @@ -1,4 +1,4 @@ MYSQL_VERSION_MAJOR=10 MYSQL_VERSION_MINOR=3 -MYSQL_VERSION_PATCH=38 +MYSQL_VERSION_PATCH=39 SERVER_MATURITY=stable -- cgit v1.2.1 From fb7d5881535574c0e33fa8338eaed9ecc7bc65c6 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 9 Mar 2023 11:32:18 +0100 Subject: main.bootstrap test cleanup --- mysql-test/main/bootstrap.result | 42 ++++++++++++++-- mysql-test/main/bootstrap.test | 102 +++++++++++++++++++-------------------- 2 files changed, 89 insertions(+), 55 deletions(-) diff --git a/mysql-test/main/bootstrap.result b/mysql-test/main/bootstrap.result index 0ba87be092e..7ea00923823 100644 --- a/mysql-test/main/bootstrap.result +++ b/mysql-test/main/bootstrap.result @@ -1,13 +1,26 @@ -drop table if exists t1; +# +# test mysqld in bootstrap mode +# +# +# Check that --bootstrap reads from stdin +# drop table t1; +# +# Check that --bootstrap of file with SQL error returns error +# drop table t1; ERROR 42S02: Unknown table 'test.t1' +# +# Bootstrap with a large thd->net.max_packet +# set @my_max_allowed_packet= @@max_allowed_packet; set @@global.max_allowed_packet= greatest(1073741824, @@max_allowed_packet); set @max_allowed_packed=@@global.max_allowed_packet; set global max_allowed_packet=@my_max_allowed_packet; drop table t1; -End of 5.1 tests +# +# End of 5.1 tests +# # # Bug #11766306: 59393: HAVE_INNODB=YES WHEN MYSQLD # STARTED WITH --SKIP-INNODB @@ -15,7 +28,21 @@ End of 5.1 tests SELECT 'bug' as '' FROM INFORMATION_SCHEMA.ENGINES WHERE engine='innodb' and SUPPORT='YES'; -End of 5.5 tests +# +# MDEV-13063 Server crashes in intern_plugin_lock or assertion `plugin_ptr->ref_count == 1' fails in plugin_init +# +# +# MDEV-19349 mysql_install_db: segfault at tmp_file_prefix check +# +# +# End of 5.5 tests +# +# +# Check that --bootstrap can install and uninstall plugins +# +# +# Check that installed plugins are *not* automatically loaded in --bootstrap +# flush tables; show create table t1; Table Create Table @@ -27,3 +54,12 @@ select * from mysql.plugin; name dl EXAMPLE ha_example.so truncate table mysql.plugin; +# +# MDEV-9969 mysql_install_db error processing ignore_db_dirs. +# +# +# MDEV-13397 MariaDB upgrade fail when using default_time_zone +# +# +# End of 10.3 tests +# diff --git a/mysql-test/main/bootstrap.test b/mysql-test/main/bootstrap.test index 683033979fe..318842b550b 100644 --- a/mysql-test/main/bootstrap.test +++ b/mysql-test/main/bootstrap.test @@ -1,16 +1,20 @@ -# -# test mysqld in bootstrap mode -# ---disable_warnings -drop table if exists t1; ---enable_warnings +--echo # +--echo # test mysqld in bootstrap mode +--echo # +--source include/not_windows_embedded.inc +--source include/have_example_plugin.inc + +--let test_bootstrap=$MYSQLTEST_VARDIR/tmp/test_bootstrap.sql +--write_file $test_bootstrap +use test; +EOF # Add the datadir to the bootstrap command let $MYSQLD_DATADIR= `select @@datadir`; let $MYSQLD_BOOTSTRAP_CMD= $MYSQLD_BOOTSTRAP_CMD --datadir=$MYSQLD_DATADIR --tmpdir=$MYSQL_TMP_DIR --default-storage-engine=MyISAM --loose-skip-innodb --plugin-maturity=unknown; -# -# Check that --bootstrap reads from stdin -# +--echo # +--echo # Check that --bootstrap reads from stdin +--echo # --write_file $MYSQLTEST_VARDIR/tmp/bootstrap_test.sql use test; CREATE TABLE t1(a int); @@ -18,9 +22,9 @@ EOF --exec $MYSQLD_BOOTSTRAP_CMD < $MYSQLTEST_VARDIR/tmp/bootstrap_test.sql >> $MYSQLTEST_VARDIR/tmp/bootstrap.log 2>&1 drop table t1; remove_file $MYSQLTEST_VARDIR/tmp/bootstrap_test.sql; -# -# Check that --bootstrap of file with SQL error returns error -# +--echo # +--echo # Check that --bootstrap of file with SQL error returns error +--echo # --write_file $MYSQLTEST_VARDIR/tmp/bootstrap_error.sql use test; CREATE TABLE t1; @@ -32,9 +36,9 @@ EOF drop table t1; remove_file $MYSQLTEST_VARDIR/tmp/bootstrap_error.sql; -# -# Bootstrap with a large thd->net.max_packet -# +--echo # +--echo # Bootstrap with a large thd->net.max_packet +--echo # set @my_max_allowed_packet= @@max_allowed_packet; set @@global.max_allowed_packet= greatest(1073741824, @@max_allowed_packet); set @max_allowed_packed=@@global.max_allowed_packet; @@ -49,7 +53,9 @@ remove_file $MYSQLTEST_VARDIR/tmp/long_query.sql; set global max_allowed_packet=@my_max_allowed_packet; drop table t1; ---echo End of 5.1 tests +--echo # +--echo # End of 5.1 tests +--echo # --echo # --echo # Bug #11766306: 59393: HAVE_INNODB=YES WHEN MYSQLD @@ -60,28 +66,24 @@ drop table t1; SELECT 'bug' as '' FROM INFORMATION_SCHEMA.ENGINES WHERE engine='innodb' and SUPPORT='YES'; -# -# MDEV-13063 Server crashes in intern_plugin_lock or assertion `plugin_ptr->ref_count == 1' fails in plugin_init -# +--echo # +--echo # MDEV-13063 Server crashes in intern_plugin_lock or assertion `plugin_ptr->ref_count == 1' fails in plugin_init +--echo # --error 1 --exec $MYSQLD_BOOTSTRAP_CMD --myisam_recover_options=NONE -# -# MDEV-19349 mysql_install_db: segfault at tmp_file_prefix check -# ---write_file $MYSQLTEST_VARDIR/tmp/1 -use test; -EOF ---exec $MYSQLD_BOOTSTRAP_CMD < $MYSQLTEST_VARDIR/tmp/1 >> $MYSQLTEST_VARDIR/tmp/bootstrap.log 2>&1 ---remove_file $MYSQLTEST_VARDIR/tmp/1 +--echo # +--echo # MDEV-19349 mysql_install_db: segfault at tmp_file_prefix check +--echo # +--exec $MYSQLD_BOOTSTRAP_CMD < $test_bootstrap >> $MYSQLTEST_VARDIR/tmp/bootstrap.log 2>&1 ---echo End of 5.5 tests +--echo # +--echo # End of 5.5 tests +--echo # ---source include/not_windows_embedded.inc ---source include/have_example_plugin.inc -# -# Check that --bootstrap can install and uninstall plugins -# +--echo # +--echo # Check that --bootstrap can install and uninstall plugins +--echo # let $PLUGIN_DIR=`select @@plugin_dir`; --write_file $MYSQLTEST_VARDIR/tmp/install_plugin.sql install soname 'ha_example'; @@ -90,9 +92,9 @@ EOF --exec $MYSQLD_BOOTSTRAP_CMD --plugin-dir=$PLUGIN_DIR < $MYSQLTEST_VARDIR/tmp/install_plugin.sql >> $MYSQLTEST_VARDIR/tmp/bootstrap.log 2>&1 --remove_file $MYSQLTEST_VARDIR/tmp/install_plugin.sql -# -# Check that installed plugins are *not* automatically loaded in --bootstrap -# +--echo # +--echo # Check that installed plugins are *not* automatically loaded in --bootstrap +--echo # --write_file $MYSQLTEST_VARDIR/tmp/bootstrap_plugins.sql SET SQL_MODE=""; use test; @@ -107,21 +109,17 @@ drop table t1; select * from mysql.plugin; truncate table mysql.plugin; +--echo # +--echo # MDEV-9969 mysql_install_db error processing ignore_db_dirs. +--echo # +--exec $MYSQLD_BOOTSTRAP_CMD --ignore-db-dirs='some_dir' --ignore-db-dirs='some_dir' < $test_bootstrap >> $MYSQLTEST_VARDIR/tmp/bootstrap.log 2>&1 -# -# MDEV-9969 mysql_install_db error processing ignore_db_dirs. -# ---write_file $MYSQLTEST_VARDIR/tmp/bootstrap_9969.sql -use test; -EOF ---exec $MYSQLD_BOOTSTRAP_CMD --ignore-db-dirs='some_dir' --ignore-db-dirs='some_dir' < $MYSQLTEST_VARDIR/tmp/bootstrap_9969.sql >> $MYSQLTEST_VARDIR/tmp/bootstrap.log 2>&1 ---remove_file $MYSQLTEST_VARDIR/tmp/bootstrap_9969.sql +--echo # +--echo # MDEV-13397 MariaDB upgrade fail when using default_time_zone +--echo # +--exec $MYSQLD_BOOTSTRAP_CMD --default-time-zone=Europe/Moscow < $test_bootstrap >> $MYSQLTEST_VARDIR/tmp/bootstrap.log 2>&1 -# -# MDEV-13397 MariaDB upgrade fail when using default_time_zone -# ---write_file $MYSQLTEST_VARDIR/tmp/bootstrap_9969.sql -use test; -EOF ---exec $MYSQLD_BOOTSTRAP_CMD --default-time-zone=Europe/Moscow < $MYSQLTEST_VARDIR/tmp/bootstrap_9969.sql >> $MYSQLTEST_VARDIR/tmp/bootstrap.log 2>&1 ---remove_file $MYSQLTEST_VARDIR/tmp/bootstrap_9969.sql +--echo # +--echo # End of 10.3 tests +--echo # +--remove_file $test_bootstrap -- cgit v1.2.1 From 4c4939bbf619d7e516131c0b3e5691b1c2d2ff8f Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 9 Mar 2023 11:22:41 +0100 Subject: MDEV-30818 invalid ssl prevents bootstrap in bootstrap the server reads stdin and does not listen to network. it won't use ssl anyway --- mysql-test/main/bootstrap.result | 3 +++ mysql-test/main/bootstrap.test | 5 +++++ sql/mysqld.cc | 5 ++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/mysql-test/main/bootstrap.result b/mysql-test/main/bootstrap.result index 7ea00923823..0b256263a29 100644 --- a/mysql-test/main/bootstrap.result +++ b/mysql-test/main/bootstrap.result @@ -61,5 +61,8 @@ truncate table mysql.plugin; # MDEV-13397 MariaDB upgrade fail when using default_time_zone # # +# MDEV-30818 invalid ssl prevents bootstrap +# +# # End of 10.3 tests # diff --git a/mysql-test/main/bootstrap.test b/mysql-test/main/bootstrap.test index 318842b550b..e3c121f201f 100644 --- a/mysql-test/main/bootstrap.test +++ b/mysql-test/main/bootstrap.test @@ -119,6 +119,11 @@ truncate table mysql.plugin; --echo # --exec $MYSQLD_BOOTSTRAP_CMD --default-time-zone=Europe/Moscow < $test_bootstrap >> $MYSQLTEST_VARDIR/tmp/bootstrap.log 2>&1 +--echo # +--echo # MDEV-30818 invalid ssl prevents bootstrap +--echo # +--exec $MYSQLD_BOOTSTRAP_CMD --ssl-ca=/dev/nonexistent < $test_bootstrap >> $MYSQLTEST_VARDIR/tmp/bootstrap.log 2>&1 + --echo # --echo # End of 10.3 tests --echo # diff --git a/sql/mysqld.cc b/sql/mysqld.cc index ea3849ed9f9..68f66c37b6c 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -5026,7 +5026,10 @@ static void init_ssl() { sql_print_error("Failed to setup SSL"); sql_print_error("SSL error: %s", sslGetErrString(error)); - unireg_abort(1); + if (!opt_bootstrap) + unireg_abort(1); + opt_use_ssl = 0; + have_ssl= SHOW_OPTION_DISABLED; } if (global_system_variables.log_warnings > 0) { -- cgit v1.2.1 From 79e27a6bf9e05eb2296e59a4ea4cd1334195faca Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 5 Apr 2023 23:34:03 +0200 Subject: MDEV-25887 "Got notification message from PID xxxx, but reception only permitted for main PID yyyy" in systemd during SST server has systemd support and calls sd_notify() to communicate the status to systemd. mariabackup links the whole server in, but it should not notify systemd, because it's not started or managed by systemd. --- extra/mariabackup/xtrabackup.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 5e01e64f490..ad35749d0ff 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -113,6 +113,12 @@ Street, Fifth Floor, Boston, MA 02110-1335 USA #define MB_CORRUPTED_PAGES_FILE "innodb_corrupted_pages" +// disable server's systemd notification code +extern "C" { +int sd_notify() { return 0; } +int sd_notifyf() { return 0; } +} + int sys_var_init(); /* === xtrabackup specific options === */ -- cgit v1.2.1 From 54715a1074a0c9fc12a5d52152df73826a484df7 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Thu, 6 Apr 2023 09:57:58 +0400 Subject: MDEV-30072 Wrong ORDER BY for a partitioned prefix key + NOPAD This problem was earlier fixed by MDEV-30034. Adding MTR tests only. --- mysql-test/main/ctype_uca_partitions.result | 40 +++++++++++++++++++++++++++++ mysql-test/main/ctype_uca_partitions.test | 32 +++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/mysql-test/main/ctype_uca_partitions.result b/mysql-test/main/ctype_uca_partitions.result index d7b79046b34..373fe914527 100644 --- a/mysql-test/main/ctype_uca_partitions.result +++ b/mysql-test/main/ctype_uca_partitions.result @@ -84,3 +84,43 @@ O P Y DROP TABLE t1; +# +# Start of 10.4 tests +# +# +# MDEV-30072 Wrong ORDER BY for a partitioned prefix key + NOPAD +# +SET NAMES utf8mb4; +CREATE TABLE t1 +( +id INT, +data VARCHAR(20), +KEY data_id (data,id) +) COLLATE utf8mb3_unicode_nopad_ci ENGINE=MyISAM +PARTITION BY RANGE COLUMNS (id) +( +PARTITION p10 VALUES LESS THAN (20), +PARTITION p20 VALUES LESS THAN MAXVALUE +); +INSERT INTO t1 VALUES (30, 'ss '), (10, 'ß '); +SELECT id FROM t1 WHERE data='ss ' ORDER BY id; +id +10 +30 +SELECT id FROM t1 WHERE data='ss ' ORDER BY id DESC; +id +30 +10 +ALTER TABLE t1 DROP KEY data_id, ADD KEY data_id2(data(10),id); +SELECT id FROM t1 WHERE data='ss ' ORDER BY id; +id +10 +30 +SELECT id FROM t1 WHERE data='ss ' ORDER BY id DESC; +id +30 +10 +DROP TABLE t1; +# +# End of 10.4 tests +# diff --git a/mysql-test/main/ctype_uca_partitions.test b/mysql-test/main/ctype_uca_partitions.test index 5734bb52008..81f1a091574 100644 --- a/mysql-test/main/ctype_uca_partitions.test +++ b/mysql-test/main/ctype_uca_partitions.test @@ -36,3 +36,35 @@ SELECT * FROM t1 PARTITION (p0) ORDER BY c1; SELECT * FROM t1 PARTITION (p1) ORDER BY c1; SELECT * FROM t1 PARTITION (p2) ORDER BY c1; DROP TABLE t1; + +--echo # +--echo # Start of 10.4 tests +--echo # + +--echo # +--echo # MDEV-30072 Wrong ORDER BY for a partitioned prefix key + NOPAD +--echo # + +SET NAMES utf8mb4; +CREATE TABLE t1 +( + id INT, + data VARCHAR(20), + KEY data_id (data,id) +) COLLATE utf8mb3_unicode_nopad_ci ENGINE=MyISAM +PARTITION BY RANGE COLUMNS (id) +( + PARTITION p10 VALUES LESS THAN (20), + PARTITION p20 VALUES LESS THAN MAXVALUE +); +INSERT INTO t1 VALUES (30, 'ss '), (10, 'ß '); +SELECT id FROM t1 WHERE data='ss ' ORDER BY id; +SELECT id FROM t1 WHERE data='ss ' ORDER BY id DESC; +ALTER TABLE t1 DROP KEY data_id, ADD KEY data_id2(data(10),id); +SELECT id FROM t1 WHERE data='ss ' ORDER BY id; +SELECT id FROM t1 WHERE data='ss ' ORDER BY id DESC; +DROP TABLE t1; + +--echo # +--echo # End of 10.4 tests +--echo # -- cgit v1.2.1 From ed2adc8c6f986f7e9c81d7a99f85cad0e2d46d80 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Thu, 6 Apr 2023 14:50:26 +0400 Subject: MDEV-28190 sql_mode makes MDEV-371 virtual column expressions nondeterministic This problem was fixed earlier by MDEV-27653. Adding MTR tests only. --- .../include/sql_mode_pad_char_to_full_length.inc | 31 +++++++ .../main/sql_mode_pad_char_to_full_length.result | 94 ++++++++++++++++++++++ .../main/sql_mode_pad_char_to_full_length.test | 19 +++++ .../r/sql_mode_pad_char_to_full_length.result | 51 ++++++++++++ .../innodb/t/sql_mode_pad_char_to_full_length.test | 18 +++++ 5 files changed, 213 insertions(+) create mode 100644 mysql-test/include/sql_mode_pad_char_to_full_length.inc create mode 100644 mysql-test/main/sql_mode_pad_char_to_full_length.result create mode 100644 mysql-test/main/sql_mode_pad_char_to_full_length.test create mode 100644 mysql-test/suite/innodb/r/sql_mode_pad_char_to_full_length.result create mode 100644 mysql-test/suite/innodb/t/sql_mode_pad_char_to_full_length.test diff --git a/mysql-test/include/sql_mode_pad_char_to_full_length.inc b/mysql-test/include/sql_mode_pad_char_to_full_length.inc new file mode 100644 index 00000000000..df03c4dbc28 --- /dev/null +++ b/mysql-test/include/sql_mode_pad_char_to_full_length.inc @@ -0,0 +1,31 @@ +--echo # +--echo # MDEV-28190 sql_mode makes MDEV-371 virtual column expressions nondeterministic +--echo # + +CREATE TABLE t1 (a INT,b CHAR(20)); +SHOW CREATE TABLE t1; +CREATE UNIQUE INDEX bi USING HASH ON t1 (b); +INSERT INTO t1 VALUES (0,0); +SET sql_mode='pad_char_to_full_length'; +DELETE FROM t1; +DROP TABLE t1; + + +SET sql_mode=''; +CREATE TABLE t1 (a INT,b CHAR(20)); +SHOW CREATE TABLE t1; +CREATE UNIQUE INDEX bi USING HASH ON t1 (b); +SET sql_mode='pad_char_to_full_length'; +INSERT INTO t1 VALUES (0,0); +DELETE FROM t1; +DROP TABLE t1; + + +SET sql_mode=''; +CREATE OR REPLACE TABLE t1 (a CHAR(20),b CHAR(20)); +SHOW CREATE TABLE t1; +CREATE UNIQUE INDEX bi USING HASH ON t1 (b); +INSERT INTO t1 VALUES (0,0); +SET sql_mode='pad_char_to_full_length'; +DELETE FROM t1; +DROP TABLE t1; diff --git a/mysql-test/main/sql_mode_pad_char_to_full_length.result b/mysql-test/main/sql_mode_pad_char_to_full_length.result new file mode 100644 index 00000000000..6f68aade613 --- /dev/null +++ b/mysql-test/main/sql_mode_pad_char_to_full_length.result @@ -0,0 +1,94 @@ +# +# Start of 10.4 tests +# +# +# MDEV-28190 sql_mode makes MDEV-371 virtual column expressions nondeterministic +# +SET default_storage_engine=MyISAM; +# +# MDEV-28190 sql_mode makes MDEV-371 virtual column expressions nondeterministic +# +CREATE TABLE t1 (a INT,b CHAR(20)); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` char(20) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +CREATE UNIQUE INDEX bi USING HASH ON t1 (b); +INSERT INTO t1 VALUES (0,0); +SET sql_mode='pad_char_to_full_length'; +DELETE FROM t1; +DROP TABLE t1; +SET sql_mode=''; +CREATE TABLE t1 (a INT,b CHAR(20)); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` char(20) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +CREATE UNIQUE INDEX bi USING HASH ON t1 (b); +SET sql_mode='pad_char_to_full_length'; +INSERT INTO t1 VALUES (0,0); +DELETE FROM t1; +DROP TABLE t1; +SET sql_mode=''; +CREATE OR REPLACE TABLE t1 (a CHAR(20),b CHAR(20)); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` char(20) DEFAULT NULL, + `b` char(20) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +CREATE UNIQUE INDEX bi USING HASH ON t1 (b); +INSERT INTO t1 VALUES (0,0); +SET sql_mode='pad_char_to_full_length'; +DELETE FROM t1; +DROP TABLE t1; +SET default_storage_engine=MEMORY; +# +# MDEV-28190 sql_mode makes MDEV-371 virtual column expressions nondeterministic +# +CREATE TABLE t1 (a INT,b CHAR(20)); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` char(20) DEFAULT NULL +) ENGINE=MEMORY DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +CREATE UNIQUE INDEX bi USING HASH ON t1 (b); +INSERT INTO t1 VALUES (0,0); +SET sql_mode='pad_char_to_full_length'; +DELETE FROM t1; +DROP TABLE t1; +SET sql_mode=''; +CREATE TABLE t1 (a INT,b CHAR(20)); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` char(20) DEFAULT NULL +) ENGINE=MEMORY DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +CREATE UNIQUE INDEX bi USING HASH ON t1 (b); +SET sql_mode='pad_char_to_full_length'; +INSERT INTO t1 VALUES (0,0); +DELETE FROM t1; +DROP TABLE t1; +SET sql_mode=''; +CREATE OR REPLACE TABLE t1 (a CHAR(20),b CHAR(20)); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` char(20) DEFAULT NULL, + `b` char(20) DEFAULT NULL +) ENGINE=MEMORY DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +CREATE UNIQUE INDEX bi USING HASH ON t1 (b); +INSERT INTO t1 VALUES (0,0); +SET sql_mode='pad_char_to_full_length'; +DELETE FROM t1; +DROP TABLE t1; +SET default_storage_engine=DEFAULT; +# +# End of 10.4 tests +# diff --git a/mysql-test/main/sql_mode_pad_char_to_full_length.test b/mysql-test/main/sql_mode_pad_char_to_full_length.test new file mode 100644 index 00000000000..4d492bc1b70 --- /dev/null +++ b/mysql-test/main/sql_mode_pad_char_to_full_length.test @@ -0,0 +1,19 @@ +--echo # +--echo # Start of 10.4 tests +--echo # + +--echo # +--echo # MDEV-28190 sql_mode makes MDEV-371 virtual column expressions nondeterministic +--echo # + +SET default_storage_engine=MyISAM; +--source include/sql_mode_pad_char_to_full_length.inc + +SET default_storage_engine=MEMORY; +--source include/sql_mode_pad_char_to_full_length.inc + +SET default_storage_engine=DEFAULT; + +--echo # +--echo # End of 10.4 tests +--echo # diff --git a/mysql-test/suite/innodb/r/sql_mode_pad_char_to_full_length.result b/mysql-test/suite/innodb/r/sql_mode_pad_char_to_full_length.result new file mode 100644 index 00000000000..09c1cf57497 --- /dev/null +++ b/mysql-test/suite/innodb/r/sql_mode_pad_char_to_full_length.result @@ -0,0 +1,51 @@ +SET default_storage_engine=InnoDB; +# +# Start of 10.4 tests +# +# +# MDEV-28190 sql_mode makes MDEV-371 virtual column expressions nondeterministic +# +# +# MDEV-28190 sql_mode makes MDEV-371 virtual column expressions nondeterministic +# +CREATE TABLE t1 (a INT,b CHAR(20)); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` char(20) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +CREATE UNIQUE INDEX bi USING HASH ON t1 (b); +INSERT INTO t1 VALUES (0,0); +SET sql_mode='pad_char_to_full_length'; +DELETE FROM t1; +DROP TABLE t1; +SET sql_mode=''; +CREATE TABLE t1 (a INT,b CHAR(20)); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL, + `b` char(20) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +CREATE UNIQUE INDEX bi USING HASH ON t1 (b); +SET sql_mode='pad_char_to_full_length'; +INSERT INTO t1 VALUES (0,0); +DELETE FROM t1; +DROP TABLE t1; +SET sql_mode=''; +CREATE OR REPLACE TABLE t1 (a CHAR(20),b CHAR(20)); +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` char(20) DEFAULT NULL, + `b` char(20) DEFAULT NULL +) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci +CREATE UNIQUE INDEX bi USING HASH ON t1 (b); +INSERT INTO t1 VALUES (0,0); +SET sql_mode='pad_char_to_full_length'; +DELETE FROM t1; +DROP TABLE t1; +# +# End of 10.4 tests +# diff --git a/mysql-test/suite/innodb/t/sql_mode_pad_char_to_full_length.test b/mysql-test/suite/innodb/t/sql_mode_pad_char_to_full_length.test new file mode 100644 index 00000000000..ba286c744d9 --- /dev/null +++ b/mysql-test/suite/innodb/t/sql_mode_pad_char_to_full_length.test @@ -0,0 +1,18 @@ +--source include/have_innodb.inc + +SET default_storage_engine=InnoDB; + +--echo # +--echo # Start of 10.4 tests +--echo # + +--echo # +--echo # MDEV-28190 sql_mode makes MDEV-371 virtual column expressions nondeterministic +--echo # + +--source include/sql_mode_pad_char_to_full_length.inc + + +--echo # +--echo # End of 10.4 tests +--echo # -- cgit v1.2.1 From 4daea2f8b69417881b4f123a252954f22b499df1 Mon Sep 17 00:00:00 2001 From: lilinjie <1136268146@qq.com> Date: Fri, 31 Mar 2023 14:11:04 +0800 Subject: fix typo Signed-off-by: lilinjie <1136268146@qq.com> --- sql/log.cc | 2 +- sql/log_event.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/log.cc b/sql/log.cc index d2cbb49e280..8c75ae847b3 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -408,7 +408,7 @@ private: Rows_log_event *m_pending; /* - Bit flags for what has been writting to cache. Used to + Bit flags for what has been writing to cache. Used to discard logs without any data changes. see enum_logged_status; */ diff --git a/sql/log_event.h b/sql/log_event.h index b12ee07b0e2..5cd303e288c 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -726,7 +726,7 @@ enum Log_event_type /* - Bit flags for what has been writting to cache. Used to + Bit flags for what has been writing to cache. Used to discard logs with table map events but not row events and nothing else important. This is stored by cache. */ -- cgit v1.2.1 From f83b7ae13d9619af29d74a27dd253b67aa0ee4fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 6 Apr 2023 07:50:23 +0300 Subject: MDEV-26175 : Assertion `! thd->in_sub_stmt' failed in bool trans_rollback_stmt(THD*) If we are inside stored function or trigger we should not commit or rollback current statement transaction. Signed-off-by: Julius Goryavsky --- mysql-test/suite/galera/r/mdev-26175.result | 24 ++++++++++++++++++++++ mysql-test/suite/galera/t/galera_sequences.test | 1 + mysql-test/suite/galera/t/mdev-26175.test | 27 +++++++++++++++++++++++++ sql/handler.cc | 8 +++++++- 4 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 mysql-test/suite/galera/r/mdev-26175.result create mode 100644 mysql-test/suite/galera/t/mdev-26175.test diff --git a/mysql-test/suite/galera/r/mdev-26175.result b/mysql-test/suite/galera/r/mdev-26175.result new file mode 100644 index 00000000000..f84244fe916 --- /dev/null +++ b/mysql-test/suite/galera/r/mdev-26175.result @@ -0,0 +1,24 @@ +connection node_2; +connection node_1; +connection node_1; +SET sql_mode="no_zero_date"; +SET GLOBAL wsrep_max_ws_rows=1; +CREATE TABLE t2 (a INT); +CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY) ENGINE=InnoDB; +CREATE TRIGGER tgr BEFORE INSERT ON t1 FOR EACH ROW INSERT INTO t2 VALUES (0); +INSERT INTO t1 VALUES (0),(1); +ERROR HY000: wsrep_max_ws_rows exceeded +SELECT * FROM t1; +a +SELECT * FROM t2; +a +connection node_2; +SELECT * FROM t1; +a +SELECT * FROM t2; +a +connection node_1; +SET sql_mode=DEFAULT; +SET GLOBAL wsrep_max_ws_rows=DEFAULT; +DROP TRIGGER tgr; +DROP TABLE t1, t2; diff --git a/mysql-test/suite/galera/t/galera_sequences.test b/mysql-test/suite/galera/t/galera_sequences.test index faa3b46d2a7..613823d83e9 100644 --- a/mysql-test/suite/galera/t/galera_sequences.test +++ b/mysql-test/suite/galera/t/galera_sequences.test @@ -1,4 +1,5 @@ --source include/galera_cluster.inc +--source include/have_innodb.inc # # MDEV-19353 : Alter Sequence do not replicate to another nodes with in Galera Cluster diff --git a/mysql-test/suite/galera/t/mdev-26175.test b/mysql-test/suite/galera/t/mdev-26175.test new file mode 100644 index 00000000000..1a3f1153e03 --- /dev/null +++ b/mysql-test/suite/galera/t/mdev-26175.test @@ -0,0 +1,27 @@ +--source include/galera_cluster.inc +--source include/have_innodb.inc + +# +# MDEV-26175 : Assertion `! thd->in_sub_stmt' failed in bool trans_rollback_stmt(THD*) +# +--connection node_1 +SET sql_mode="no_zero_date"; +SET GLOBAL wsrep_max_ws_rows=1; +CREATE TABLE t2 (a INT); +CREATE TABLE t1 (a INT NOT NULL PRIMARY KEY) ENGINE=InnoDB; +CREATE TRIGGER tgr BEFORE INSERT ON t1 FOR EACH ROW INSERT INTO t2 VALUES (0); + +--error ER_ERROR_DURING_COMMIT +INSERT INTO t1 VALUES (0),(1); +SELECT * FROM t1; +SELECT * FROM t2; + +--connection node_2 +SELECT * FROM t1; +SELECT * FROM t2; + +--connection node_1 +SET sql_mode=DEFAULT; +SET GLOBAL wsrep_max_ws_rows=DEFAULT; +DROP TRIGGER tgr; +DROP TABLE t1, t2; diff --git a/sql/handler.cc b/sql/handler.cc index 8ed8ee64880..7eaaaf63f00 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -6618,7 +6618,13 @@ static int wsrep_after_row(THD *thd) wsrep_thd_is_local(thd) && thd->wsrep_affected_rows > wsrep_max_ws_rows) { - trans_rollback_stmt(thd) || trans_rollback(thd); + /* + If we are inside stored function or trigger we should not commit or + rollback current statement transaction. See comment in ha_commit_trans() + call for more information. + */ + if (!thd->in_sub_stmt) + trans_rollback_stmt(thd) || trans_rollback(thd); my_message(ER_ERROR_DURING_COMMIT, "wsrep_max_ws_rows exceeded", MYF(0)); DBUG_RETURN(ER_ERROR_DURING_COMMIT); } -- cgit v1.2.1 From 7bcfa00a6a19ae74afe985213e09fc1f0c0540db Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Wed, 12 Apr 2023 11:40:46 +0400 Subject: MDEV-31039 mariadb-backup: remove global variables ds_data and ds_meta This is a non-functional change. simplifying the code logic: - removing global variables ds_data and ds_meta - passing these variables as parameters to functions instead - adding helper classes: Datasink_free_list and Backup_datasinks - moving some function accepting a ds_ctxt parameter as methods to ds_ctxt. --- extra/mariabackup/backup_copy.cc | 169 ++++++++++++++--------------- extra/mariabackup/backup_copy.h | 19 +--- extra/mariabackup/backup_mysql.cc | 32 +++--- extra/mariabackup/backup_mysql.h | 15 +-- extra/mariabackup/datasink.h | 29 +++++ extra/mariabackup/write_filt.cc | 14 ++- extra/mariabackup/write_filt.h | 3 +- extra/mariabackup/xtrabackup.cc | 219 ++++++++++++++++++++++---------------- extra/mariabackup/xtrabackup.h | 10 +- 9 files changed, 289 insertions(+), 221 deletions(-) diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index 608be7125bd..ffaf6dc98e4 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -73,9 +73,8 @@ bool binlog_locked; static void rocksdb_create_checkpoint(); static bool has_rocksdb_plugin(); -static void copy_or_move_dir(const char *from, const char *to, bool copy, bool allow_hardlinks); -static void rocksdb_backup_checkpoint(); -static void rocksdb_copy_back(); +static void rocksdb_backup_checkpoint(ds_ctxt *ds_data); +static void rocksdb_copy_back(ds_ctxt *ds_data); static bool is_abs_path(const char *path) { @@ -131,7 +130,7 @@ struct datadir_thread_ctxt_t { bool ret; }; -static bool backup_files_from_datadir(const char *dir_path); +static bool backup_files_from_datadir(ds_ctxt *ds_data, const char *dir_path); /************************************************************************ Retirn true if character if file separator */ @@ -804,7 +803,7 @@ if passes the rules for partial backup. @return true if file backed up or skipped successfully. */ static bool -datafile_copy_backup(const char *filepath, uint thread_n) +datafile_copy_backup(ds_ctxt *ds_data, const char *filepath, uint thread_n) { const char *ext_list[] = {"frm", "isl", "MYD", "MYI", "MAD", "MAI", "MRG", "TRG", "TRN", "ARM", "ARZ", "CSM", "CSV", "opt", "par", @@ -825,7 +824,7 @@ datafile_copy_backup(const char *filepath, uint thread_n) } if (filename_matches(filepath, ext_list)) { - return copy_file(ds_data, filepath, filepath, thread_n); + return ds_data->copy_file(filepath, filepath, thread_n); } return(true); @@ -866,7 +865,8 @@ datafile_rsync_backup(const char *filepath, bool save_to_list, FILE *f) return(true); } -bool backup_file_print_buf(const char *filename, const char *buf, int buf_len) +bool ds_ctxt_t::backup_file_print_buf(const char *filename, + const char *buf, int buf_len) { ds_file_t *dstfile = NULL; MY_STAT stat; /* unused for now */ @@ -877,7 +877,7 @@ bool backup_file_print_buf(const char *filename, const char *buf, int buf_len) stat.st_size = buf_len; stat.st_mtime = my_time(0); - dstfile = ds_open(ds_data, filename, &stat); + dstfile = ds_open(this, filename, &stat); if (dstfile == NULL) { msg("error: Can't open the destination stream for %s", filename); @@ -916,9 +916,9 @@ error_close: return true; }; -static bool -backup_file_vprintf(const char *filename, const char *fmt, va_list ap) +ds_ctxt_t::backup_file_vprintf(const char *filename, + const char *fmt, va_list ap) { char *buf = 0; int buf_len; @@ -929,7 +929,7 @@ backup_file_vprintf(const char *filename, const char *fmt, va_list ap) } bool -backup_file_printf(const char *filename, const char *fmt, ...) +ds_ctxt_t::backup_file_printf(const char *filename, const char *fmt, ...) { bool result; va_list ap; @@ -1055,16 +1055,15 @@ static int fix_win_file_permissions(const char *file) Copy file for backup/restore. @return true in case of success. */ bool -copy_file(ds_ctxt_t *datasink, - const char *src_file_path, - const char *dst_file_path, - uint thread_n) +ds_ctxt_t::copy_file(const char *src_file_path, + const char *dst_file_path, + uint thread_n) { char dst_name[FN_REFLEN]; ds_file_t *dstfile = NULL; datafile_cur_t cursor; xb_fil_cur_result_t res; - DBUG_ASSERT(datasink->datasink->remove); + DBUG_ASSERT(datasink->remove); const char *dst_path = (xtrabackup_copy_back || xtrabackup_move_back)? dst_file_path : trim_dotslash(dst_file_path); @@ -1075,7 +1074,7 @@ copy_file(ds_ctxt_t *datasink, strncpy(dst_name, cursor.rel_path, sizeof(dst_name)); - dstfile = ds_open(datasink, dst_path, &cursor.statinfo); + dstfile = ds_open(this, dst_path, &cursor.statinfo); if (dstfile == NULL) { msg(thread_n,"error: " "cannot open the destination stream for %s", dst_name); @@ -1112,7 +1111,7 @@ copy_file(ds_ctxt_t *datasink, error: datafile_close(&cursor); if (dstfile != NULL) { - datasink->datasink->remove(dstfile->path); + datasink->remove(dstfile->path); ds_close(dstfile); } @@ -1126,12 +1125,10 @@ error_close: Try to move file by renaming it. If source and destination are on different devices fall back to copy and unlink. @return true in case of success. */ -static bool -move_file(ds_ctxt_t *datasink, - const char *src_file_path, - const char *dst_file_path, - const char *dst_dir, uint thread_n) +ds_ctxt_t::move_file(const char *src_file_path, + const char *dst_file_path, + const char *dst_dir, uint thread_n) { char errbuf[MYSYS_STRERROR_SIZE]; char dst_file_path_abs[FN_REFLEN]; @@ -1158,7 +1155,7 @@ move_file(ds_ctxt_t *datasink, if (my_rename(src_file_path, dst_file_path_abs, MYF(0)) != 0) { if (my_errno == EXDEV) { /* Fallback to copy/unlink */ - if(!copy_file(datasink, src_file_path, + if(!copy_file(src_file_path, dst_file_path, thread_n)) return false; msg(thread_n,"Removing %s", src_file_path); @@ -1242,13 +1239,13 @@ Copy or move file depending on current mode. @return true in case of success. */ static bool -copy_or_move_file(const char *src_file_path, +copy_or_move_file(ds_ctxt *datasink0, const char *src_file_path, const char *dst_file_path, const char *dst_dir, uint thread_n, bool copy = xtrabackup_copy_back) { - ds_ctxt_t *datasink = ds_data; /* copy to datadir by default */ + ds_ctxt_t *datasink = datasink0; /* copy to datadir by default */ char filedir[FN_REFLEN]; size_t filedir_len; bool ret; @@ -1296,13 +1293,13 @@ copy_or_move_file(const char *src_file_path, } ret = (copy ? - copy_file(datasink, src_file_path, dst_file_path, thread_n) : - move_file(datasink, src_file_path, dst_file_path, + datasink->copy_file(src_file_path, dst_file_path, thread_n) : + datasink->move_file(src_file_path, dst_file_path, dst_dir, thread_n)); cleanup: - if (datasink != ds_data) { + if (datasink != datasink0) { ds_destroy(datasink); } @@ -1314,7 +1311,7 @@ cleanup: static bool -backup_files(const char *from, bool prep_mode) +backup_files(ds_ctxt *ds_data, const char *from, bool prep_mode) { char rsync_tmpfile_name[FN_REFLEN]; FILE *rsync_tmpfile = NULL; @@ -1352,7 +1349,7 @@ backup_files(const char *from, bool prep_mode) ret = datafile_rsync_backup(node.filepath, !prep_mode, rsync_tmpfile); } else { - ret = datafile_copy_backup(node.filepath, 1); + ret = datafile_copy_backup(ds_data, node.filepath, 1); } if (!ret) { msg("Failed to copy file %s", node.filepath); @@ -1363,7 +1360,7 @@ backup_files(const char *from, bool prep_mode) char path[FN_REFLEN]; snprintf(path, sizeof(path), "%s/db.opt", node.filepath); - if (!(ret = backup_file_printf( + if (!(ret = ds_data->backup_file_printf( trim_dotslash(path), "%s", ""))) { msg("Failed to create file %s", path); goto out; @@ -1452,7 +1449,6 @@ out: return(ret); } -void backup_fix_ddl(CorruptedPages &); lsn_t get_current_lsn(MYSQL *connection) { @@ -1477,7 +1473,8 @@ lsn_t get_current_lsn(MYSQL *connection) lsn_t server_lsn_after_lock; extern void backup_wait_for_lsn(lsn_t lsn); /** Start --backup */ -bool backup_start(CorruptedPages &corrupted_pages) +bool backup_start(ds_ctxt *ds_data, ds_ctxt *ds_meta, + CorruptedPages &corrupted_pages) { if (!opt_no_lock) { if (opt_safe_slave_backup) { @@ -1486,7 +1483,7 @@ bool backup_start(CorruptedPages &corrupted_pages) } } - if (!backup_files(fil_path_to_mysql_datadir, true)) { + if (!backup_files(ds_data, fil_path_to_mysql_datadir, true)) { return(false); } @@ -1498,11 +1495,11 @@ bool backup_start(CorruptedPages &corrupted_pages) server_lsn_after_lock = get_current_lsn(mysql_connection); } - if (!backup_files(fil_path_to_mysql_datadir, false)) { + if (!backup_files(ds_data, fil_path_to_mysql_datadir, false)) { return(false); } - if (!backup_files_from_datadir(fil_path_to_mysql_datadir)) { + if (!backup_files_from_datadir(ds_data, fil_path_to_mysql_datadir)) { return false; } @@ -1512,7 +1509,7 @@ bool backup_start(CorruptedPages &corrupted_pages) msg("Waiting for log copy thread to read lsn %llu", (ulonglong)server_lsn_after_lock); backup_wait_for_lsn(server_lsn_after_lock); - backup_fix_ddl(corrupted_pages); + corrupted_pages.backup_fix_ddl(ds_data, ds_meta); // There is no need to stop slave thread before coping non-Innodb data when // --no-lock option is used because --no-lock option requires that no DDL or @@ -1528,7 +1525,7 @@ bool backup_start(CorruptedPages &corrupted_pages) if (opt_slave_info) { lock_binlog_maybe(mysql_connection); - if (!write_slave_info(mysql_connection)) { + if (!write_slave_info(ds_data, mysql_connection)) { return(false); } } @@ -1540,7 +1537,7 @@ bool backup_start(CorruptedPages &corrupted_pages) avoid that is to have a single process, i.e. merge innobackupex and xtrabackup. */ if (opt_galera_info) { - if (!write_galera_info(mysql_connection)) { + if (!write_galera_info(ds_data, mysql_connection)) { return(false); } } @@ -1548,7 +1545,7 @@ bool backup_start(CorruptedPages &corrupted_pages) if (opt_binlog_info == BINLOG_INFO_ON) { lock_binlog_maybe(mysql_connection); - write_binlog_info(mysql_connection); + write_binlog_info(ds_data, mysql_connection); } if (have_flush_engine_logs && !opt_no_lock) { @@ -1585,20 +1582,20 @@ void backup_release() static const char *default_buffer_pool_file = "ib_buffer_pool"; /** Finish after backup_start() and backup_release() */ -bool backup_finish() +bool backup_finish(ds_ctxt *ds_data) { /* Copy buffer pool dump or LRU dump */ if (!opt_rsync && opt_galera_info) { if (buffer_pool_filename && file_exists(buffer_pool_filename)) { - copy_file(ds_data, buffer_pool_filename, default_buffer_pool_file, 0); + ds_data->copy_file(buffer_pool_filename, default_buffer_pool_file, 0); } if (file_exists("ib_lru_dump")) { - copy_file(ds_data, "ib_lru_dump", "ib_lru_dump", 0); + ds_data->copy_file("ib_lru_dump", "ib_lru_dump", 0); } } if (has_rocksdb_plugin()) { - rocksdb_backup_checkpoint(); + rocksdb_backup_checkpoint(ds_data); } msg("Backup created in directory '%s'", xtrabackup_target_dir); @@ -1610,11 +1607,11 @@ bool backup_finish() mysql_slave_position); } - if (!write_backup_config_file()) { + if (!write_backup_config_file(ds_data)) { return(false); } - if (!write_xtrabackup_info(mysql_connection, XTRABACKUP_INFO, + if (!write_xtrabackup_info(ds_data, mysql_connection, XTRABACKUP_INFO, opt_history != 0, true)) { return(false); } @@ -1681,6 +1678,7 @@ ibx_copy_incremental_over_full() bool ret = true; char path[FN_REFLEN]; int i; + ds_ctxt *ds_data= NULL; DBUG_ASSERT(!opt_galera_info); datadir_node_init(&node); @@ -1708,15 +1706,15 @@ ibx_copy_incremental_over_full() unlink(node.filepath_rel); } - if (!(ret = copy_file(ds_data, node.filepath, - node.filepath_rel, 1))) { + if (!(ret = ds_data->copy_file(node.filepath, + node.filepath_rel, 1))) { msg("Failed to copy file %s", node.filepath); goto cleanup; } } - if (!(ret = backup_files_from_datadir(xtrabackup_incremental_dir))) + if (!(ret = backup_files_from_datadir(ds_data, xtrabackup_incremental_dir))) goto cleanup; /* copy supplementary files */ @@ -1731,7 +1729,7 @@ ibx_copy_incremental_over_full() if (file_exists(sup_files[i])) { unlink(sup_files[i]); } - copy_file(ds_data, path, sup_files[i], 0); + ds_data->copy_file(path, sup_files[i], 0); } } @@ -1745,7 +1743,7 @@ ibx_copy_incremental_over_full() if (my_mkdir(ROCKSDB_BACKUP_DIR, 0777, MYF(0))) { die("my_mkdir failed for " ROCKSDB_BACKUP_DIR); } - copy_or_move_dir(path, ROCKSDB_BACKUP_DIR, true, true); + ds_data->copy_or_move_dir(path, ROCKSDB_BACKUP_DIR, true, true); } ibx_incremental_drop_databases(xtrabackup_target_dir, xtrabackup_incremental_dir); @@ -1894,7 +1892,7 @@ copy_back() dst_dir = dst_dir_buf.make(srv_undo_dir); - ds_data = ds_create(dst_dir, DS_TYPE_LOCAL); + ds_ctxt *ds_tmp = ds_create(dst_dir, DS_TYPE_LOCAL); for (uint i = 1; i <= TRX_SYS_MAX_UNDO_SPACES; i++) { char filename[20]; @@ -1902,14 +1900,14 @@ copy_back() if (!file_exists(filename)) { break; } - if (!(ret = copy_or_move_file(filename, filename, + if (!(ret = copy_or_move_file(ds_tmp, filename, filename, dst_dir, 1))) { goto cleanup; } } - ds_destroy(ds_data); - ds_data = NULL; + ds_destroy(ds_tmp); + ds_tmp = NULL; /* copy redo logs */ @@ -1918,7 +1916,7 @@ copy_back() /* --backup generates a single ib_logfile0, which we must copy if it exists. */ - ds_data = ds_create(dst_dir, DS_TYPE_LOCAL); + ds_tmp = ds_create(dst_dir, DS_TYPE_LOCAL); MY_STAT stat_arg; if (!my_stat("ib_logfile0", &stat_arg, MYF(0)) || !stat_arg.st_size) { /* After completed --prepare, redo log files are redundant. @@ -1932,17 +1930,17 @@ copy_back() dst_dir, i); unlink(filename); } - } else if (!(ret = copy_or_move_file("ib_logfile0", "ib_logfile0", + } else if (!(ret = copy_or_move_file(ds_tmp, "ib_logfile0", "ib_logfile0", dst_dir, 1))) { goto cleanup; } - ds_destroy(ds_data); + ds_destroy(ds_tmp); /* copy innodb system tablespace(s) */ dst_dir = dst_dir_buf.make(innobase_data_home_dir); - ds_data = ds_create(dst_dir, DS_TYPE_LOCAL); + ds_tmp = ds_create(dst_dir, DS_TYPE_LOCAL); for (Tablespace::const_iterator iter(srv_sys_space.begin()), end(srv_sys_space.end()); @@ -1950,16 +1948,16 @@ copy_back() ++iter) { const char *filename = base_name(iter->name()); - if (!(ret = copy_or_move_file(filename, iter->name(), + if (!(ret = copy_or_move_file(ds_tmp, filename, iter->name(), dst_dir, 1))) { goto cleanup; } } - ds_destroy(ds_data); + ds_destroy(ds_tmp); /* copy the rest of tablespaces */ - ds_data = ds_create(mysql_data_home, DS_TYPE_LOCAL); + ds_tmp = ds_create(mysql_data_home, DS_TYPE_LOCAL); it = datadir_iter_new(".", false); @@ -2046,7 +2044,7 @@ copy_back() continue; } - if (!(ret = copy_or_move_file(node.filepath, node.filepath_rel, + if (!(ret = copy_or_move_file(ds_tmp, node.filepath, node.filepath_rel, mysql_data_home, 1))) { goto cleanup; } @@ -2056,12 +2054,12 @@ copy_back() if (file_exists(default_buffer_pool_file) && innobase_buffer_pool_filename) { - copy_or_move_file(default_buffer_pool_file, + copy_or_move_file(ds_tmp, default_buffer_pool_file, innobase_buffer_pool_filename, mysql_data_home, 0); } - rocksdb_copy_back(); + rocksdb_copy_back(ds_tmp); cleanup: if (it != NULL) { @@ -2070,11 +2068,11 @@ cleanup: datadir_node_free(&node); - if (ds_data != NULL) { - ds_destroy(ds_data); + if (ds_tmp != NULL) { + ds_destroy(ds_tmp); } - ds_data = NULL; + ds_tmp = NULL; sync_check_close(); return(ret); @@ -2182,7 +2180,7 @@ decrypt_decompress() } /* copy the rest of tablespaces */ - ds_data = ds_create(".", DS_TYPE_LOCAL); + ds_ctxt *ds_tmp = ds_create(".", DS_TYPE_LOCAL); it = datadir_iter_new(".", false); @@ -2195,11 +2193,11 @@ decrypt_decompress() datadir_iter_free(it); } - if (ds_data != NULL) { - ds_destroy(ds_data); + if (ds_tmp != NULL) { + ds_destroy(ds_tmp); } - ds_data = NULL; + ds_tmp = NULL; sync_check_close(); @@ -2211,7 +2209,7 @@ decrypt_decompress() Do not copy the Innodb files (ibdata1, redo log files), as this is done in a separate step. */ -static bool backup_files_from_datadir(const char *dir_path) +static bool backup_files_from_datadir(ds_ctxt *ds_data, const char *dir_path) { os_file_dir_t dir = os_file_opendir(dir_path); if (dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) return false; @@ -2241,7 +2239,7 @@ static bool backup_files_from_datadir(const char *dir_path) std::string full_path(dir_path); full_path.append(1, OS_PATH_SEPARATOR).append(info.name); - if (!(ret = copy_file(ds_data, full_path.c_str() , info.name, 1))) + if (!(ret = ds_data->copy_file(full_path.c_str() , info.name, 1))) break; } os_file_closedir(dir); @@ -2291,13 +2289,14 @@ static char *trim_trailing_dir_sep(char *path) Create a file hardlink. @return true on success, false on error. */ -static bool make_hardlink(const char *from_path, const char *to_path) +bool +ds_ctxt_t::make_hardlink(const char *from_path, const char *to_path) { DBUG_EXECUTE_IF("no_hardlinks", return false;); char to_path_full[FN_REFLEN]; if (!is_abs_path(to_path)) { - fn_format(to_path_full, to_path, ds_data->root, "", MYF(MY_RELATIVE_PATH)); + fn_format(to_path_full, to_path, root, "", MYF(MY_RELATIVE_PATH)); } else { @@ -2318,7 +2317,9 @@ static bool make_hardlink(const char *from_path, const char *to_path) Has optimization that allows to use hardlinks when possible (source and destination are directories on the same device) */ -static void copy_or_move_dir(const char *from, const char *to, bool do_copy, bool allow_hardlinks) +void +ds_ctxt_t::copy_or_move_dir(const char *from, const char *to, + bool do_copy, bool allow_hardlinks) { datadir_node_t node; datadir_node_init(&node); @@ -2346,8 +2347,8 @@ static void copy_or_move_dir(const char *from, const char *to, bool do_copy, boo if (!rc) { rc = (do_copy ? - copy_file(ds_data, from_path, to_path, 1) : - move_file(ds_data, from_path, node.filepath_rel, + copy_file(from_path, to_path, 1) : + move_file(from_path, node.filepath_rel, to, 1)); } if (!rc) @@ -2444,7 +2445,7 @@ static void rocksdb_create_checkpoint() remove temp.checkpoint directory (in server's datadir) and release user level lock acquired inside rocksdb_create_checkpoint(). */ -static void rocksdb_backup_checkpoint() +static void rocksdb_backup_checkpoint(ds_ctxt *ds_data) { msg("Backing up rocksdb files."); char rocksdb_backup_dir[FN_REFLEN]; @@ -2456,7 +2457,7 @@ static void rocksdb_backup_checkpoint() die("Can't create rocksdb backup directory %s", rocksdb_backup_dir); } } - copy_or_move_dir(rocksdb_checkpoint_dir, ROCKSDB_BACKUP_DIR, true, backup_to_directory); + ds_data->copy_or_move_dir(rocksdb_checkpoint_dir, ROCKSDB_BACKUP_DIR, true, backup_to_directory); rocksdb_remove_checkpoint_directory(); rocksdb_unlock_checkpoint(); } @@ -2464,7 +2465,7 @@ static void rocksdb_backup_checkpoint() /* Copies #rocksdb directory to the $rockdb_data_dir, on copy-back */ -static void rocksdb_copy_back() { +static void rocksdb_copy_back(ds_ctxt *ds_data) { if (access(ROCKSDB_BACKUP_DIR, 0)) return; char rocksdb_home_dir[FN_REFLEN]; @@ -2476,5 +2477,5 @@ static void rocksdb_copy_back() { xb_rocksdb_datadir?trim_dotslash(xb_rocksdb_datadir): ROCKSDB_BACKUP_DIR); } mkdirp(rocksdb_home_dir, 0777, MYF(0)); - copy_or_move_dir(ROCKSDB_BACKUP_DIR, rocksdb_home_dir, xtrabackup_copy_back, xtrabackup_copy_back); + ds_data->copy_or_move_dir(ROCKSDB_BACKUP_DIR, rocksdb_home_dir, xtrabackup_copy_back, xtrabackup_copy_back); } diff --git a/extra/mariabackup/backup_copy.h b/extra/mariabackup/backup_copy.h index 62b2b1bc232..b4a323f2e89 100644 --- a/extra/mariabackup/backup_copy.h +++ b/extra/mariabackup/backup_copy.h @@ -14,30 +14,18 @@ extern bool binlog_locked; -bool -backup_file_printf(const char *filename, const char *fmt, ...) - ATTRIBUTE_FORMAT(printf, 2, 0); - /************************************************************************ Return true if first and second arguments are the same path. */ bool equal_paths(const char *first, const char *second); -/************************************************************************ -Copy file for backup/restore. -@return true in case of success. */ -bool -copy_file(ds_ctxt_t *datasink, - const char *src_file_path, - const char *dst_file_path, - uint thread_n); - /** Start --backup */ -bool backup_start(CorruptedPages &corrupted_pages); +bool backup_start(ds_ctxt *ds_data, ds_ctxt *ds_meta, + CorruptedPages &corrupted_pages); /** Release resources after backup_start() */ void backup_release(); /** Finish after backup_start() and backup_release() */ -bool backup_finish(); +bool backup_finish(ds_ctxt *ds_data); bool apply_log_finish(); bool @@ -51,6 +39,5 @@ directory_exists(const char *dir, bool create); lsn_t get_current_lsn(MYSQL *connection); -bool backup_file_print_buf(const char *filename, const char *buf, int buf_len); #endif diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index b895f8c4561..6003bfb36c4 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -1383,7 +1383,7 @@ variable. @returns true on success */ bool -write_slave_info(MYSQL *connection) +write_slave_info(ds_ctxt *datasink, MYSQL *connection) { String sql, comment; bool show_all_slaves_status= false; @@ -1413,7 +1413,8 @@ write_slave_info(MYSQL *connection) } mysql_slave_position= strdup(comment.c_ptr()); - return backup_file_print_buf(XTRABACKUP_SLAVE_INFO, sql.ptr(), sql.length()); + return datasink->backup_file_print_buf(XTRABACKUP_SLAVE_INFO, + sql.ptr(), sql.length()); } @@ -1421,7 +1422,7 @@ write_slave_info(MYSQL *connection) Retrieves MySQL Galera and saves it in a file. It also prints it to stdout. */ bool -write_galera_info(MYSQL *connection) +write_galera_info(ds_ctxt *datasink, MYSQL *connection) { char *state_uuid = NULL, *state_uuid55 = NULL; char *last_committed = NULL, *last_committed55 = NULL; @@ -1453,12 +1454,12 @@ write_galera_info(MYSQL *connection) goto cleanup; } - result = backup_file_printf(XTRABACKUP_GALERA_INFO, + result = datasink->backup_file_printf(XTRABACKUP_GALERA_INFO, "%s:%s\n", state_uuid ? state_uuid : state_uuid55, last_committed ? last_committed : last_committed55); if (result) { - write_current_binlog_file(connection); + write_current_binlog_file(datasink, connection); } cleanup: @@ -1472,7 +1473,7 @@ cleanup: Flush and copy the current binary log file into the backup, if GTID is enabled */ bool -write_current_binlog_file(MYSQL *connection) +write_current_binlog_file(ds_ctxt *datasink, MYSQL *connection) { char *executed_gtid_set = NULL; char *gtid_binlog_state = NULL; @@ -1542,7 +1543,7 @@ write_current_binlog_file(MYSQL *connection) snprintf(filepath, sizeof(filepath), "%s%c%s", log_bin_dir, FN_LIBCHAR, log_bin_file); - result = copy_file(ds_data, filepath, log_bin_file, 0); + result = datasink->copy_file(filepath, log_bin_file, 0); } cleanup: @@ -1558,7 +1559,7 @@ cleanup: Retrieves MySQL binlog position and saves it in a file. It also prints it to stdout. */ bool -write_binlog_info(MYSQL *connection) +write_binlog_info(ds_ctxt *datasink, MYSQL *connection) { char *filename = NULL; char *position = NULL; @@ -1603,14 +1604,14 @@ write_binlog_info(MYSQL *connection) "filename '%s', position '%s', " "GTID of the last change '%s'", filename, position, gtid) != -1); - result = backup_file_printf(XTRABACKUP_BINLOG_INFO, + result = datasink->backup_file_printf(XTRABACKUP_BINLOG_INFO, "%s\t%s\t%s\n", filename, position, gtid); } else { ut_a(asprintf(&mysql_binlog_position, "filename '%s', position '%s'", filename, position) != -1); - result = backup_file_printf(XTRABACKUP_BINLOG_INFO, + result = datasink->backup_file_printf(XTRABACKUP_BINLOG_INFO, "%s\t%s\n", filename, position); } @@ -1650,8 +1651,9 @@ PERCONA_SCHEMA.xtrabackup_history and writes a new history record to the table containing all the history info particular to the just completed backup. */ bool -write_xtrabackup_info(MYSQL *connection, const char * filename, bool history, - bool stream) +write_xtrabackup_info(ds_ctxt *datasink, + MYSQL *connection, const char * filename, bool history, + bool stream) { bool result = true; @@ -1727,7 +1729,7 @@ write_xtrabackup_info(MYSQL *connection, const char * filename, bool history, } if (stream) { - backup_file_printf(filename, "%s", buf); + datasink->backup_file_printf(filename, "%s", buf); } else { fp = fopen(filename, "w"); if (!fp) { @@ -1848,9 +1850,9 @@ static std::string make_local_paths(const char *data_file_path) return buf.str(); } -bool write_backup_config_file() +bool write_backup_config_file(ds_ctxt *datasink) { - int rc= backup_file_printf("backup-my.cnf", + int rc= datasink->backup_file_printf("backup-my.cnf", "# This MySQL options file was generated by innobackupex.\n\n" "# The MySQL server\n" "[mysqld]\n" diff --git a/extra/mariabackup/backup_mysql.h b/extra/mariabackup/backup_mysql.h index b61fa2362c6..d80f3bb7bc1 100644 --- a/extra/mariabackup/backup_mysql.h +++ b/extra/mariabackup/backup_mysql.h @@ -62,17 +62,18 @@ void unlock_all(MYSQL *connection); bool -write_current_binlog_file(MYSQL *connection); +write_current_binlog_file(ds_ctxt *datasink, MYSQL *connection); bool -write_binlog_info(MYSQL *connection); +write_binlog_info(ds_ctxt *datasink, MYSQL *connection); bool -write_xtrabackup_info(MYSQL *connection, const char * filename, bool history, - bool stream); +write_xtrabackup_info(ds_ctxt *datasink, + MYSQL *connection, const char * filename, bool history, + bool stream); bool -write_backup_config_file(); +write_backup_config_file(ds_ctxt *datasink); bool lock_binlog_maybe(MYSQL *connection); @@ -84,10 +85,10 @@ bool wait_for_safe_slave(MYSQL *connection); bool -write_galera_info(MYSQL *connection); +write_galera_info(ds_ctxt *datasink, MYSQL *connection); bool -write_slave_info(MYSQL *connection); +write_slave_info(ds_ctxt *datasink, MYSQL *connection); #endif diff --git a/extra/mariabackup/datasink.h b/extra/mariabackup/datasink.h index 4bede4ec9e7..57468e0c9c7 100644 --- a/extra/mariabackup/datasink.h +++ b/extra/mariabackup/datasink.h @@ -37,6 +37,35 @@ typedef struct ds_ctxt { char *root; void *ptr; struct ds_ctxt *pipe_ctxt; + /* + Copy file for backup/restore. + @return true in case of success. + */ + bool copy_file(const char *src_file_path, + const char *dst_file_path, + uint thread_n); + + bool move_file(const char *src_file_path, + const char *dst_file_path, + const char *dst_dir, + uint thread_n); + + bool make_hardlink(const char *from_path, const char *to_path); + + void copy_or_move_dir(const char *from, const char *to, + bool do_copy, bool allow_hardlinks); + + bool backup_file_vprintf(const char *filename, + const char *fmt, va_list ap); + + bool backup_file_print_buf(const char *filename, + const char *buf, + int buf_len); + + bool backup_file_printf(const char *filename, + const char *fmt, ...) + ATTRIBUTE_FORMAT(printf, 2, 0); + } ds_ctxt_t; typedef struct { diff --git a/extra/mariabackup/write_filt.cc b/extra/mariabackup/write_filt.cc index c144a60e378..dd77a9bb953 100644 --- a/extra/mariabackup/write_filt.cc +++ b/extra/mariabackup/write_filt.cc @@ -31,7 +31,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA /************************************************************************ Write-through page write filter. */ -static my_bool wf_wt_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, +static my_bool wf_wt_init(ds_ctxt *ds_meta, + xb_write_filt_ctxt_t *ctxt, char *dst_name, xb_fil_cur_t *cursor, CorruptedPages *corrupted_pages); static my_bool wf_wt_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile); @@ -44,7 +45,8 @@ xb_write_filt_t wf_write_through = { /************************************************************************ Incremental page write filter. */ -static my_bool wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, +static my_bool wf_incremental_init(ds_ctxt *ds_meta, + xb_write_filt_ctxt_t *ctxt, char *dst_name, xb_fil_cur_t *cursor, CorruptedPages *corrupted_pages); static my_bool wf_incremental_process(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile); @@ -64,7 +66,8 @@ Initialize incremental page write filter. @return TRUE on success, FALSE on error. */ static my_bool -wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, +wf_incremental_init(ds_ctxt *ds_meta, + xb_write_filt_ctxt_t *ctxt, char *dst_name, xb_fil_cur_t *cursor, CorruptedPages *corrupted_pages) { char meta_name[FN_REFLEN]; @@ -88,7 +91,7 @@ wf_incremental_init(xb_write_filt_ctxt_t *ctxt, char *dst_name, XB_DELTA_INFO_SUFFIX); const xb_delta_info_t info(cursor->page_size, cursor->zip_size, cursor->space_id); - if (!xb_write_delta_metadata(meta_name, &info)) { + if (!xb_write_delta_metadata(ds_meta, meta_name, &info)) { msg(cursor->thread_n,"Error: " "failed to write meta info for %s", cursor->rel_path); @@ -195,7 +198,8 @@ Initialize the write-through page write filter. @return TRUE on success, FALSE on error. */ static my_bool -wf_wt_init(xb_write_filt_ctxt_t *ctxt, char *dst_name __attribute__((unused)), +wf_wt_init(ds_ctxt *ds_meta __attribute__((unused)), + xb_write_filt_ctxt_t *ctxt, char *dst_name __attribute__((unused)), xb_fil_cur_t *cursor, CorruptedPages *) { ctxt->cursor = cursor; diff --git a/extra/mariabackup/write_filt.h b/extra/mariabackup/write_filt.h index 6c3ef24291f..a0ce0778a7f 100644 --- a/extra/mariabackup/write_filt.h +++ b/extra/mariabackup/write_filt.h @@ -45,7 +45,8 @@ typedef struct { typedef struct { - my_bool (*init)(xb_write_filt_ctxt_t *ctxt, char *dst_name, + my_bool (*init)(ds_ctxt *ds_meta, + xb_write_filt_ctxt_t *ctxt, char *dst_name, xb_fil_cur_t *cursor, CorruptedPages *corrupted_pages); my_bool (*process)(xb_write_filt_ctxt_t *ctxt, ds_file_t *dstfile); my_bool (*finalize)(xb_write_filt_ctxt_t *, ds_file_t *dstfile); diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index ad35749d0ff..ee12034c910 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -291,10 +291,66 @@ char *xb_plugin_dir; char *xb_plugin_load; my_bool xb_close_files; -/* Datasinks */ -ds_ctxt_t *ds_data = NULL; -ds_ctxt_t *ds_meta = NULL; -ds_ctxt_t *ds_redo = NULL; + +class Datasink_free_list +{ +protected: + /* + Simple datasink creation tracking... + add datasinks in the reverse order you want them destroyed. + */ +#define XTRABACKUP_MAX_DATASINKS 10 + ds_ctxt_t *m_datasinks_to_destroy[XTRABACKUP_MAX_DATASINKS]; + uint m_actual_datasinks_to_destroy; +public: + Datasink_free_list() + :m_actual_datasinks_to_destroy(0) + { } + + void add_datasink_to_destroy(ds_ctxt_t *ds) + { + xb_ad(m_actual_datasinks_to_destroy < XTRABACKUP_MAX_DATASINKS); + m_datasinks_to_destroy[m_actual_datasinks_to_destroy] = ds; + m_actual_datasinks_to_destroy++; + } + + /* + Destroy datasinks. + Destruction is done in the specific order to not violate their order in the + pipeline so that each datasink is able to flush data down the pipeline. + */ + void destroy() + { + for (uint i= m_actual_datasinks_to_destroy; i > 0; i--) + { + ds_destroy(m_datasinks_to_destroy[i - 1]); + m_datasinks_to_destroy[i - 1] = NULL; + } + } +}; + + +class Backup_datasinks: public Datasink_free_list +{ +public: + ds_ctxt_t *m_data; + ds_ctxt_t *m_meta; + ds_ctxt_t *m_redo; + + Backup_datasinks() + :m_data(NULL), + m_meta(NULL), + m_redo(NULL) + { } + void init(); + void destroy() + { + Datasink_free_list::destroy(); + *this= Backup_datasinks(); + } + bool backup_low(); +}; + static bool innobackupex_mode = false; @@ -440,7 +496,8 @@ void CorruptedPages::rename_space(ulint space_id, const std::string &new_name) ut_a(!pthread_mutex_unlock(&m_mutex)); } -bool CorruptedPages::print_to_file(const char *filename) const +bool CorruptedPages::print_to_file(ds_ctxt *ds_data, + const char *filename) const { std::ostringstream out; ut_a(!pthread_mutex_lock(&m_mutex)); @@ -468,8 +525,8 @@ bool CorruptedPages::print_to_file(const char *filename) const out << "\n"; } ut_a(!pthread_mutex_unlock(&m_mutex)); - if (xtrabackup_backup) - return backup_file_print_buf(filename, out.str().c_str(), + if (ds_data) + return ds_data->backup_file_print_buf(filename, out.str().c_str(), static_cast(out.str().size())); std::ofstream outfile; outfile.open(filename); @@ -587,19 +644,6 @@ void CorruptedPages::zero_out_free_pages() ut_free(buf); } -/* Simple datasink creation tracking...add datasinks in the reverse order you -want them destroyed. */ -#define XTRABACKUP_MAX_DATASINKS 10 -static ds_ctxt_t *datasinks[XTRABACKUP_MAX_DATASINKS]; -static uint actual_datasinks = 0; -static inline -void -xtrabackup_add_datasink(ds_ctxt_t *ds) -{ - xb_ad(actual_datasinks < XTRABACKUP_MAX_DATASINKS); - datasinks[actual_datasinks] = ds; actual_datasinks++; -} - typedef void (*process_single_tablespace_func_t)(const char *dirname, const char *filname, bool is_remote, @@ -955,6 +999,7 @@ typedef struct { pthread_mutex_t* count_mutex; os_thread_id_t id; CorruptedPages *corrupted_pages; + Backup_datasinks *datasinks; } data_thread_ctxt_t; /* ======== for option and variables ======== */ @@ -2456,7 +2501,8 @@ xb_read_delta_metadata(const char *filepath, xb_delta_info_t *info) Write meta info for an incremental delta. @return TRUE on success, FALSE on failure. */ my_bool -xb_write_delta_metadata(const char *filename, const xb_delta_info_t *info) +xb_write_delta_metadata(ds_ctxt *ds_meta, + const char *filename, const xb_delta_info_t *info) { ds_file_t *f; char buf[64]; @@ -2771,7 +2817,9 @@ xb_get_copy_action(const char *dflt) for full backup, pages filter for incremental backup, etc. @return FALSE on success and TRUE on error */ -static my_bool xtrabackup_copy_datafile(fil_node_t *node, uint thread_n, +static my_bool xtrabackup_copy_datafile(ds_ctxt *ds_data, + ds_ctxt *ds_meta, + fil_node_t *node, uint thread_n, const char *dest_name, const xb_write_filt_t &write_filter, CorruptedPages &corrupted_pages) @@ -2839,7 +2887,7 @@ static my_bool xtrabackup_copy_datafile(fil_node_t *node, uint thread_n, ut_a(write_filter.process != NULL); if (write_filter.init != NULL && - !write_filter.init(&write_filt_ctxt, dst_name, &cursor, + !write_filter.init(ds_meta, &write_filt_ctxt, dst_name, &cursor, opt_log_innodb_page_corruption ? &corrupted_pages : NULL)) { msg (thread_n, "mariabackup: error: failed to initialize page write filter."); goto error; @@ -3204,7 +3252,8 @@ DECLARE_THREAD(data_copy_thread_func)( DBUG_EXECUTE_FOR_KEY("wait_innodb_redo_before_copy", node->space->name, backup_wait_for_lsn(get_current_lsn(mysql_connection));); /* copy the datafile */ - if (xtrabackup_copy_datafile(node, num, NULL, + if (xtrabackup_copy_datafile(ctxt->datasinks->m_data, + ctxt->datasinks->m_meta, node, num, NULL, xtrabackup_incremental ? wf_incremental : wf_write_through, *ctxt->corrupted_pages)) die("failed to copy datafile."); @@ -3230,22 +3279,21 @@ Otherwise (i.e. when streaming in the 'tar' format) we need 2 separate datasinks for the data stream (and don't allow parallel data copying) and for metainfo files (including ib_logfile0). The second datasink writes to temporary files first, and then streams them in a serialized way when closed. */ -static void -xtrabackup_init_datasinks(void) +void Backup_datasinks::init() { /* Start building out the pipelines from the terminus back */ if (xtrabackup_stream) { /* All streaming goes to stdout */ - ds_data = ds_meta = ds_redo = ds_create(xtrabackup_target_dir, - DS_TYPE_STDOUT); + m_data = m_meta = m_redo = ds_create(xtrabackup_target_dir, + DS_TYPE_STDOUT); } else { /* Local filesystem */ - ds_data = ds_meta = ds_redo = ds_create(xtrabackup_target_dir, - DS_TYPE_LOCAL); + m_data = m_meta = m_redo = ds_create(xtrabackup_target_dir, + DS_TYPE_LOCAL); } /* Track it for destruction */ - xtrabackup_add_datasink(ds_data); + add_datasink_to_destroy(m_data); /* Stream formatting */ if (xtrabackup_stream) { @@ -3254,66 +3302,50 @@ xtrabackup_init_datasinks(void) ut_a(xtrabackup_stream_fmt == XB_STREAM_FMT_XBSTREAM); ds = ds_create(xtrabackup_target_dir, DS_TYPE_XBSTREAM); - xtrabackup_add_datasink(ds); + add_datasink_to_destroy(ds); - ds_set_pipe(ds, ds_data); - ds_data = ds; + ds_set_pipe(ds, m_data); + m_data = ds; - ds_redo = ds_meta = ds_data; + m_redo = m_meta = m_data; } - /* Compression for ds_data and ds_redo */ + /* Compression for m_data and m_redo */ if (xtrabackup_compress) { ds_ctxt_t *ds; /* Use a 1 MB buffer for compressed output stream */ ds = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER); ds_buffer_set_size(ds, 1024 * 1024); - xtrabackup_add_datasink(ds); - ds_set_pipe(ds, ds_data); - if (ds_data != ds_redo) { - ds_data = ds; + add_datasink_to_destroy(ds); + ds_set_pipe(ds, m_data); + if (m_data != m_redo) { + m_data = ds; ds = ds_create(xtrabackup_target_dir, DS_TYPE_BUFFER); ds_buffer_set_size(ds, 1024 * 1024); - xtrabackup_add_datasink(ds); - ds_set_pipe(ds, ds_redo); - ds_redo = ds; + add_datasink_to_destroy(ds); + ds_set_pipe(ds, m_redo); + m_redo = ds; } else { - ds_redo = ds_data = ds; + m_redo = m_data = ds; } ds = ds_create(xtrabackup_target_dir, DS_TYPE_COMPRESS); - xtrabackup_add_datasink(ds); - ds_set_pipe(ds, ds_data); - if (ds_data != ds_redo) { - ds_data = ds; + add_datasink_to_destroy(ds); + ds_set_pipe(ds, m_data); + if (m_data != m_redo) { + m_data = ds; ds = ds_create(xtrabackup_target_dir, DS_TYPE_COMPRESS); - xtrabackup_add_datasink(ds); - ds_set_pipe(ds, ds_redo); - ds_redo = ds; + add_datasink_to_destroy(ds); + ds_set_pipe(ds, m_redo); + m_redo = ds; } else { - ds_redo = ds_data = ds; + m_redo = m_data = ds; } } } -/************************************************************************ -Destroy datasinks. - -Destruction is done in the specific order to not violate their order in the -pipeline so that each datasink is able to flush data down the pipeline. */ -static void xtrabackup_destroy_datasinks(void) -{ - for (uint i = actual_datasinks; i > 0; i--) { - ds_destroy(datasinks[i-1]); - datasinks[i-1] = NULL; - } - ds_data = NULL; - ds_meta = NULL; - ds_redo = NULL; -} - #define SRV_MAX_N_PENDING_SYNC_IOS 100 /** Initialize the tablespace cache subsystem. */ @@ -4447,7 +4479,7 @@ static void stop_backup_threads() /** Implement the core of --backup @return whether the operation succeeded */ -static bool xtrabackup_backup_low() +bool Backup_datasinks::backup_low() { ut_ad(!metadata_to_lsn); @@ -4499,7 +4531,7 @@ static bool xtrabackup_backup_low() } metadata_last_lsn = log_copy_scanned_lsn; - if (!xtrabackup_stream_metadata(ds_meta)) { + if (!xtrabackup_stream_metadata(m_meta)) { msg("Error: failed to stream metadata."); return false; } @@ -4515,7 +4547,8 @@ static bool xtrabackup_backup_low() } sprintf(filename, "%s/%s", xtrabackup_extra_lsndir, XTRABACKUP_INFO); - if (!write_xtrabackup_info(mysql_connection, filename, false, false)) { + if (!write_xtrabackup_info(m_data, + mysql_connection, filename, false, false)) { msg("Error: failed to write info " "to '%s'.", filename); return false; @@ -4535,6 +4568,7 @@ static bool xtrabackup_backup_func() pthread_mutex_t count_mutex; CorruptedPages corrupted_pages; data_thread_ctxt_t *data_threads; + Backup_datasinks backup_datasinks; pthread_mutex_init(&backup_mutex, NULL); pthread_cond_init(&scanned_lsn_cond, NULL); @@ -4686,7 +4720,7 @@ reread_log_header: log_mutex_exit(); - xtrabackup_init_datasinks(); + backup_datasinks.init(); if (!select_history()) { goto fail; @@ -4694,7 +4728,7 @@ reread_log_header: /* open the log file */ memset(&stat_info, 0, sizeof(MY_STAT)); - dst_log_file = ds_open(ds_redo, "ib_logfile0", &stat_info); + dst_log_file = ds_open(backup_datasinks.m_redo, "ib_logfile0", &stat_info); if (dst_log_file == NULL) { msg("Error: failed to open the target stream for " "'ib_logfile0'."); @@ -4812,6 +4846,7 @@ fail_before_log_copying_thread_start: data_threads[i].count = &count; data_threads[i].count_mutex = &count_mutex; data_threads[i].corrupted_pages = &corrupted_pages; + data_threads[i].datasinks= &backup_datasinks; os_thread_create(data_copy_thread_func, data_threads + i, &data_threads[i].id); } @@ -4832,10 +4867,13 @@ fail_before_log_copying_thread_start: datafiles_iter_free(it); } - bool ok = backup_start(corrupted_pages); + DBUG_ASSERT(backup_datasinks.m_data); + DBUG_ASSERT(backup_datasinks.m_meta); + bool ok = backup_start(backup_datasinks.m_data, + backup_datasinks.m_meta, corrupted_pages); if (ok) { - ok = xtrabackup_backup_low(); + ok = backup_datasinks.backup_low(); backup_release(); @@ -4845,12 +4883,13 @@ fail_before_log_copying_thread_start: ); if (ok) { - backup_finish(); + backup_finish(backup_datasinks.m_data); } } if (opt_log_innodb_page_corruption) - ok = corrupted_pages.print_to_file(MB_CORRUPTED_PAGES_FILE); + ok = corrupted_pages.print_to_file(backup_datasinks.m_data, + MB_CORRUPTED_PAGES_FILE); if (!ok) { goto fail; @@ -4859,7 +4898,7 @@ fail_before_log_copying_thread_start: if (changed_page_bitmap) { xb_page_bitmap_deinit(changed_page_bitmap); } - xtrabackup_destroy_datasinks(); + backup_datasinks.destroy(); msg("Redo log (from LSN " LSN_PF " to " LSN_PF ") was copied.", checkpoint_lsn_start, log_copy_scanned_lsn); @@ -4906,7 +4945,7 @@ FTWRL. This ensures consistent backup in presence of DDL. It is the responsibility of the prepare phase to deal with .new, .ren, and .del files. */ -void backup_fix_ddl(CorruptedPages &corrupted_pages) +void CorruptedPages::backup_fix_ddl(ds_ctxt *ds_data, ds_ctxt *ds_meta) { std::set new_tables; std::set dropped_tables; @@ -4929,7 +4968,7 @@ void backup_fix_ddl(CorruptedPages &corrupted_pages) if (ddl_tracker.drops.find(id) != ddl_tracker.drops.end()) { dropped_tables.insert(name); - corrupted_pages.drop_space(id); + drop_space(id); continue; } @@ -4951,20 +4990,20 @@ void backup_fix_ddl(CorruptedPages &corrupted_pages) of it because of optimized DDL. We emulate a drop/create.*/ dropped_tables.insert(name); if (opt_log_innodb_page_corruption) - corrupted_pages.drop_space(id); + drop_space(id); new_tables.insert(new_name); } else { /* Renamed, and no optimized DDL*/ renamed_tables[name] = new_name; if (opt_log_innodb_page_corruption) - corrupted_pages.rename_space(id, new_name); + rename_space(id, new_name); } } else if (has_optimized_ddl) { /* Table was recreated, or optimized DDL ran. In both cases we need a full copy in the backup.*/ new_tables.insert(name); if (opt_log_innodb_page_corruption) - corrupted_pages.drop_space(id); + drop_space(id); } } @@ -4985,7 +5024,7 @@ void backup_fix_ddl(CorruptedPages &corrupted_pages) dropped_tables.erase(name); new_tables.insert(name); if (opt_log_innodb_page_corruption) - corrupted_pages.drop_space(id); + drop_space(id); } } @@ -4994,7 +5033,8 @@ void backup_fix_ddl(CorruptedPages &corrupted_pages) iter != renamed_tables.end(); ++iter) { const std::string old_name = iter->first; std::string new_name = iter->second; - backup_file_printf((old_name + ".ren").c_str(), "%s", new_name.c_str()); + DBUG_ASSERT(ds_data); + ds_data->backup_file_printf((old_name + ".ren").c_str(), "%s", new_name.c_str()); } // Mark tablespaces for drop @@ -5002,7 +5042,7 @@ void backup_fix_ddl(CorruptedPages &corrupted_pages) iter != dropped_tables.end(); iter++) { const std::string name(*iter); - backup_file_printf((name + ".del").c_str(), "%s", ""); + ds_data->backup_file_printf((name + ".del").c_str(), "%s", ""); } // Load and copy new tables. @@ -5046,8 +5086,9 @@ void backup_fix_ddl(CorruptedPages &corrupted_pages) continue; std::string dest_name(node->space->name); dest_name.append(".new"); - xtrabackup_copy_datafile(node, 0, dest_name.c_str(), wf_write_through, - corrupted_pages); + xtrabackup_copy_datafile(ds_data, ds_meta, + node, 0, dest_name.c_str(), + wf_write_through, *this); } datafiles_iter_free(it); @@ -6163,7 +6204,7 @@ static bool xtrabackup_prepare_func(char** argv) } } else - corrupted_pages.print_to_file(MB_CORRUPTED_PAGES_FILE); + corrupted_pages.print_to_file(NULL, MB_CORRUPTED_PAGES_FILE); if (xtrabackup_rollback_xa) { diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index 854456c3afd..de3a96443a3 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -46,11 +46,13 @@ public: bool contains(ulint space_id, ulint page_no) const; void drop_space(ulint space_id); void rename_space(ulint space_id, const std::string &new_name); - bool print_to_file(const char *file_name) const; + bool print_to_file(ds_ctxt *ds_data, const char *file_name) const; void read_from_file(const char *file_name); bool empty() const; void zero_out_free_pages(); + void backup_fix_ddl(ds_ctxt *ds_data, ds_ctxt *ds_meta); + private: void add_page_no_lock(const char *space_name, ulint space_id, ulint page_no, bool convert_space_name); @@ -63,6 +65,7 @@ private: container_t m_spaces; }; + /* value of the --incremental option */ extern lsn_t incremental_lsn; @@ -76,8 +79,6 @@ extern char *xb_rocksdb_datadir; extern my_bool xb_backup_rocksdb; extern uint opt_protocol; -extern ds_ctxt_t *ds_meta; -extern ds_ctxt_t *ds_data; /* The last checkpoint LSN at the backup startup time */ extern lsn_t checkpoint_lsn_start; @@ -177,7 +178,8 @@ extern ulong opt_binlog_info; extern ulong xtrabackup_innodb_force_recovery; void xtrabackup_io_throttling(void); -my_bool xb_write_delta_metadata(const char *filename, +my_bool xb_write_delta_metadata(ds_ctxt *ds_meta, + const char *filename, const xb_delta_info_t *info); /************************************************************************ -- cgit v1.2.1 From ef4d09948d5ff38f5dff8974005ba222a4b18462 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Tue, 11 Apr 2023 21:21:45 -0700 Subject: MDEV-20773 Error from UPDATE when estimating selectivity of a range This bug could affect multi-update statements as well as single-table update statements processed as multi-updates when the where condition contained a range condition over a non-indexed varchar column. The optimizer calculates selectivity of such range conditions using histograms. For each range the buckets containing endpoints of the the range are determined with a procedure that stores the values of the endpoints in the space of the record buffer where values of the columns are usually stored. For a range over a varchar column the value of a endpoint may exceed the size of the buffer and in such case the value is stored with truncation. This truncations cannot affect the result of the calculation of the range selectivity as the calculation employes only the beginning of the value string. However it can trigger generation of an unexpected error on this truncation if an update statement is processed. This patch prohibits truncation messages when selectivity of a range condition is calculated for a non-indexed column. Approved by Oleksandr Byelkin --- mysql-test/main/update.result | 29 +++++++++++++++++++++++++++++ mysql-test/main/update.test | 23 +++++++++++++++++++++++ sql/opt_range.cc | 3 +++ 3 files changed, 55 insertions(+) diff --git a/mysql-test/main/update.result b/mysql-test/main/update.result index f5edf1c6be3..7b6426d2ec5 100644 --- a/mysql-test/main/update.result +++ b/mysql-test/main/update.result @@ -734,3 +734,32 @@ UPDATE t1,t2 SET t1.i1 = -39 WHERE t2.d1 <> t1.i1 AND t2.d1 = t1.d2; ERROR 22007: Incorrect datetime value: '19' for column `test`.`t1`.`i1` at row 1 DROP TABLE t1,t2; # End of MariaDB 10.2 tests +# +# MDEV-20773: UPDATE with LIKE predicate over non-indexed column +# of VARCHAR type +# +create table t1 (a1 varchar(30), a2 varchar(30) collate utf8_bin); +insert into t1 values +('aa','zzz'), ('b','xxaa'), ('ccc','yyy'), ('ddd','xxb'); +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +test.t1 analyze status Engine-independent statistics collected +test.t1 analyze status OK +explain extended +update t1 set a1 = 'u' + where a2 like 'xx%' and exists(select 1 from t1 where t1.a1 < 'c'); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY t1 ALL NULL NULL NULL NULL 4 49.22 Using where +2 SUBQUERY t1 ALL NULL NULL NULL NULL 4 50.00 Using where +Warnings: +Note 1003 /* select#1 */ update `test`.`t1` set `test`.`t1`.`a1` = 'u' where `test`.`t1`.`a2` like 'xx%' +update t1 set a1 = 'u' + where a2 like 'xx%' and exists(select 1 from t1 where t1.a1 < 'c'); +select * from t1; +a1 a2 +aa zzz +u xxaa +ccc yyy +u xxb +drop table t1; +# End of MariaDB 10.4 tests diff --git a/mysql-test/main/update.test b/mysql-test/main/update.test index 8a6949447ee..147d69d50c9 100644 --- a/mysql-test/main/update.test +++ b/mysql-test/main/update.test @@ -676,3 +676,26 @@ UPDATE t1,t2 SET t1.i1 = -39 WHERE t2.d1 <> t1.i1 AND t2.d1 = t1.d2; DROP TABLE t1,t2; --echo # End of MariaDB 10.2 tests + +--echo # +--echo # MDEV-20773: UPDATE with LIKE predicate over non-indexed column +--echo # of VARCHAR type +--echo # + +create table t1 (a1 varchar(30), a2 varchar(30) collate utf8_bin); +insert into t1 values + ('aa','zzz'), ('b','xxaa'), ('ccc','yyy'), ('ddd','xxb'); +analyze table t1 persistent for all; + +explain extended +update t1 set a1 = 'u' + where a2 like 'xx%' and exists(select 1 from t1 where t1.a1 < 'c'); + +update t1 set a1 = 'u' + where a2 like 'xx%' and exists(select 1 from t1 where t1.a1 < 'c'); + +select * from t1; + +drop table t1; + +--echo # End of MariaDB 10.4 tests diff --git a/sql/opt_range.cc b/sql/opt_range.cc index c34420181a2..69a95f8da44 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -3518,7 +3518,10 @@ bool calculate_cond_selectivity_for_table(THD *thd, TABLE *table, Item **cond) } else { + enum_check_fields save_count_cuted_fields= thd->count_cuted_fields; + thd->count_cuted_fields= CHECK_FIELD_IGNORE; rows= records_in_column_ranges(¶m, idx, key); + thd->count_cuted_fields= save_count_cuted_fields; if (rows != DBL_MAX) { key->field->cond_selectivity= rows/table_records; -- cgit v1.2.1 From d1a4315f4cb096c2fd81c96bc4afc6bb618bae49 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Thu, 13 Apr 2023 07:49:35 +0200 Subject: MDEV-30402: Encrypted mariabackup SST breaks on distributions with newer socat This commit adds a new 'no-sni' option to socat which is required to properly authenticate with newer socat versions (after version 1.7.4+). This option is needed to disable the automatic use of the SNI feature (Server Name Indication) since the SST script directly specifies the commonname if necessary and automatic activation of the SNI feature is unnecessary in such scenarios. --- scripts/wsrep_sst_mariabackup.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 7e26af83701..b46e64a9e8b 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -340,6 +340,9 @@ get_transfer() "Use workaround for socat $SOCAT_VERSION bug" fi fi + if check_for_version "$SOCAT_VERSION" '1.7.4'; then + tcmd="$tcmd,no-sni=1" + fi fi if [ "${sockopt#*,dhparam=}" = "$sockopt" ]; then -- cgit v1.2.1 From 2e1c532bd2d9f9a35559e54f66d33c81e33009b1 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Fri, 24 Mar 2023 13:04:05 +1100 Subject: alloca() fix Corrections from 1e58b8afc086da755cf9209ed17fc36351da5563. * Re-add #pragma alloca for AIX - now in my_alloca.h --- include/my_alloca.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/include/my_alloca.h b/include/my_alloca.h index 25fd8867e69..de5f32bb886 100644 --- a/include/my_alloca.h +++ b/include/my_alloca.h @@ -34,7 +34,10 @@ #endif #endif -#if defined(HAVE_ALLOCA) +#if defined(_AIX) && !defined(__GNUC__) && !defined(_AIX43) +#pragma alloca +#endif /* _AIX */ + /* If the GCC/LLVM compiler from the MinGW is used, alloca may not be defined when using the MSVC CRT: @@ -42,6 +45,5 @@ #if defined(__GNUC__) && !defined(HAVE_ALLOCA_H) && !defined(alloca) #define alloca __builtin_alloca #endif /* GNUC */ -#endif #endif /* MY_ALLOCA_INCLUDED */ -- cgit v1.2.1 From 6c196090c8c265bfd93df1e2ee6b18cda2b1d2d8 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Thu, 13 Apr 2023 20:13:13 +0300 Subject: Fix compilation on gcc 11.2.0 It is used in the out-of date Ubuntu 21.10 Impish. --- storage/innobase/rem/rem0rec.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/innobase/rem/rem0rec.cc b/storage/innobase/rem/rem0rec.cc index ae8c3c4513e..d202afa9e20 100644 --- a/storage/innobase/rem/rem0rec.cc +++ b/storage/innobase/rem/rem0rec.cc @@ -242,9 +242,9 @@ enum rec_leaf_format { REC_LEAF_INSTANT }; -#if defined __GNUC__ && !defined __clang__ && __GNUC__ < 11 +#if defined __GNUC__ && !defined __clang__ && __GNUC__ < 12 # pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wconversion" /* GCC 5 to 10 need this */ +# pragma GCC diagnostic ignored "-Wconversion" /* GCC 5 to 11 need this */ #endif /** Determine the offset to each field in a leaf-page record in ROW_FORMAT=COMPACT,DYNAMIC,COMPRESSED. -- cgit v1.2.1 From 0cca8166f3111901019dcd33747a1a1dfd9e66d1 Mon Sep 17 00:00:00 2001 From: Vlad Lesin Date: Fri, 7 Apr 2023 00:16:16 +0300 Subject: MDEV-30775 Performance regression in fil_space_t::try_to_close() introduced in MDEV-23855 Post-push fix. 10.5 MDEV-30775 fix inserts just opened tablespace just after the element which fil_system.space_list_last_opened points to. In MDEV-25223 fil_system_t::space_list was changed from UT_LIST to ilist. ilist<...>::insert(iterator pos, reference value) inserts element to list before pos. But it was not taken into account during 10.5->10.6 merge in 85cbfaefee694cdd490b357444f24ff16b8042e8, and the fix does not work properly, i.e. it inserted just opened tablespace to the position preceding fil_system.space_list_last_opened. --- storage/innobase/fil/fil0fil.cc | 17 +++++++++++++---- storage/innobase/include/fil0fil.h | 1 + 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 7e2b04eaa73..00ded1ab27e 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -803,8 +803,17 @@ pfs_os_file_t fil_system_t::detach(fil_space_t *space, bool detach_handle) space_list_t::iterator s= space_list_t::iterator(space); if (space_list_last_opened == space) { - space_list_t::iterator prev= s; - space_list_last_opened= &*--prev; + if (s == space_list.begin()) + { + ut_ad(srv_operation > SRV_OPERATION_EXPORT_RESTORED || + srv_shutdown_state > SRV_SHUTDOWN_NONE); + space_list_last_opened= nullptr; + } + else + { + space_list_t::iterator prev= s; + space_list_last_opened= &*--prev; + } } space_list.erase(s); } @@ -1317,9 +1326,9 @@ void fil_system_t::close() void fil_system_t::add_opened_last_to_space_list(fil_space_t *space) { if (UNIV_LIKELY(space_list_last_opened != nullptr)) - space_list.insert(space_list_t::iterator(space_list_last_opened), *space); + space_list.insert(++space_list_t::iterator(space_list_last_opened), *space); else - space_list.push_back(*space); + space_list.push_front(*space); space_list_last_opened= space; } diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 12ac86dc9be..165994eef35 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -1552,6 +1552,7 @@ public: if (space_list_last_opened == space) { + ut_ad(s != space_list.begin()); space_list_t::iterator prev= s; space_list_last_opened= &*--prev; } -- cgit v1.2.1 From 71f16c836f5f91e2983c28d6a14bf3687bea78bb Mon Sep 17 00:00:00 2001 From: Vlad Lesin Date: Thu, 13 Apr 2023 17:56:38 +0300 Subject: MDEV-31049 fil_delete_tablespace() returns wrong file handle if tablespace was closed by parallel thread fil_delete_tablespace() stores file handle in local variable and calls mtr_t::commit_file()=>fil_system_t::detach(..., detach_handle=true), which sets space->chain.start->handle = OS_FILE_CLOSED. fil_system_t::detach() is invoked under fil_system.mutex. But before the mutex is acquired some parallel thread can change space->chain.start->handle. fil_delete_tablespace() returns value, stored in local variable, i.e. wrong value. File handle can be closed, for example, from buf_flush_space() when the limit of innodb_open_files exceded and fil_space_t::get() causes fil_space_t::try_to_close() call. fil_space_t::try_to_close() is executed under fil_system.mutex. And mtr_t::commit_file() locks it for fil_system_t::detach() call. fil_system_t::detach() returns detached file handle if its argument detach_handle is true. The fix is to let mtr_t::commit_file() to pass that detached file handle to fil_delete_tablespace(). --- storage/innobase/fil/fil0fil.cc | 4 +--- storage/innobase/include/mtr0mtr.h | 12 +++++++++--- storage/innobase/mtr/mtr0mtr.cc | 12 ++++++++++-- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 00ded1ab27e..19ebdc8d67e 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1687,9 +1687,7 @@ pfs_os_file_t fil_delete_tablespace(ulint id) mtr_t mtr; mtr.start(); mtr.log_file_op(FILE_DELETE, id, space->chain.start->name); - handle= space->chain.start->handle; - mtr.commit_file(*space, nullptr); - + mtr.commit_file(*space, nullptr, &handle); fil_space_free_low(space); } diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h index 60e01abe18d..1c044319ca0 100644 --- a/storage/innobase/include/mtr0mtr.h +++ b/storage/innobase/include/mtr0mtr.h @@ -93,10 +93,16 @@ struct mtr_t { ATTRIBUTE_COLD void commit_shrink(fil_space_t &space); /** Commit a mini-transaction that is deleting or renaming a file. - @param space tablespace that is being renamed or deleted - @param name new file name (nullptr=the file will be deleted) + @param space tablespace that is being renamed or deleted + @param name new file name (nullptr=the file will be deleted) + @param detached_handle if detached_handle != nullptr and if space is detached + during the function execution the file handle if its + node will be set to OS_FILE_CLOSED, and the previous + value of the file handle will be assigned to the + address, pointed by detached_handle. @return whether the operation succeeded */ - ATTRIBUTE_COLD bool commit_file(fil_space_t &space, const char *name); + ATTRIBUTE_COLD bool commit_file(fil_space_t &space, const char *name, + pfs_os_file_t *detached_handle= nullptr); /** Commit a mini-transaction that did not modify any pages, but generated some redo log on a higher level, such as diff --git a/storage/innobase/mtr/mtr0mtr.cc b/storage/innobase/mtr/mtr0mtr.cc index 2c004cb0aa6..8817c77a6f4 100644 --- a/storage/innobase/mtr/mtr0mtr.cc +++ b/storage/innobase/mtr/mtr0mtr.cc @@ -333,8 +333,14 @@ void mtr_t::commit_shrink(fil_space_t &space) /** Commit a mini-transaction that is deleting or renaming a file. @param space tablespace that is being renamed or deleted @param name new file name (nullptr=the file will be deleted) +@param detached_handle if detached_handle != nullptr and if space is detached + during the function execution the file handle if its + node will be set to OS_FILE_CLOSED, and the previous + value of the file handle will be assigned to the + address, pointed by detached_handle. @return whether the operation succeeded */ -bool mtr_t::commit_file(fil_space_t &space, const char *name) +bool mtr_t::commit_file(fil_space_t &space, const char *name, + pfs_os_file_t *detached_handle) { ut_ad(is_active()); ut_ad(!is_inside_ibuf()); @@ -402,7 +408,9 @@ bool mtr_t::commit_file(fil_space_t &space, const char *name) ut_ad(!space.referenced()); ut_ad(space.is_stopping()); - fil_system.detach(&space, true); + pfs_os_file_t handle = fil_system.detach(&space, true); + if (detached_handle) + *detached_handle = handle; mysql_mutex_unlock(&fil_system.mutex); success= true; -- cgit v1.2.1 From f575de39afacb24cd43c40bf43c27bfcf97a670b Mon Sep 17 00:00:00 2001 From: Florian Weimer Date: Tue, 11 Apr 2023 09:39:40 +0200 Subject: rocksdb: Define _GNU_SOURCE during fallocate CMake probe The glibc headers declare fallocate only if _GNU_SOURCE is defined. Without this change, the probe fails with C compilers which do not support implicit function declarations even if the system does in fact support the fallocate function. Upstream rocksdb does not need this because the probe is run with the C++ compiler, and current g++ versions define _GNU_SOURCE automatically. --- storage/rocksdb/build_rocksdb.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/rocksdb/build_rocksdb.cmake b/storage/rocksdb/build_rocksdb.cmake index 647e51e2f90..ba894d83d75 100644 --- a/storage/rocksdb/build_rocksdb.cmake +++ b/storage/rocksdb/build_rocksdb.cmake @@ -134,8 +134,8 @@ option(WITH_FALLOCATE "build with fallocate" ON) if(WITH_FALLOCATE AND UNIX) include(CheckCSourceCompiles) CHECK_C_SOURCE_COMPILES(" +#define _GNU_SOURCE #include -#include int main() { int fd = open(\"/dev/null\", 0); fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024); -- cgit v1.2.1 From 3b85e3dcc11e9638c9670a299eccdb77a51c1a19 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Sat, 15 Apr 2023 07:52:17 +0800 Subject: MDEV-30687: Make small facelifting to autobake-debs.sh (fix) Appending to 'eatmydata' will obviously cause an executable that doesn't exist. Use an array to create the entire executable. Also while we are at it, check the fakeroot actually works before using it. --- debian/autobake-deb.sh | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index cf14eb701b5..eed1f65fef6 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -170,34 +170,39 @@ dch -b -D "${LSBNAME}" -v "${VERSION}" "Automatic build with ${LOGSTRING}." --co echo "Creating package version ${VERSION} ... " -BUILDPACKAGE_DPKGCMD="" +BUILDPACKAGE_DPKGCMD=() + +# Fakeroot test +if fakeroot true; then + BUILDPACKAGE_DPKGCMD+=( "fakeroot" "--" ) +fi # Use eatmydata is available to build faster with less I/O, skipping fsync() # during the entire build process (safe because a build can always be restarted) if which eatmydata > /dev/null then - BUILDPACKAGE_DPKGCMD="eatmydata" + BUILDPACKAGE_DPKGCMD+=("eatmydata") fi -BUILDPACKAGE_DPKGCMD+="dpkg-buildpackage" +BUILDPACKAGE_DPKGCMD+=("dpkg-buildpackage") # Using dpkg-buildpackage args # -us Allow unsigned sources # -uc Allow unsigned changes # -I Tar ignore -BUILDPACKAGE_DPKG_ARGS=(-us -uc -I) +BUILDPACKAGE_DPKGCMD+=(-us -uc -I) # There can be also extra flags that are appended to args if [ -n "$BUILDPACKAGE_FLAGS" ] then read -ra BUILDPACKAGE_TMP_ARGS <<< "$BUILDPACKAGE_FLAGS" - BUILDPACKAGE_DPKG_ARGS=("${BUILDPACKAGE_DPKG_ARGS[@]} ${BUILDPACKAGE_TMP_ARGS[@]}") + BUILDPACKAGE_DPKGCMD+=( "${BUILDPACKAGE_TMP_ARGS[@]}" ) fi # Build the package # Pass -I so that .git and other unnecessary temporary and source control files # will be ignored by dpkg-source when creating the tar.gz source package. -fakeroot -- "${BUILDPACKAGE_DPKGCMD}" "${BUILDPACKAGE_DPKG_ARGS[@]}" +"${BUILDPACKAGE_DPKGCMD[@]}" # If the step above fails due to missing dependencies, you can manually run # sudo mk-build-deps debian/control -r -i -- cgit v1.2.1 From 8f87023d3f3fbaad4e33991713db884cbe052fbc Mon Sep 17 00:00:00 2001 From: Andrei Date: Fri, 14 Apr 2023 13:13:03 +0300 Subject: MDEV-28777 binlog.binlog_truncate_multi_engine failed in bb with Lost connection The 2013 error was right to catch the case B of the test unprepared for an expected simulated crash. The test gets refined to SELECT a (type of) bool value before the crash is invoked. --- mysql-test/suite/binlog/t/binlog_truncate_multi_engine.inc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/binlog/t/binlog_truncate_multi_engine.inc b/mysql-test/suite/binlog/t/binlog_truncate_multi_engine.inc index 52ce4741eaa..f3801070851 100644 --- a/mysql-test/suite/binlog/t/binlog_truncate_multi_engine.inc +++ b/mysql-test/suite/binlog/t/binlog_truncate_multi_engine.inc @@ -20,7 +20,9 @@ connect(con1,localhost,root,,); --source include/show_binary_logs.inc INSERT INTO t1 VALUES (1, REPEAT("x", 1)); INSERT INTO t2 VALUES (1, REPEAT("x", 1)); -if (`SELECT $case = "B"`) +--let $is_case_B=`SELECT $case = "B"` + +if ($is_case_B) { --write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect wait-binlog_truncate_multi_engine.test @@ -39,12 +41,12 @@ if (`SELECT $debug_sync_action != ""`) send COMMIT; --connection default -if (`SELECT $case = "B"`) +if ($is_case_B) { --source include/wait_until_disconnected.inc --source include/start_mysqld.inc } -if (`SELECT $case != "B"`) +if (!$is_case_B) { SET DEBUG_SYNC= "now WAIT_FOR con1_ready"; --echo List of binary logs after rotation -- cgit v1.2.1 From 1995c626a563aa3b2760704fdc6502f69aa35dbf Mon Sep 17 00:00:00 2001 From: Daniel Lenski Date: Fri, 14 Apr 2023 12:38:16 -0700 Subject: [MDEV-30854] Do not use " as string delimiter in mariadb-tzinfo-to-sql If SQL_MODE contains ANSI_QUOTES (https://mariadb.com/kb/en/sql-mode/), then the double-quote character (") is not a legal string delimiter. In https://github.com/MariaDB/server/commit/13e77930e615f05cc74d408110e887b00e1abcc9#diff-a333d4ebb2d73b6361ef7dfebc86d883f7e19853b4a9eb85984b039058fae47cR2431-R2435, Daniel Black introduced a case where the double-quote character would be used as a string delimiter in the SQL queries generated by mariadb-tzinfo-to-sql. This tool tool generates SQL queries which should be able to run on any MariaDB server of the matching version. Therefore, it should be extremely conservative in the SQL that it outputs, in order to maximize the chance that it can run regardless of the build or execution environment of the server. See MDEV-18778, MDEV-28263, and MDEV-28782 for previous cases where MariaDB has FAILED TO ENSURE that the generated timezone.sql actually works in different build and execution environments. More test coverage is clearly needed here. All new code of the whole pull request, including one or several files that are either new files or modified ones, are contributed under the BSD-new license. I am contributing on behalf of my employer Amazon Web Services, Inc. --- mysql-test/main/mysql_tzinfo_to_sql_symlink.result | 42 +++++++++++----------- sql/tztime.cc | 4 +-- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/mysql-test/main/mysql_tzinfo_to_sql_symlink.result b/mysql-test/main/mysql_tzinfo_to_sql_symlink.result index b6b35e44988..97548768a2d 100644 --- a/mysql-test/main/mysql_tzinfo_to_sql_symlink.result +++ b/mysql-test/main/mysql_tzinfo_to_sql_symlink.result @@ -10,13 +10,13 @@ CREATE TABLE time_zone_leap_second LIKE mysql.time_zone_leap_second; set @wsrep_is_on=(select coalesce(sum(SESSION_VALUE='ON'), 0) from information_schema.SYSTEM_VARIABLES WHERE VARIABLE_NAME='wsrep_on'); SET STATEMENT SQL_MODE='' FOR SELECT concat('%', GROUP_CONCAT(OPTION), '%') INTO @replicate_opt FROM (SELECT DISTINCT concat('REPLICATE_', UPPER(ENGINE)) AS OPTION FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN ('time_zone', 'time_zone_name', 'time_zone_transition', 'time_zone_transition_type', 'time_zone_leap_second') AND ENGINE in ('MyISAM', 'Aria')) AS o ORDER BY OPTION DESC; set @wsrep_cannot_replicate_tz=@wsrep_is_on AND (select coalesce(sum(GLOBAL_VALUE NOT LIKE @replicate_opt), 0) from information_schema.SYSTEM_VARIABLES WHERE VARIABLE_NAME='wsrep_mode'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_name_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_name'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_name_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_name''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_name ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_transition_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_transition'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_transition_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_transition''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_transition ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_transition_type_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_transition_type'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_transition_type_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_transition_type''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_transition_type ENGINE=InnoDB', 'do 0'); TRUNCATE TABLE time_zone; TRUNCATE TABLE time_zone_name; @@ -59,13 +59,13 @@ execute immediate if(@wsrep_cannot_replicate_tz, concat('ALTER TABLE time_zone_t set @wsrep_is_on=(select coalesce(sum(SESSION_VALUE='ON'), 0) from information_schema.SYSTEM_VARIABLES WHERE VARIABLE_NAME='wsrep_on'); SET STATEMENT SQL_MODE='' FOR SELECT concat('%', GROUP_CONCAT(OPTION), '%') INTO @replicate_opt FROM (SELECT DISTINCT concat('REPLICATE_', UPPER(ENGINE)) AS OPTION FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN ('time_zone', 'time_zone_name', 'time_zone_transition', 'time_zone_transition_type', 'time_zone_leap_second') AND ENGINE in ('MyISAM', 'Aria')) AS o ORDER BY OPTION DESC; set @wsrep_cannot_replicate_tz=@wsrep_is_on AND (select coalesce(sum(GLOBAL_VALUE NOT LIKE @replicate_opt), 0) from information_schema.SYSTEM_VARIABLES WHERE VARIABLE_NAME='wsrep_mode'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_name_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_name'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_name_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_name''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_name ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_transition_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_transition'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_transition_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_transition''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_transition ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_transition_type_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_transition_type'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_transition_type_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_transition_type''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_transition_type ENGINE=InnoDB', 'do 0'); TRUNCATE TABLE time_zone; TRUNCATE TABLE time_zone_name; @@ -191,13 +191,13 @@ TRUNCATE TABLE time_zone_leap_second; set @wsrep_is_on=(select coalesce(sum(SESSION_VALUE='ON'), 0) from information_schema.SYSTEM_VARIABLES WHERE VARIABLE_NAME='wsrep_on'); SET STATEMENT SQL_MODE='' FOR SELECT concat('%', GROUP_CONCAT(OPTION), '%') INTO @replicate_opt FROM (SELECT DISTINCT concat('REPLICATE_', UPPER(ENGINE)) AS OPTION FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN ('time_zone', 'time_zone_name', 'time_zone_transition', 'time_zone_transition_type', 'time_zone_leap_second') AND ENGINE in ('MyISAM', 'Aria')) AS o ORDER BY OPTION DESC; set @wsrep_cannot_replicate_tz=@wsrep_is_on AND (select coalesce(sum(GLOBAL_VALUE NOT LIKE @replicate_opt), 0) from information_schema.SYSTEM_VARIABLES WHERE VARIABLE_NAME='wsrep_mode'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_name_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_name'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_name_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_name''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_name ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_transition_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_transition'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_transition_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_transition''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_transition ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_transition_type_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_transition_type'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_transition_type_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_transition_type''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_transition_type ENGINE=InnoDB', 'do 0'); /*M!100602 execute immediate if(@wsrep_cannot_replicate_tz, 'start transaction', 'LOCK TABLES time_zone WRITE, time_zone_leap_second WRITE, @@ -313,20 +313,20 @@ TRUNCATE TABLE time_zone_leap_second; set @wsrep_is_on=(select coalesce(sum(SESSION_VALUE='ON'), 0) from information_schema.SYSTEM_VARIABLES WHERE VARIABLE_NAME='wsrep_on'); SET STATEMENT SQL_MODE='' FOR SELECT concat('%', GROUP_CONCAT(OPTION), '%') INTO @replicate_opt FROM (SELECT DISTINCT concat('REPLICATE_', UPPER(ENGINE)) AS OPTION FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN ('time_zone', 'time_zone_name', 'time_zone_transition', 'time_zone_transition_type', 'time_zone_leap_second') AND ENGINE in ('MyISAM', 'Aria')) AS o ORDER BY OPTION DESC; set @wsrep_cannot_replicate_tz=@wsrep_is_on AND (select coalesce(sum(GLOBAL_VALUE NOT LIKE @replicate_opt), 0) from information_schema.SYSTEM_VARIABLES WHERE VARIABLE_NAME='wsrep_mode'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_name_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_name'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_name_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_name''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_name ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_transition_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_transition'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_transition_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_transition''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_transition ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_transition_type_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_transition_type'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_transition_type_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_transition_type''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_transition_type ENGINE=InnoDB', 'do 0'); /*M!100602 execute immediate if(@wsrep_cannot_replicate_tz, 'start transaction', 'LOCK TABLES time_zone WRITE, time_zone_leap_second WRITE, time_zone_name WRITE, time_zone_transition WRITE, time_zone_transition_type WRITE')*/; -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_leap_second_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_leap_second'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_leap_second_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_leap_second''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_leap_second ENGINE=InnoDB', 'do 0'); TRUNCATE TABLE time_zone_leap_second; execute immediate if(@wsrep_cannot_replicate_tz, concat('ALTER TABLE time_zone_leap_second ENGINE=', @time_zone_leap_second_engine), 'do 0'); @@ -497,13 +497,13 @@ set sql_mode=default; set @wsrep_is_on=(select coalesce(sum(SESSION_VALUE='ON'), 0) from information_schema.SYSTEM_VARIABLES WHERE VARIABLE_NAME='wsrep_on'); SET STATEMENT SQL_MODE='' FOR SELECT concat('%', GROUP_CONCAT(OPTION), '%') INTO @replicate_opt FROM (SELECT DISTINCT concat('REPLICATE_', UPPER(ENGINE)) AS OPTION FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN ('time_zone', 'time_zone_name', 'time_zone_transition', 'time_zone_transition_type', 'time_zone_leap_second') AND ENGINE in ('MyISAM', 'Aria')) AS o ORDER BY OPTION DESC; set @wsrep_cannot_replicate_tz=@wsrep_is_on AND (select coalesce(sum(GLOBAL_VALUE NOT LIKE @replicate_opt), 0) from information_schema.SYSTEM_VARIABLES WHERE VARIABLE_NAME='wsrep_mode'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_name_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_name'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_name_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_name''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_name ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_transition_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_transition'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_transition_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_transition''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_transition ENGINE=InnoDB', 'do 0'); -execute immediate if(@wsrep_cannot_replicate_tz, "select ENGINE into @time_zone_transition_type_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME='time_zone_transition_type'", 'do 0'); +execute immediate if(@wsrep_cannot_replicate_tz, 'select ENGINE into @time_zone_transition_type_engine from information_schema.TABLES where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''time_zone_transition_type''', 'do 0'); execute immediate if(@wsrep_cannot_replicate_tz, 'ALTER TABLE time_zone_transition_type ENGINE=InnoDB', 'do 0'); TRUNCATE TABLE time_zone; TRUNCATE TABLE time_zone_name; diff --git a/sql/tztime.cc b/sql/tztime.cc index 8286744e192..1c482718cc8 100644 --- a/sql/tztime.cc +++ b/sql/tztime.cc @@ -2429,9 +2429,9 @@ print_tz_as_sql(const char* tz_name, const TIME_ZONE_INFO *sp) #define SAVE_ENGINE(e) \ - "\"select ENGINE into @" e "_engine" \ + "'select ENGINE into @" e "_engine" \ " from information_schema.TABLES" \ - " where TABLE_SCHEMA=DATABASE() and TABLE_NAME='" e "'\"" + " where TABLE_SCHEMA=DATABASE() and TABLE_NAME=''" e "'''" /* Print info about leap seconds in time zone as SQL statements -- cgit v1.2.1 From bc3bfcf943b817b19a41e4f599b4f2e9a259b263 Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Mon, 20 Mar 2023 15:20:32 +0100 Subject: MDEV-30862 Assertion `mode_ == m_high_priority' failed CREATE TABLE AS SELECT is not supported in combination with streaming replication. --- mysql-test/suite/galera_sr/r/MDEV-30862.result | 11 +++++++++++ mysql-test/suite/galera_sr/t/MDEV-30862.test | 24 ++++++++++++++++++++++++ sql/sql_table.cc | 13 +++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 mysql-test/suite/galera_sr/r/MDEV-30862.result create mode 100644 mysql-test/suite/galera_sr/t/MDEV-30862.test diff --git a/mysql-test/suite/galera_sr/r/MDEV-30862.result b/mysql-test/suite/galera_sr/r/MDEV-30862.result new file mode 100644 index 00000000000..43da77f24df --- /dev/null +++ b/mysql-test/suite/galera_sr/r/MDEV-30862.result @@ -0,0 +1,11 @@ +connection node_2; +connection node_1; +SET autocommit=0; +SET SESSION wsrep_trx_fragment_size=1; +CREATE TABLE t2 SELECT seq FROM seq_1_to_50; +ERROR 42000: CREATE TABLE AS SELECT is not supported with streaming replication +CREATE TABLE t1 (f1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY); +INSERT INTO t1 VALUES(DEFAULT); +CREATE TABLE t2 SELECT * FROM t1; +ERROR 42000: CREATE TABLE AS SELECT is not supported with streaming replication +DROP TABLE t1; diff --git a/mysql-test/suite/galera_sr/t/MDEV-30862.test b/mysql-test/suite/galera_sr/t/MDEV-30862.test new file mode 100644 index 00000000000..6be77b4d71b --- /dev/null +++ b/mysql-test/suite/galera_sr/t/MDEV-30862.test @@ -0,0 +1,24 @@ +# +# MDEV-30862 Assertion `mode_ == m_high_priority' failed in +# void wsrep::client_state::after_applying() +# + +--source include/galera_cluster.inc +--source include/have_sequence.inc + +SET autocommit=0; +SET SESSION wsrep_trx_fragment_size=1; +--error ER_NOT_ALLOWED_COMMAND +CREATE TABLE t2 SELECT seq FROM seq_1_to_50; + + +# +# Same test without using seq +# +CREATE TABLE t1 (f1 INT NOT NULL AUTO_INCREMENT PRIMARY KEY); +INSERT INTO t1 VALUES(DEFAULT); +--error ER_NOT_ALLOWED_COMMAND +CREATE TABLE t2 SELECT * FROM t1; + + +DROP TABLE t1; diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 9f13dcde40f..1e88e7722e3 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -11569,6 +11569,19 @@ bool Sql_cmd_create_table_like::execute(THD *thd) } #endif +#ifdef WITH_WSREP + if (select_lex->item_list.elements && // With SELECT + WSREP(thd) && thd->variables.wsrep_trx_fragment_size > 0) + { + my_message( + ER_NOT_ALLOWED_COMMAND, + "CREATE TABLE AS SELECT is not supported with streaming replication", + MYF(0)); + res= 1; + goto end_with_restore_list; + } +#endif /* WITH_WSREP */ + if (select_lex->item_list.elements || select_lex->tvc) // With select or TVC { select_result *result; -- cgit v1.2.1 From be7ef6566fab6088b5222eae184226ed6b5994d3 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Tue, 28 Mar 2023 10:25:59 +0300 Subject: MDEV-30605: Wrong result while using index for group-by A GROUP BY query which uses "MIN(pk)" and has "pk<>const" in the WHERE clause would produce wrong result when handled with "Using index for group-by". Here "pk" column is the table's primary key. The problem was introduced by fix for MDEV-23634. It made the range optimizer to not produce ranges for conditions in form "pk != const". However, LooseScan code requires that the optimizer is able to convert the condition on the MIN/MAX column into an equivalent range. The range is used to locate the row that has the MIN/MAX value. LooseScan checks this in check_group_min_max_predicates(). This fix makes the code in that function to take into account that "pk != const" does not produce a range. --- mysql-test/main/group_min_max.result | 12 ++++++++++ mysql-test/main/group_min_max.test | 11 +++++++++ sql/opt_range.cc | 46 ++++++++++++++++++++++-------------- 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/mysql-test/main/group_min_max.result b/mysql-test/main/group_min_max.result index d1bd4d8cedb..a87a79fbc56 100644 --- a/mysql-test/main/group_min_max.result +++ b/mysql-test/main/group_min_max.result @@ -4083,5 +4083,17 @@ MIN(pk) 1 DROP TABLE t1, t2; # +# MDEV-30605 Wrong result while using index for group-by +# +CREATE TABLE t1 (pk INT primary key, a int, key(a)) engine=innodb; +INSERT INTO t1 VALUES (1,-1),(2,8),(3,5),(4,-1),(5,10), (6,-1); +SELECT MIN(pk), a FROM t1 WHERE pk <> 1 GROUP BY a; +MIN(pk) a +4 -1 +3 5 +2 8 +5 10 +DROP TABLE t1; +# # End of 10.5 tests # diff --git a/mysql-test/main/group_min_max.test b/mysql-test/main/group_min_max.test index 7de57d75d36..5f7981b8b30 100644 --- a/mysql-test/main/group_min_max.test +++ b/mysql-test/main/group_min_max.test @@ -1737,6 +1737,17 @@ SELECT SQL_BUFFER_RESULT MIN(pk) FROM t1, t2; SELECT MIN(pk) FROM t1, t2; DROP TABLE t1, t2; +--echo # +--echo # MDEV-30605 Wrong result while using index for group-by +--echo # + +CREATE TABLE t1 (pk INT primary key, a int, key(a)) engine=innodb; +INSERT INTO t1 VALUES (1,-1),(2,8),(3,5),(4,-1),(5,10), (6,-1); + +SELECT MIN(pk), a FROM t1 WHERE pk <> 1 GROUP BY a; + +DROP TABLE t1; + --echo # --echo # End of 10.5 tests --echo # diff --git a/sql/opt_range.cc b/sql/opt_range.cc index 223799a3235..82b19e23fd4 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -461,7 +461,7 @@ void print_range_for_non_indexed_field(String *out, Field *field, static void print_min_range_operator(String *out, const ha_rkey_function flag); static void print_max_range_operator(String *out, const ha_rkey_function flag); -static bool is_field_an_unique_index(RANGE_OPT_PARAM *param, Field *field); +static bool is_field_an_unique_index(Field *field); /* SEL_IMERGE is a list of possible ways to do index merge, i.e. it is @@ -7752,8 +7752,13 @@ SEL_TREE *Item_func_ne::get_func_mm_tree(RANGE_OPT_PARAM *param, If this condition is a "col1<>...", where there is a UNIQUE KEY(col1), do not construct a SEL_TREE from it. A condition that excludes just one row in the table is not selective (unless there are only a few rows) + + Note: this logic must be in sync with code in + check_group_min_max_predicates(). That function walks an Item* condition + and checks if the range optimizer would produce an equivalent range for + it. */ - if (is_field_an_unique_index(param, field)) + if (param->using_real_indexes && is_field_an_unique_index(field)) DBUG_RETURN(NULL); DBUG_RETURN(get_ne_mm_tree(param, field, value, value)); } @@ -7865,7 +7870,7 @@ SEL_TREE *Item_func_in::get_func_mm_tree(RANGE_OPT_PARAM *param, - if there are a lot of constants, the overhead of building and processing enormous range list is not worth it. */ - if (is_field_an_unique_index(param, field)) + if (param->using_real_indexes && is_field_an_unique_index(field)) DBUG_RETURN(0); /* Get a SEL_TREE for "(-inf|NULL) < X < c_0" interval. */ @@ -8574,24 +8579,18 @@ SEL_TREE *Item_equal::get_mm_tree(RANGE_OPT_PARAM *param, Item **cond_ptr) In the future we could also add "almost unique" indexes where any value is present only in a few rows (but necessarily exactly one row) */ -static bool is_field_an_unique_index(RANGE_OPT_PARAM *param, Field *field) +static bool is_field_an_unique_index(Field *field) { DBUG_ENTER("is_field_an_unique_index"); - - // The check for using_real_indexes is there because of the heuristics - // this function is used for. - if (param->using_real_indexes) + key_map::Iterator it(field->key_start); + uint key_no; + while ((key_no= it++) != key_map::Iterator::BITMAP_END) { - key_map::Iterator it(field->key_start); - uint key_no; - while ((key_no= it++) != key_map::Iterator::BITMAP_END) + KEY *key_info= &field->table->key_info[key_no]; + if (key_info->user_defined_key_parts == 1 && + (key_info->flags & HA_NOSAME)) { - KEY *key_info= &field->table->key_info[key_no]; - if (key_info->user_defined_key_parts == 1 && - (key_info->flags & HA_NOSAME)) - { - DBUG_RETURN(true); - } + DBUG_RETURN(true); } } DBUG_RETURN(false); @@ -13475,7 +13474,7 @@ cost_group_min_max(TABLE* table, KEY *index_info, uint used_key_parts, - (C between const_i and const_j) - C IS NULL - C IS NOT NULL - - C != const + - C != const (unless C is the primary key) SA4. If Q has a GROUP BY clause, there are no other aggregate functions except MIN and MAX. For queries with DISTINCT, aggregate functions are allowed. @@ -14358,6 +14357,17 @@ check_group_min_max_predicates(Item *cond, Item_field *min_max_arg_item, if (!simple_pred(pred, args, &inv)) DBUG_RETURN(FALSE); + /* + Follow the logic in Item_func_ne::get_func_mm_tree(): condition + in form "tbl.primary_key <> const" is not used to produce intervals. + + If the condition doesn't have an equivalent interval, this means we + fail LooseScan's condition SA3. Return FALSE to indicate this. + */ + if (pred_type == Item_func::NE_FUNC && + is_field_an_unique_index(min_max_arg_item->field)) + DBUG_RETURN(FALSE); + if (args[0] && args[1]) // this is a binary function or BETWEEN { DBUG_ASSERT(pred->fixed_type_handler()); -- cgit v1.2.1 From 485a1b1f116f0c5e73fce3a97ffdac84c861b3c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 18 Apr 2023 14:54:40 +0300 Subject: MDEV-30863 Server freeze, all threads in trx_assign_rseg_low() trx_assign_rseg_low(): Simplify the debug check. trx_rseg_t::reinit(): Reset the skip_allocation() flag. This logic was broken in the merge commit 3e2ad0e918d5d38322994ec9e08fc5dda3a80707 of commit 0de3be8cfdfc26f5c236eaefe12d03c7b4af22c8 (that is, innodb_undo_log_truncate=ON would never be "completed"). Tested by: Matthias Leich --- storage/innobase/include/trx0rseg.h | 3 ++- storage/innobase/trx/trx0rseg.cc | 1 + storage/innobase/trx/trx0trx.cc | 20 +++++--------------- 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/storage/innobase/include/trx0rseg.h b/storage/innobase/include/trx0rseg.h index 8b7eacbbc18..1d95b7d2e7a 100644 --- a/storage/innobase/include/trx0rseg.h +++ b/storage/innobase/include/trx0rseg.h @@ -129,7 +129,8 @@ public: #endif } /** @return whether the segment is marked for undo truncation */ - bool skip_allocation() const { return ref_load() & SKIP; } + bool skip_allocation() const + { return ref.load(std::memory_order_acquire) & SKIP; } /** Increment the reference count */ void acquire() { ut_d(auto r=) ref.fetch_add(REF); ut_ad(!(r & SKIP)); } diff --git a/storage/innobase/trx/trx0rseg.cc b/storage/innobase/trx/trx0rseg.cc index d30300d70a7..6d95dcf06f1 100644 --- a/storage/innobase/trx/trx0rseg.cc +++ b/storage/innobase/trx/trx0rseg.cc @@ -403,6 +403,7 @@ void trx_rseg_t::reinit(uint32_t page) last_commit_and_offset= 0; last_page_no= FIL_NULL; curr_size= 1; + ref.store(0, std::memory_order_release); } /** Read the undo log lists. diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index d7ab02844bf..ed1187e179b 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -811,30 +811,20 @@ static void trx_assign_rseg_low(trx_t *trx) static Atomic_counter rseg_slot; unsigned slot = rseg_slot++ % TRX_SYS_N_RSEGS; ut_d(if (trx_rseg_n_slots_debug) slot = 0); + ut_d(const auto start_scan_slot = slot); trx_rseg_t* rseg; -#ifdef UNIV_DEBUG - ulint start_scan_slot = slot; - bool look_for_rollover = false; -#endif /* UNIV_DEBUG */ - bool allocated; do { for (;;) { rseg = &trx_sys.rseg_array[slot]; -#ifdef UNIV_DEBUG - /* Ensure that we are not revisiting the same - slot that we have already inspected. */ - if (look_for_rollover) { + do { + ut_d(if (!trx_rseg_n_slots_debug) continue); + slot = (slot + 1) % TRX_SYS_N_RSEGS; ut_ad(start_scan_slot != slot); - } - look_for_rollover = true; -#endif /* UNIV_DEBUG */ - - ut_d(if (!trx_rseg_n_slots_debug)) - slot = (slot + 1) % TRX_SYS_N_RSEGS; + } while (0); if (!rseg->space) { continue; -- cgit v1.2.1 From feeeacc4d747868c234425dc12c157c6e5fa8fbb Mon Sep 17 00:00:00 2001 From: Daniele Sciascia Date: Wed, 29 Mar 2023 13:55:30 +0200 Subject: MDEV-30955 Explicit locks released too early in rollback path Assertion `thd->mdl_context.is_lock_owner()` fires when a client is disconnected, while transaction and and a table is opened through `HANDLER` interface. Reason for the assertion is that when a connection closes, its ongoing transaction is eventually rolled back in `Wsrep_client_state::bf_rollback()`. This method also releases explicit which are expected to survive beyond the transaction lifetime. This patch also removes calls to `mysql_ull_cleanup()`. User level locks are not supported in combination with Galera, making these calls unnecessary. --- mysql-test/suite/galera/r/MDEV-30955.result | 26 +++++++++++ mysql-test/suite/galera/t/MDEV-30955.test | 70 +++++++++++++++++++++++++++++ sql/sql_parse.cc | 2 +- sql/wsrep_client_service.cc | 2 - sql/wsrep_high_priority_service.cc | 2 - 5 files changed, 97 insertions(+), 5 deletions(-) create mode 100644 mysql-test/suite/galera/r/MDEV-30955.result create mode 100644 mysql-test/suite/galera/t/MDEV-30955.test diff --git a/mysql-test/suite/galera/r/MDEV-30955.result b/mysql-test/suite/galera/r/MDEV-30955.result new file mode 100644 index 00000000000..2a090cb58bc --- /dev/null +++ b/mysql-test/suite/galera/r/MDEV-30955.result @@ -0,0 +1,26 @@ +connection node_2; +connection node_1; +CREATE TABLE t (a CHAR(1) KEY); +START TRANSACTION; +HANDLER t OPEN; +disconnect node_1; +connect node_1, 127.0.0.1, root, , test, $NODE_MYPORT_1; +DROP TABLE t; +BACKUP STAGE START; +START TRANSACTION; +disconnect node_1; +connect node_1, 127.0.0.1, root, , test, $NODE_MYPORT_1; +connection node_1; +CREATE TABLE t1 (f1 INTEGER PRIMARY KEY); +CREATE TABLE t2 (f1 INTEGER PRIMARY KEY); +START TRANSACTION; +INSERT INTO t1 VALUES(1); +HANDLER t2 OPEN; +connection node_2; +INSERT INTO t1 VALUES(1); +connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1; +connection node_1a; +connection node_1; +COMMIT; +ERROR 40001: Deadlock found when trying to get lock; try restarting transaction +DROP TABLE t1,t2; diff --git a/mysql-test/suite/galera/t/MDEV-30955.test b/mysql-test/suite/galera/t/MDEV-30955.test new file mode 100644 index 00000000000..18577120e83 --- /dev/null +++ b/mysql-test/suite/galera/t/MDEV-30955.test @@ -0,0 +1,70 @@ +# +# MDEV-30955 +# Assertion `thd->mdl_context.is_lock_owner(MDL_key::TABLE, +# table->s->db.str, table->s->table_name.str, MDL_SHARED)' +# failed in close_thread_table() +# + +--source include/galera_cluster.inc + +# +# Test 1: Assertion thd->mdl_context.is_lock_owner() +# failed in close_thread_table() +# +CREATE TABLE t (a CHAR(1) KEY); +START TRANSACTION; +HANDLER t OPEN; + +# +# If bug is present the transaction will be aborted +# through Wsrep_client_service::bf_rollback() and +# release explicit locks too early. Later, during +# THD::cleanup(), table t will be closed and the +# THD is expected to be owner of the MDL lock that +# was just released. +# +--disconnect node_1 + +--connect node_1, 127.0.0.1, root, , test, $NODE_MYPORT_1 +DROP TABLE t; + + +# +# Test 2: Similar issue reproduces also with BACKUP STAGE locks. +# See comments in MDEV-25037 +# + +BACKUP STAGE START; +START TRANSACTION; +--disconnect node_1 +--connect node_1, 127.0.0.1, root, , test, $NODE_MYPORT_1 + + +# +# Test 3: Assertion `!thd->mdl_context.has_locks()' failed +# in do_command() +# + +--connection node_1 +CREATE TABLE t1 (f1 INTEGER PRIMARY KEY); +CREATE TABLE t2 (f1 INTEGER PRIMARY KEY); + +--let $bf_count = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.global_status WHERE VARIABLE_NAME = 'wsrep_local_bf_aborts'` + +START TRANSACTION; +INSERT INTO t1 VALUES(1); +HANDLER t2 OPEN; + +--connection node_2 +INSERT INTO t1 VALUES(1); + +--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1 +--connection node_1a +--let $wait_condition = SELECT VARIABLE_VALUE = $bf_count + 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_bf_aborts' +--source include/wait_condition.inc + +--connection node_1 +--error ER_LOCK_DEADLOCK +COMMIT; + +DROP TABLE t1,t2; diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index c495ae2d6c4..c95993e1604 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -1309,7 +1309,7 @@ bool do_command(THD *thd) in wsrep_before_command(). */ WSREP_LOG_THD(thd, "enter found BF aborted"); - DBUG_ASSERT(!thd->mdl_context.has_locks()); + DBUG_ASSERT(!thd->mdl_context.has_transactional_locks()); DBUG_ASSERT(!thd->get_stmt_da()->is_set()); /* We let COM_QUIT and COM_STMT_CLOSE to execute even if wsrep aborted. */ if (command == COM_STMT_EXECUTE) diff --git a/sql/wsrep_client_service.cc b/sql/wsrep_client_service.cc index f00dfccf274..0399cf4f442 100644 --- a/sql/wsrep_client_service.cc +++ b/sql/wsrep_client_service.cc @@ -362,8 +362,6 @@ int Wsrep_client_service::bf_rollback() m_thd->global_read_lock.unlock_global_read_lock(m_thd); } m_thd->release_transactional_locks(); - mysql_ull_cleanup(m_thd); - m_thd->mdl_context.release_explicit_locks(); DBUG_RETURN(ret); } diff --git a/sql/wsrep_high_priority_service.cc b/sql/wsrep_high_priority_service.cc index 96269481559..3c6524b7ddf 100644 --- a/sql/wsrep_high_priority_service.cc +++ b/sql/wsrep_high_priority_service.cc @@ -357,8 +357,6 @@ int Wsrep_high_priority_service::rollback(const wsrep::ws_handle& ws_handle, m_thd->wsrep_cs().prepare_for_ordering(ws_handle, ws_meta, false); int ret= (trans_rollback_stmt(m_thd) || trans_rollback(m_thd)); m_thd->release_transactional_locks(); - mysql_ull_cleanup(m_thd); - m_thd->mdl_context.release_explicit_locks(); free_root(m_thd->mem_root, MYF(MY_KEEP_PREALLOC)); -- cgit v1.2.1 From 75063d128812347228873e2dce4ae7799f348ebf Mon Sep 17 00:00:00 2001 From: Yuchen Pei Date: Tue, 4 Apr 2023 15:54:26 +1000 Subject: MDEV-30542 Add multilength spider self-reference detection test --- .../spider/bugfix/r/self_reference_multi.result | 21 ++++++++++++++++ .../spider/bugfix/t/self_reference_multi.test | 29 ++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 storage/spider/mysql-test/spider/bugfix/r/self_reference_multi.result create mode 100644 storage/spider/mysql-test/spider/bugfix/t/self_reference_multi.test diff --git a/storage/spider/mysql-test/spider/bugfix/r/self_reference_multi.result b/storage/spider/mysql-test/spider/bugfix/r/self_reference_multi.result new file mode 100644 index 00000000000..c4399ddf9d2 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/self_reference_multi.result @@ -0,0 +1,21 @@ +for master_1 +for child2 +for child3 + +MDEV-6268 SPIDER table with no COMMENT clause causes queries to wait forever + +CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t2 (c int); +create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; +create table t0 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t1"'; +alter table t2 ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t0"'; +select * from t0; +ERROR HY000: An infinite loop is detected when opening table test.t0 +select * from t1; +ERROR HY000: An infinite loop is detected when opening table test.t0 +select * from t2; +ERROR HY000: An infinite loop is detected when opening table test.t0 +drop table t0, t1, t2; +for master_1 +for child2 +for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/t/self_reference_multi.test b/storage/spider/mysql-test/spider/bugfix/t/self_reference_multi.test new file mode 100644 index 00000000000..8b6f070d167 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/self_reference_multi.test @@ -0,0 +1,29 @@ +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log + +--echo +--echo MDEV-6268 SPIDER table with no COMMENT clause causes queries to wait forever +--echo + +--replace_regex /SOCKET ".*"/SOCKET "$MASTER_1_MYSOCK"/ +eval CREATE SERVER srv FOREIGN DATA WRAPPER MYSQL OPTIONS (SOCKET "$MASTER_1_MYSOCK", DATABASE 'test',user 'root'); +create table t2 (c int); +create table t1 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t2"'; +create table t0 (c int) ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t1"'; +alter table t2 ENGINE=Spider COMMENT='WRAPPER "mysql", srv "srv",TABLE "t0"'; +--error 12719 +select * from t0; +--error 12719 +select * from t1; +--error 12719 +select * from t2; +drop table t0, t1, t2; + +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log -- cgit v1.2.1 From d6651864773afc2b7534285ee0d50d4fd6b47e9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 19 Apr 2023 14:08:53 +0300 Subject: MDEV-28976: mtr must wait for server to actually die do_shutdown_server(): Call wait_until_dead() also when we are forcibly killing the process (timeout=0). We have evidence that killing the process may take some time and cause mystery failures in crash recovery tests. For InnoDB, several failures were observed between commit da094188f60bf67e3d90227304a4ea256fe2630f and commit 0ee1082bd2e7e7049c4f0e686bad53cf7ba053ab when no advisory file locking was being used by default. --- client/mysqltest.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/mysqltest.cc b/client/mysqltest.cc index 6330d4f881d..9e9b122c5fe 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -5191,7 +5191,7 @@ void do_shutdown_server(struct st_command *command) if (timeout) (void) my_kill(pid, SIGABRT); /* Give server a few seconds to die in all cases */ - if (!timeout || wait_until_dead(pid, timeout < 5 ? 5 : timeout)) + if (wait_until_dead(pid, timeout < 5 ? 5 : timeout)) { (void) my_kill(pid, SIGKILL); } -- cgit v1.2.1 From b2bbc66a41a7b3b622fbcd477e777c45e3248886 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Mon, 10 Apr 2023 11:57:39 +0530 Subject: MDEV-24011 InnoDB: Failing assertion: index_cache->words == NULL in fts0fts.cc line 551 This issue happens when race condition happens when DDL and fts optimize thread. DDL adds the new index to fts cache. At the same time, fts optimize thread clears the cache and reinitialize it. Take cache init lock before reinitializing the cache. fts_sync_commit() should take dict_sys mutex to avoid the deadlock with create index. --- storage/innobase/fts/fts0fts.cc | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index 224fc9593b7..8ce6fee0b76 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -851,7 +851,7 @@ fts_drop_index( dberr_t err = DB_SUCCESS; ut_a(indexes); - + ut_d(dict_sys.assert_locked()); if ((ib_vector_size(indexes) == 1 && (index == static_cast( ib_vector_getp(table->fts->indexes, 0))) @@ -873,7 +873,9 @@ fts_drop_index( current_doc_id = table->fts->cache->next_doc_id; first_doc_id = table->fts->cache->first_doc_id; + rw_lock_x_lock(&table->fts->cache->init_lock); fts_cache_clear(table->fts->cache); + rw_lock_x_unlock(&table->fts->cache->init_lock); fts_cache_destroy(table->fts->cache); table->fts->cache = fts_cache_create(table); table->fts->cache->next_doc_id = current_doc_id; @@ -4192,9 +4194,15 @@ fts_sync_commit( /* We need to do this within the deleted lock since fts_delete() can attempt to add a deleted doc id to the cache deleted id array. */ + mutex_enter(&dict_sys.mutex); + sync->table->fts->dict_locked = true; + rw_lock_x_lock(&cache->init_lock); fts_cache_clear(cache); DEBUG_SYNC_C("fts_deleted_doc_ids_clear"); fts_cache_init(cache); + rw_lock_x_unlock(&cache->init_lock); + sync->table->fts->dict_locked = false; + mutex_exit(&dict_sys.mutex); rw_lock_x_unlock(&cache->lock); if (UNIV_LIKELY(error == DB_SUCCESS)) { -- cgit v1.2.1 From 2bfd04e3145b238df5f31143b98b1df501f43d1e Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Tue, 11 Apr 2023 18:36:55 +0530 Subject: MDEV-31025 Redundant table alter fails when fixed column stored externally row_merge_buf_add(): Has strict assert that fixed length mismatch shouldn't happen while rebuilding the redundant row format table btr_index_rec_validate(): Fixed size column can be stored externally. So sum of inline stored length and external stored length of the column should be equal to total column length --- .../suite/innodb/r/default_row_format_alter.result | 20 ++++++++++++++++++++ .../suite/innodb/t/default_row_format_alter.test | 17 +++++++++++++++++ storage/innobase/btr/btr0btr.cc | 2 +- storage/innobase/row/row0merge.cc | 5 ----- 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/mysql-test/suite/innodb/r/default_row_format_alter.result b/mysql-test/suite/innodb/r/default_row_format_alter.result index 42cbab8a5f2..33936b59003 100644 --- a/mysql-test/suite/innodb/r/default_row_format_alter.result +++ b/mysql-test/suite/innodb/r/default_row_format_alter.result @@ -129,5 +129,25 @@ SELECT ROW_FORMAT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME='t1'; ROW_FORMAT Dynamic DROP TABLE t1; +# +# MDEV-31025 Redundant table alter fails when fixed column +# stored externally +# +set @old_sql_mode = @@sql_mode; +SET @@sql_mode=''; +CREATE TABLE t1(pk INT,c CHAR(255),c2 CHAR(255),c3 CHAR(255), +c4 char(255), c5 char(255), c6 char(255), +c7 char(255), c8 char(255), primary key(pk) +)Engine=InnoDB character set utf32 ROW_FORMAT=REDUNDANT; +INSERT INTO t1(pk, c) VALUES (1, repeat('a', 255)); +ALTER TABLE t1 FORCE; +CHECK TABLE t1; +Table Op Msg_type Msg_text +test.t1 check status OK +SELECT LENGTH(c) FROM t1; +LENGTH(c) +1020 +DROP TABLE t1; +set @@sql_mode = @old_sql_mode; # End of 10.4 tests SET GLOBAL innodb_default_row_format = @row_format; diff --git a/mysql-test/suite/innodb/t/default_row_format_alter.test b/mysql-test/suite/innodb/t/default_row_format_alter.test index f5dd246efb5..5f2170454f3 100644 --- a/mysql-test/suite/innodb/t/default_row_format_alter.test +++ b/mysql-test/suite/innodb/t/default_row_format_alter.test @@ -150,6 +150,23 @@ ALTER TABLE t1 DROP b; SELECT ROW_FORMAT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME='t1'; DROP TABLE t1; +--echo # +--echo # MDEV-31025 Redundant table alter fails when fixed column +--echo # stored externally +--echo # +set @old_sql_mode = @@sql_mode; +SET @@sql_mode=''; +CREATE TABLE t1(pk INT,c CHAR(255),c2 CHAR(255),c3 CHAR(255), + c4 char(255), c5 char(255), c6 char(255), + c7 char(255), c8 char(255), primary key(pk) + )Engine=InnoDB character set utf32 ROW_FORMAT=REDUNDANT; +INSERT INTO t1(pk, c) VALUES (1, repeat('a', 255)); +ALTER TABLE t1 FORCE; +CHECK TABLE t1; +SELECT LENGTH(c) FROM t1; +DROP TABLE t1; +set @@sql_mode = @old_sql_mode; + --echo # End of 10.4 tests SET GLOBAL innodb_default_row_format = @row_format; diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index f54a8e1125a..2df94715750 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -4713,7 +4713,7 @@ n_field_mismatch: len -= BTR_EXTERN_FIELD_REF_SIZE; ulint extern_len = mach_read_from_4( data + len + BTR_EXTERN_LEN + 4); - if (fixed_size == extern_len) { + if (fixed_size == extern_len + len) { goto next_field; } } diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index 9d755ce6f1e..f0aed489f22 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -681,11 +681,6 @@ error: row_field, field, col->len, old_table->space->zip_size(), conv_heap); - } else { - /* Field length mismatch should not - happen when rebuilding redundant row - format table. */ - ut_ad(index->table->not_redundant()); } } } -- cgit v1.2.1 From 660afb1e9c11d4fe1ba806557c60cda3f62b1be1 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Thu, 13 Apr 2023 16:26:03 +0530 Subject: MDEV-30076 ibuf_insert tries to insert the entry for uncommitted index - Change buffer should not buffer the changes for uncommitted index --- storage/innobase/ibuf/ibuf0ibuf.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 3fe74c3a270..d611c7793f7 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -3577,6 +3577,10 @@ ibuf_insert( ulint zip_size, que_thr_t* thr) { + if (!index->is_committed()) { + return false; + } + dberr_t err; ulint entry_size; ibool no_counter; -- cgit v1.2.1 From 1892f5d8fcb73350973f93b1a89cdc1482a5c116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 19 Apr 2023 14:46:49 +0300 Subject: MDEV-30863 fixup: Hang in a debug build trx_assign_rseg_low(): Correct a debug injection condition. --- storage/innobase/trx/trx0trx.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index ed1187e179b..a84ae220839 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -821,7 +821,7 @@ static void trx_assign_rseg_low(trx_t *trx) rseg = &trx_sys.rseg_array[slot]; do { - ut_d(if (!trx_rseg_n_slots_debug) continue); + ut_d(if (trx_rseg_n_slots_debug) continue); slot = (slot + 1) % TRX_SYS_N_RSEGS; ut_ad(start_scan_slot != slot); } while (0); -- cgit v1.2.1 From 210db2935cb3802f6806ba3b23c32263611e3e2f Mon Sep 17 00:00:00 2001 From: Denis Protivensky Date: Tue, 14 Mar 2023 14:08:12 +0300 Subject: MDEV-30804 Rollback multi-engine transaction requiring 2PC but committing in one phase Signed-off-by: Julius Goryavsky --- mysql-test/suite/galera/r/MDEV-30804.result | 11 +++++++++++ mysql-test/suite/galera/t/MDEV-30804.cnf | 7 +++++++ mysql-test/suite/galera/t/MDEV-30804.test | 21 +++++++++++++++++++++ sql/handler.cc | 14 +++++++++++++- 4 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 mysql-test/suite/galera/r/MDEV-30804.result create mode 100644 mysql-test/suite/galera/t/MDEV-30804.cnf create mode 100644 mysql-test/suite/galera/t/MDEV-30804.test diff --git a/mysql-test/suite/galera/r/MDEV-30804.result b/mysql-test/suite/galera/r/MDEV-30804.result new file mode 100644 index 00000000000..2bf323d19f8 --- /dev/null +++ b/mysql-test/suite/galera/r/MDEV-30804.result @@ -0,0 +1,11 @@ +connection node_2; +connection node_1; +CREATE TABLE t (a INT) ENGINE=Aria; +CREATE TABLE t1 (a INT) ENGINE=InnoDB; +START TRANSACTION; +INSERT INTO t VALUES ('1'); +INSERT INTO t1 VALUES ('1'); +COMMIT; +ERROR HY000: Transactional commit not supported by involved engine(s) +DROP TABLE t; +DROP TABLE t1; diff --git a/mysql-test/suite/galera/t/MDEV-30804.cnf b/mysql-test/suite/galera/t/MDEV-30804.cnf new file mode 100644 index 00000000000..9dbd81f758d --- /dev/null +++ b/mysql-test/suite/galera/t/MDEV-30804.cnf @@ -0,0 +1,7 @@ +!include ../galera_2nodes.cnf + +[mysqld.1] +log-bin + +[mysqld.2] +log-bin diff --git a/mysql-test/suite/galera/t/MDEV-30804.test b/mysql-test/suite/galera/t/MDEV-30804.test new file mode 100644 index 00000000000..561953a0578 --- /dev/null +++ b/mysql-test/suite/galera/t/MDEV-30804.test @@ -0,0 +1,21 @@ +# +# Test that transaction requiring two-phase commit and involving +# storage engines not supporting it rolls back with a message. +# + +--source include/galera_cluster.inc +--source include/have_innodb.inc +--source include/have_aria.inc + +CREATE TABLE t (a INT) ENGINE=Aria; +CREATE TABLE t1 (a INT) ENGINE=InnoDB; + +START TRANSACTION; +INSERT INTO t VALUES ('1'); +INSERT INTO t1 VALUES ('1'); + +--error ER_ERROR_DURING_COMMIT +COMMIT; + +DROP TABLE t; +DROP TABLE t1; diff --git a/sql/handler.cc b/sql/handler.cc index 4afd30021ee..e0dd51376ad 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -1733,7 +1733,19 @@ int ha_commit_trans(THD *thd, bool all) ordering is normally done. Commit ordering must be done here. */ if (run_wsrep_hooks) - error= wsrep_before_commit(thd, all); + { + // This commit involves more than one storage engine and requires + // two phases, but some engines don't support it. + // Issue a message to the client and roll back the transaction. + if (trans->no_2pc && rw_ha_count > 1) + { + my_message(ER_ERROR_DURING_COMMIT, "Transactional commit not supported " + "by involved engine(s)", MYF(0)); + error= 1; + } + else + error= wsrep_before_commit(thd, all); + } if (error) { ha_rollback_trans(thd, FALSE); -- cgit v1.2.1 From 78368e5866383333407a4e2752c8b62df8fb8c29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 19 Apr 2023 15:52:11 +0300 Subject: MDEV-30863 fixup: Assertion failure when using innodb_undo_tablespaces=0 trx_assign_rseg_low(): Let us restore the debug variable look_for_rollover to avoid assertion failures when a server that was created with multiple undo tablespaces is being started with innodb_undo_tablespaces=0. --- storage/innobase/trx/trx0trx.cc | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index a84ae220839..88e42b2ebad 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -812,6 +812,7 @@ static void trx_assign_rseg_low(trx_t *trx) unsigned slot = rseg_slot++ % TRX_SYS_N_RSEGS; ut_d(if (trx_rseg_n_slots_debug) slot = 0); ut_d(const auto start_scan_slot = slot); + ut_d(bool look_for_rollover = false); trx_rseg_t* rseg; bool allocated; @@ -819,12 +820,10 @@ static void trx_assign_rseg_low(trx_t *trx) do { for (;;) { rseg = &trx_sys.rseg_array[slot]; - - do { - ut_d(if (trx_rseg_n_slots_debug) continue); - slot = (slot + 1) % TRX_SYS_N_RSEGS; - ut_ad(start_scan_slot != slot); - } while (0); + ut_ad(!look_for_rollover || start_scan_slot != slot); + ut_d(look_for_rollover = true); + ut_d(if (!trx_rseg_n_slots_debug)) + slot = (slot + 1) % TRX_SYS_N_RSEGS; if (!rseg->space) { continue; -- cgit v1.2.1 From 854e8b189e422e2d2e61e66380a18b9f8e33646d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 19 Apr 2023 15:53:26 +0300 Subject: MDEV-28976 fixup: A better fix do_shutdown_server(): After sending SIGKILL, invoke wait_until_dead(). Thanks to Sergei Golubchik for pointing out that the previous fix does not actually work. --- client/mysqltest.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/mysqltest.cc b/client/mysqltest.cc index 9e9b122c5fe..e745f8a3d2f 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -5191,9 +5191,10 @@ void do_shutdown_server(struct st_command *command) if (timeout) (void) my_kill(pid, SIGABRT); /* Give server a few seconds to die in all cases */ - if (wait_until_dead(pid, timeout < 5 ? 5 : timeout)) + if (!timeout || wait_until_dead(pid, timeout < 5 ? 5 : timeout)) { (void) my_kill(pid, SIGKILL); + wait_until_dead(pid, 5); } } DBUG_VOID_RETURN; -- cgit v1.2.1 From 0cda0e4e150864c0bd0062d45d554508bc77fc90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 19 Apr 2023 18:56:58 +0300 Subject: MDEV-31080 fil_validate() failures during deferred tablespace recovery fil_space_t::create(), fil_space_t::add(): Expect the caller to acquire and release fil_system.mutex. In this way, creating a tablespace and adding the first (usually only) data file will be atomic. recv_sys_t::recover_deferred(): Correctly protect some changes by holding fil_system.mutex. Tested by: Matthias Leich --- extra/mariabackup/xtrabackup.cc | 17 ++++++++++------- storage/innobase/fil/fil0fil.cc | 29 +++++++++++++++++++---------- storage/innobase/fsp/fsp0space.cc | 24 +++++++++++++----------- storage/innobase/fsp/fsp0sysspace.cc | 10 ++++++---- storage/innobase/log/log0recv.cc | 18 ++++++++++++------ storage/innobase/srv/srv0start.cc | 6 ++---- 6 files changed, 62 insertions(+), 42 deletions(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 9de1ef853b9..0a0ba9cd98e 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -3454,20 +3454,20 @@ static void xb_load_single_table_tablespace(const char *dirname, bool is_empty_file = file->exists() && file->is_empty_file(); if (err == DB_SUCCESS && file->space_id() != SRV_TMP_SPACE_ID) { + mysql_mutex_lock(&fil_system.mutex); space = fil_space_t::create( file->space_id(), file->flags(), FIL_TYPE_TABLESPACE, nullptr/* TODO: crypt_data */, FIL_ENCRYPTION_DEFAULT, file->handle() != OS_FILE_CLOSED); - - ut_a(space != NULL); + ut_ad(space); fil_node_t* node= space->add( file->filepath(), skip_node_page0 ? file->detach() : pfs_os_file_t(), 0, false, false); node->deferred= defer; - mysql_mutex_lock(&fil_system.mutex); - space->read_page0(); + if (!space->read_page0()) + err = DB_CANNOT_OPEN_FILE; mysql_mutex_unlock(&fil_system.mutex); if (srv_operation == SRV_OPERATION_RESTORE_DELTA @@ -5324,9 +5324,12 @@ exit: ut_ad(fil_space_t::zip_size(flags) == info.zip_size); ut_ad(fil_space_t::physical_size(flags) == info.page_size); - if (fil_space_t::create(info.space_id, flags, - FIL_TYPE_TABLESPACE, 0, FIL_ENCRYPTION_DEFAULT, - true)) { + mysql_mutex_lock(&fil_system.mutex); + fil_space_t* space = fil_space_t::create(info.space_id, flags, + FIL_TYPE_TABLESPACE, 0, + FIL_ENCRYPTION_DEFAULT, true); + mysql_mutex_unlock(&fil_system.mutex); + if (space) { *success = xb_space_create_file(real_name, info.space_id, flags, &file); } else { diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 19ebdc8d67e..48d205f428a 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -311,6 +311,8 @@ fil_node_t* fil_space_t::add(const char* name, pfs_os_file_t handle, uint32_t size, bool is_raw, bool atomic_write, uint32_t max_pages) { + mysql_mutex_assert_owner(&fil_system.mutex); + fil_node_t* node; ut_ad(name != NULL); @@ -335,7 +337,6 @@ fil_node_t* fil_space_t::add(const char* name, pfs_os_file_t handle, node->atomic_write = atomic_write; - mysql_mutex_lock(&fil_system.mutex); this->size += size; UT_LIST_ADD_LAST(chain, node); if (node->is_open()) { @@ -346,7 +347,6 @@ fil_node_t* fil_space_t::add(const char* name, pfs_os_file_t handle, release(); } } - mysql_mutex_unlock(&fil_system.mutex); return node; } @@ -946,6 +946,7 @@ fil_space_t *fil_space_t::create(ulint id, ulint flags, { fil_space_t* space; + mysql_mutex_assert_owner(&fil_system.mutex); ut_ad(fil_system.is_initialised()); ut_ad(fil_space_t::is_valid_flags(flags & ~FSP_FLAGS_MEM_MASK, id)); ut_ad(srv_page_size == UNIV_PAGE_SIZE_ORIG || flags != 0); @@ -978,8 +979,6 @@ fil_space_t *fil_space_t::create(ulint id, ulint flags, space->latch.SRW_LOCK_INIT(fil_space_latch_key); - mysql_mutex_lock(&fil_system.mutex); - if (const fil_space_t *old_space = fil_space_get_by_id(id)) { ib::error() << "Trying to add tablespace with id " << id << " to the cache, but tablespace '" @@ -987,7 +986,6 @@ fil_space_t *fil_space_t::create(ulint id, ulint flags, ? old_space->chain.start->name : "") << "' already exists in the cache!"; - mysql_mutex_unlock(&fil_system.mutex); space->~fil_space_t(); ut_free(space); return(NULL); @@ -1034,12 +1032,12 @@ fil_space_t *fil_space_t::create(ulint id, ulint flags, if (rotate) { fil_system.default_encrypt_tables.push_back(*space); space->is_in_default_encrypt = true; - } - mysql_mutex_unlock(&fil_system.mutex); - - if (rotate && srv_n_fil_crypt_threads_started) { - fil_crypt_threads_signal(); + if (srv_n_fil_crypt_threads_started) { + mysql_mutex_unlock(&fil_system.mutex); + fil_crypt_threads_signal(); + mysql_mutex_lock(&fil_system.mutex); + } } return(space); @@ -1998,16 +1996,20 @@ err_exit: DBUG_EXECUTE_IF("checkpoint_after_file_create", log_make_checkpoint();); + mysql_mutex_lock(&fil_system.mutex); if (fil_space_t* space = fil_space_t::create(space_id, flags, FIL_TYPE_TABLESPACE, crypt_data, mode, true)) { fil_node_t* node = space->add(path, file, size, false, true); + mysql_mutex_unlock(&fil_system.mutex); IF_WIN(node->find_metadata(), node->find_metadata(file, true)); mtr.start(); mtr.set_named_space(space); ut_a(fsp_header_init(space, size, &mtr) == DB_SUCCESS); mtr.commit(); return space; + } else { + mysql_mutex_unlock(&fil_system.mutex); } if (space_name.data()) { @@ -2267,8 +2269,10 @@ skip_validate: first_page) : NULL; + mysql_mutex_lock(&fil_system.mutex); space = fil_space_t::create(id, flags, purpose, crypt_data); if (!space) { + mysql_mutex_unlock(&fil_system.mutex); goto error; } @@ -2278,6 +2282,7 @@ skip_validate: space->add( df_remote.is_open() ? df_remote.filepath() : df_default.filepath(), OS_FILE_CLOSED, 0, false, true); + mysql_mutex_unlock(&fil_system.mutex); if (must_validate && !srv_read_only_mode) { df_remote.close(); @@ -2566,10 +2571,13 @@ tablespace_check: return FIL_LOAD_INVALID; } + mysql_mutex_lock(&fil_system.mutex); + space = fil_space_t::create( space_id, flags, FIL_TYPE_TABLESPACE, crypt_data); if (space == NULL) { + mysql_mutex_unlock(&fil_system.mutex); return(FIL_LOAD_INVALID); } @@ -2581,6 +2589,7 @@ tablespace_check: let fil_node_open() do that task. */ space->add(file.filepath(), OS_FILE_CLOSED, 0, false, false); + mysql_mutex_unlock(&fil_system.mutex); return(FIL_LOAD_OK); } diff --git a/storage/innobase/fsp/fsp0space.cc b/storage/innobase/fsp/fsp0space.cc index b069250ff9f..8c6344946a4 100644 --- a/storage/innobase/fsp/fsp0space.cc +++ b/storage/innobase/fsp/fsp0space.cc @@ -88,25 +88,25 @@ Tablespace::open_or_create(bool is_temp) ut_ad(!m_files.empty()); for (iterator it = begin(); it != end(); ++it) { - if (it->m_exists) { err = it->open_or_create( m_ignore_read_only ? false : srv_read_only_mode); + if (err != DB_SUCCESS) { + return err; + } } else { err = it->open_or_create( m_ignore_read_only ? false : srv_read_only_mode); - /* Set the correct open flags now that we have - successfully created the file. */ - if (err == DB_SUCCESS) { - file_found(*it); + if (err != DB_SUCCESS) { + return err; } - } - if (err != DB_SUCCESS) { - break; + /* Set the correct open flags now that we have + successfully created the file. */ + file_found(*it); } /* We can close the handle now and open the tablespace @@ -130,20 +130,22 @@ Tablespace::open_or_create(bool is_temp) fsp_flags = FSP_FLAGS_PAGE_SSIZE(); } + mysql_mutex_lock(&fil_system.mutex); space = fil_space_t::create( m_space_id, fsp_flags, is_temp ? FIL_TYPE_TEMPORARY : FIL_TYPE_TABLESPACE, NULL); if (!space) { + mysql_mutex_unlock(&fil_system.mutex); return DB_ERROR; } + } else { + mysql_mutex_lock(&fil_system.mutex); } - - ut_a(fil_validate()); - space->add(it->m_filepath, OS_FILE_CLOSED, it->m_size, false, true); + mysql_mutex_unlock(&fil_system.mutex); } return(err); diff --git a/storage/innobase/fsp/fsp0sysspace.cc b/storage/innobase/fsp/fsp0sysspace.cc index 497e4100557..577db8eb9f2 100644 --- a/storage/innobase/fsp/fsp0sysspace.cc +++ b/storage/innobase/fsp/fsp0sysspace.cc @@ -921,6 +921,7 @@ SysTablespace::open_or_create( /* Close the curent handles, add space and file info to the fil_system cache and the Data Dictionary, and re-open them in file_system cache so that they stay open until shutdown. */ + mysql_mutex_lock(&fil_system.mutex); ulint node_counter = 0; for (files_t::iterator it = begin; it != end; ++it) { it->close(); @@ -934,7 +935,8 @@ SysTablespace::open_or_create( FIL_TYPE_TEMPORARY, NULL); ut_ad(space == fil_system.temp_space); if (!space) { - return DB_ERROR; + err = DB_ERROR; + break; } ut_ad(!space->is_compressed()); ut_ad(space->full_crc32()); @@ -945,12 +947,11 @@ SysTablespace::open_or_create( FIL_TYPE_TABLESPACE, NULL); ut_ad(space == fil_system.sys_space); if (!space) { - return DB_ERROR; + err = DB_ERROR; + break; } } - ut_a(fil_validate()); - uint32_t max_size = (++node_counter == m_files.size() ? (m_last_file_size_max == 0 ? UINT32_MAX @@ -961,6 +962,7 @@ SysTablespace::open_or_create( it->m_type != SRV_NOT_RAW, true, max_size); } + mysql_mutex_unlock(&fil_system.mutex); return(err); } diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 3b6e3008a95..f87ff15c393 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -785,9 +785,10 @@ retry: if (!os_file_status(name->c_str(), &exists, &ftype) || !exists) goto processed; } - create(it, *name, static_cast - (1U << FSP_FLAGS_FCRC32_POS_MARKER | - FSP_FLAGS_FCRC32_PAGE_SSIZE()), nullptr, 0); + if (create(it, *name, static_cast + (1U << FSP_FLAGS_FCRC32_POS_MARKER | + FSP_FLAGS_FCRC32_PAGE_SSIZE()), nullptr, 0)) + mysql_mutex_unlock(&fil_system.mutex); } } else @@ -816,7 +817,7 @@ processed: @param flags FSP_SPACE_FLAGS @param crypt_data encryption metadata @param size tablespace size in pages - @return tablespace + @return tablespace; the caller must release fil_system.mutex @retval nullptr if crypt_data is invalid */ static fil_space_t *create(const recv_spaces_t::const_iterator &it, const std::string &name, uint32_t flags, @@ -828,6 +829,7 @@ processed: ut_free(crypt_data); return nullptr; } + mysql_mutex_lock(&fil_system.mutex); fil_space_t *space= fil_space_t::create(it->first, flags, FIL_TYPE_TABLESPACE, crypt_data); ut_ad(space); @@ -900,12 +902,13 @@ processed: space->free_limit= fsp_header_get_field(page, FSP_FREE_LIMIT); space->free_len= flst_get_len(FSP_HEADER_OFFSET + FSP_FREE + page); fil_node_t *node= UT_LIST_GET_FIRST(space->chain); + mysql_mutex_unlock(&fil_system.mutex); if (!space->acquire()) - { + { free_space: fil_space_free(it->first, false); goto next_item; - } + } if (os_file_write(IORequestWrite, node->name, node->handle, page, 0, fil_space_t::physical_size(flags)) != DB_SUCCESS) @@ -975,6 +978,7 @@ bool recv_sys_t::recover_deferred(recv_sys_t::map::iterator &p, space->free_len= flst_get_len(FSP_HEADER_OFFSET + FSP_FREE + page); fil_node_t *node= UT_LIST_GET_FIRST(space->chain); node->deferred= true; + mysql_mutex_unlock(&fil_system.mutex); if (!space->acquire()) goto release_and_fail; fil_names_dirty(space); @@ -998,8 +1002,10 @@ bool recv_sys_t::recover_deferred(recv_sys_t::map::iterator &p, uint32_t(file_size / fil_space_t::physical_size(flags)); if (n_pages > size) { + mysql_mutex_lock(&fil_system.mutex); space->size= node->size= n_pages; space->set_committed_size(); + mysql_mutex_unlock(&fil_system.mutex); goto size_set; } } diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index a1368c5146c..707804f2206 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -559,14 +559,12 @@ err_exit: fil_set_max_space_id_if_bigger(space_id); + mysql_mutex_lock(&fil_system.mutex); fil_space_t *space= fil_space_t::create(space_id, fsp_flags, FIL_TYPE_TABLESPACE, nullptr, FIL_ENCRYPTION_DEFAULT, true); - ut_a(fil_validate()); - ut_a(space); - + ut_ad(space); fil_node_t *file= space->add(name, fh, 0, false, true); - mysql_mutex_lock(&fil_system.mutex); if (create) { -- cgit v1.2.1 From 27ff972be22880a4046652bc94c2f97fffb456c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 19 Apr 2023 18:57:18 +0300 Subject: MDEV-26827 fixup: Do not hog buf_pool.mutex buf_flush_LRU_list_batch(): When evicting clean pages, release and reacquire the buf_pool.mutex after every 32 pages. Also, eliminate some conditional branches. --- storage/innobase/buf/buf0flu.cc | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 5a9e3cbb34e..3ef70741da1 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1255,6 +1255,11 @@ static void buf_flush_LRU_list_batch(ulint max, bool evict, ++n->evicted; /* fall through */ case 1: + if (UNIV_LIKELY(scanned & 31)) + continue; + mysql_mutex_unlock(&buf_pool.mutex); + reacquire_mutex: + mysql_mutex_lock(&buf_pool.mutex); continue; } @@ -1290,32 +1295,37 @@ static void buf_flush_LRU_list_batch(ulint max, bool evict, auto p= buf_flush_space(space_id); space= p.first; last_space_id= space_id; + if (!space) + { + mysql_mutex_lock(&buf_pool.mutex); + goto no_space; + } mysql_mutex_lock(&buf_pool.mutex); - if (p.second) - buf_pool.stat.n_pages_written+= p.second; + buf_pool.stat.n_pages_written+= p.second; } else + { ut_ad(!space); + goto no_space; + } } else if (space->is_stopping()) { space->release(); space= nullptr; - } - - if (!space) - { + no_space: mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_flush_discard_page(bpage); + continue; } - else if (neighbors && space->is_rotational()) + + if (neighbors && space->is_rotational()) { mysql_mutex_unlock(&buf_pool.mutex); n->flushed+= buf_flush_try_neighbors(space, page_id, bpage, neighbors == 1, do_evict, n->flushed, max); -reacquire_mutex: - mysql_mutex_lock(&buf_pool.mutex); + goto reacquire_mutex; } else if (n->flushed >= max && !recv_recovery_is_on()) { -- cgit v1.2.1 From f7791cc7cbedfb29b7cdd68df000108988f77fd3 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 20 Apr 2023 10:10:23 +1000 Subject: Revert "MDEV-30186 Use of uninitialized value in substitution" This reverts commit 0e737f78980fcfe83b05c27215eb3f5ede1ea473. As noted by Andrew, this introduces race conditions in the setting and using of the global $test_name_for_report. --- mysql-test/mariadb-test-run.pl | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/mysql-test/mariadb-test-run.pl b/mysql-test/mariadb-test-run.pl index 670f63a990f..46a51d2fa58 100755 --- a/mysql-test/mariadb-test-run.pl +++ b/mysql-test/mariadb-test-run.pl @@ -145,7 +145,6 @@ my $opt_start_exit; my $start_only; my $file_wsrep_provider; my $num_saved_cores= 0; # Number of core files saved in vardir/log/ so far. -my $test_name_for_report; our @global_suppressions; @@ -516,13 +515,13 @@ sub main { } if ( not @$completed ) { - if ($test_name_for_report) - { - my $tinfo = My::Test->new(name => $test_name_for_report); - $tinfo->{result}= 'MTR_RES_FAILED'; - $tinfo->{comment}=' '; - mtr_report_test($tinfo); - } + my $test_name= mtr_grab_file($path_testlog); + $test_name =~ s/^CURRENT_TEST:\s//; + chomp($test_name); + my $tinfo = My::Test->new(name => $test_name); + $tinfo->{result}= 'MTR_RES_FAILED'; + $tinfo->{comment}=' '; + mtr_report_test($tinfo); mtr_error("Test suite aborted"); } @@ -3741,8 +3740,8 @@ sub resfile_report_test ($) { sub run_testcase ($$) { my ($tinfo, $server_socket)= @_; my $print_freq=20; - $test_name_for_report= $tinfo->{name}; - mtr_verbose("Running test:", $test_name_for_report); + + mtr_verbose("Running test:", $tinfo->{name}); $ENV{'MTR_TEST_NAME'} = $tinfo->{name}; resfile_report_test($tinfo) if $opt_resfile; @@ -5131,10 +5130,12 @@ sub mysqld_start ($$) { if (!$rc) { # Report failure about the last test case before exit - my $tinfo = My::Test->new(name => $test_name_for_report); + my $test_name= mtr_grab_file($path_current_testlog); + $test_name =~ s/^CURRENT_TEST:\s//; + my $tinfo = My::Test->new(name => $test_name); $tinfo->{result}= 'MTR_RES_FAILED'; $tinfo->{failures}= 1; - $tinfo->{logfile}=get_log_from_proc($mysqld->{'proc'}, $test_name_for_report); + $tinfo->{logfile}=get_log_from_proc($mysqld->{'proc'}, $tinfo->{name}); report_option('verbose', 1); mtr_report_test($tinfo); } -- cgit v1.2.1 From c21bc17a51f6eef6158ef80e7deeb5a280598c1d Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Thu, 20 Apr 2023 10:13:12 +1000 Subject: MDEV-30186: mtr: Use of uninitialized value $test_name in substitution There is an assumption that when there are are no completed tests, that means they are still running and then an attempt is made to identify these tests as stalled. The other possibility is however there are no tests that where run. Test this early and then exit quickly and no later misunderstandings need to be made. --- mysql-test/mariadb-test-run.pl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mysql-test/mariadb-test-run.pl b/mysql-test/mariadb-test-run.pl index 46a51d2fa58..4fc4d9f050e 100755 --- a/mysql-test/mariadb-test-run.pl +++ b/mysql-test/mariadb-test-run.pl @@ -401,6 +401,11 @@ sub main { mtr_report("Collecting tests..."); my $tests= collect_test_cases($opt_reorder, $opt_suites, \@opt_cases, \@opt_skip_test_list); + if (@$tests == 0) { + mtr_report("No tests to run..."); + exit 0; + } + mark_time_used('collect'); mysql_install_db(default_mysqld(), "$opt_vardir/install.db") unless using_extern(); -- cgit v1.2.1 From 7e31a8e7fa97a87fc164381588d172bf0e76abb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 20 Apr 2023 14:08:48 +0300 Subject: MDEV-26827 fixup: Fix os_aio_wait_until_no_pending_writes() io_callback(): Process the request before releasing the write slot. Before commit a091d6ac4e7d2d7873749e685943b3032ccfda57 when we had a duplicated counter for writes, either ordering was fine. Now, correctness depends on os_aio_wait_until_no_pending_writes(). --- storage/innobase/os/os0file.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index d366c784b96..e816f6ef7b1 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -3457,9 +3457,8 @@ static void io_callback(tpool::aiocb *cb) else { ut_ad(write_slots->contains(cb)); - const IORequest req{request}; + fil_aio_callback(request); write_slots->release(cb); - fil_aio_callback(req); } } -- cgit v1.2.1 From fc6e8a3d3264078bed28632a289130b1dc24daea Mon Sep 17 00:00:00 2001 From: Mikhail Chalov Date: Tue, 31 Jan 2023 14:14:55 -0800 Subject: Minimize unsafe C functions usage - replace strcat() and strcpy() Similar to 567b6812 continue to replace use of strcat() and strcpy() with safer options strncat() and strncpy(). All new code of the whole pull request, including one or several files that are either new files or modified ones, are contributed under the BSD-new license. I am contributing on behalf of my employer Amazon Web Services --- storage/connect/reldef.cpp | 26 ++++++++++--------- storage/connect/tabbson.cpp | 47 +++++++++++++++++----------------- storage/connect/tabdos.cpp | 61 ++++++++++++++++++++++++--------------------- storage/connect/tabext.cpp | 55 ++++++++++++++++++++++++---------------- storage/connect/tabfmt.cpp | 38 ++++++++++++++++------------ storage/connect/tabjdbc.cpp | 45 +++++++++++++++++---------------- storage/connect/tabjson.cpp | 54 +++++++++++++++++++-------------------- 7 files changed, 178 insertions(+), 148 deletions(-) diff --git a/storage/connect/reldef.cpp b/storage/connect/reldef.cpp index 144d31735d6..786a53db5a2 100644 --- a/storage/connect/reldef.cpp +++ b/storage/connect/reldef.cpp @@ -91,11 +91,11 @@ PQRYRES OEMColumns(PGLOBAL g, PTOS topt, char* tab, char* db, bool info) /* directories are used (to make this even remotely secure). */ /*********************************************************************/ if (check_valid_path(module, strlen(module))) { - strcpy(g->Message, "Module cannot contain a path"); + safe_strcpy(g->Message, sizeof(g->Message), "Module cannot contain a path"); return NULL; } else if (strlen(subtype)+1+3 >= sizeof(getname)) { - strcpy(g->Message, "Subtype string too long"); + safe_strcpy(g->Message, sizeof(g->Message), "Subtype string too long"); return NULL; } else @@ -118,7 +118,8 @@ PQRYRES OEMColumns(PGLOBAL g, PTOS topt, char* tab, char* db, bool info) FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, rc, 0, (LPTSTR)buf, sizeof(buf), NULL); - strcat(strcat(g->Message, ": "), buf); + safe_strcat(g->Message, sizeof(g->Message), ": "); + safe_strcat(g->Message, sizeof(g->Message), buf); return NULL; } // endif hDll @@ -281,7 +282,7 @@ char *RELDEF::GetStringCatInfo(PGLOBAL g, PCSZ what, PCSZ sdef) if (IsFileType(GetTypeID(ftype))) { name= Hc->GetPartName(); sval= (char*)PlugSubAlloc(g, NULL, strlen(name) + 12); - strcat(strcpy(sval, name), "."); + snprintf(sval, strlen(name) + 12, "%s.", name); n= strlen(sval); // Fold ftype to lower case @@ -622,12 +623,11 @@ PTABDEF OEMDEF::GetXdef(PGLOBAL g) /* directories are used (to make this even remotely secure). */ /*********************************************************************/ if (check_valid_path(Module, strlen(Module))) { - strcpy(g->Message, "Module cannot contain a path"); + safe_strcpy(g->Message, sizeof(g->Message), "Module cannot contain a path"); return NULL; } else // PlugSetPath(soname, Module, GetPluginDir()); // Crashes on Fedora - strncat(strcpy(soname, GetPluginDir()), Module, - sizeof(soname) - strlen(soname) - 1); + snprintf(soname, sizeof(soname), "%s%s", GetPluginDir(), Module); #if defined(_WIN32) // Is the DLL already loaded? @@ -641,7 +641,8 @@ PTABDEF OEMDEF::GetXdef(PGLOBAL g) FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, rc, 0, (LPTSTR)buf, sizeof(buf), NULL); - strcat(strcat(g->Message, ": "), buf); + safe_strcat(g->Message, sizeof(g->Message), ": "); + safe_strcat(g->Message, sizeof(g->Message), buf); return NULL; } // endif hDll @@ -661,7 +662,8 @@ PTABDEF OEMDEF::GetXdef(PGLOBAL g) FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, rc, 0, (LPTSTR)buf, sizeof(buf), NULL); - strcat(strcat(g->Message, ": "), buf); + safe_strcat(g->Message, sizeof(g->Message), ": "); + safe_strcat(g->Message, sizeof(g->Message), buf); FreeLibrary((HMODULE)Hdll); return NULL; } // endif getdef @@ -810,7 +812,7 @@ PTDB OEMDEF::GetTable(PGLOBAL g, MODE mode) else txfp = new(g) ZLBFAM(defp); #else // !GZ_SUPPORT - strcpy(g->Message, "Compress not supported"); + safe_strcpy(g->Message, sizeof(g->Message), "Compress not supported"); return NULL; #endif // !GZ_SUPPORT } else if (rfm == RECFM_VAR) { @@ -833,7 +835,7 @@ PTDB OEMDEF::GetTable(PGLOBAL g, MODE mode) else txfp = new(g) VCTFAM((PVCTDEF)defp); #else // !VCT_SUPPORT - strcpy(g->Message, "VCT no more supported"); + safe_strcpy(g->Message, sizeof(g->Message), "VCT no more supported"); return NULL; #endif // !VCT_SUPPORT } // endif's @@ -924,7 +926,7 @@ int COLDEF::Define(PGLOBAL g, void *, PCOLINFO cfp, int poff) return -1; } // endswitch - strcpy(F.Type, GetFormatType(Buf_Type)); + safe_strcpy(F.Type, sizeof(F.Type), GetFormatType(Buf_Type)); F.Length = cfp->Length; F.Prec = cfp->Scale; Offset = (cfp->Offset < 0) ? poff : cfp->Offset; diff --git a/storage/connect/tabbson.cpp b/storage/connect/tabbson.cpp index 22d8648d7c0..2ea74da94b0 100644 --- a/storage/connect/tabbson.cpp +++ b/storage/connect/tabbson.cpp @@ -39,6 +39,7 @@ #include "checklvl.h" #include "resource.h" #include "mycat.h" // for FNC_COL +#include "m_string.h" /***********************************************************************/ /* This should be an option. */ @@ -80,7 +81,7 @@ PQRYRES BSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info) } // endif info if (GetIntegerTableOption(g, topt, "Multiple", 0)) { - strcpy(g->Message, "Cannot find column definition for multiple table"); + safe_strcpy(g->Message, sizeof(g->Message), "Cannot find column definition for multiple table"); return NULL; } // endif Multiple @@ -206,7 +207,7 @@ int BSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) tdp->Uri = (dsn && *dsn ? dsn : NULL); if (!tdp->Fn && !tdp->Uri) { - strcpy(g->Message, MSG(MISSING_FNAME)); + safe_strcpy(g->Message, sizeof(g->Message), MSG(MISSING_FNAME)); return 0; } else topt->subtype = NULL; @@ -318,7 +319,7 @@ int BSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) switch (tjnp->ReadDB(g)) { case RC_EF: - strcpy(g->Message, "Void json table"); + safe_strcpy(g->Message, sizeof(g->Message), "Void json table"); case RC_FX: goto err; default: @@ -328,7 +329,7 @@ int BSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) } // endif pretty if (!(row = (jsp) ? bp->GetObject(jsp) : NULL)) { - strcpy(g->Message, "Can only retrieve columns from object rows"); + safe_strcpy(g->Message, sizeof(g->Message), "Can only retrieve columns from object rows"); goto err; } // endif row @@ -405,7 +406,7 @@ bool BSONDISC::Find(PGLOBAL g, PBVAL jvp, PCSZ key, int j) if (jvp && !bp->IsJson(jvp)) { if (JsonAllPath() && !fmt[bf]) - strcat(fmt, colname); + safe_strcat(fmt, sizeof(fmt), colname); jcol.Type = (JTYP)jvp->Type; @@ -439,7 +440,7 @@ bool BSONDISC::Find(PGLOBAL g, PBVAL jvp, PCSZ key, int j) jcol.Cbn = true; } else if (j < lvl && !Stringified(strfy, colname)) { if (!fmt[bf]) - strcat(fmt, colname); + safe_strcat(fmt, sizeof(fmt), colname); p = fmt + strlen(fmt); jsp = jvp; @@ -510,11 +511,11 @@ bool BSONDISC::Find(PGLOBAL g, PBVAL jvp, PCSZ key, int j) } else if (lvl >= 0) { if (Stringified(strfy, colname)) { if (!fmt[bf]) - strcat(fmt, colname); + safe_strcat(fmt, sizeof(fmt), colname); - strcat(fmt, ".*"); + safe_strcat(fmt, sizeof(fmt), ".*"); } else if (JsonAllPath() && !fmt[bf]) - strcat(fmt, colname); + safe_strcat(fmt, sizeof(fmt), colname); jcol.Type = TYPE_STRG; jcol.Len = sz; @@ -961,7 +962,7 @@ PVAL BCUTIL::ExpandArray(PGLOBAL g, PBVAL arp, int n) } // endif ars if (!(bvp = GetArrayValue(arp, (nodes[n].Rx = nodes[n].Nx)))) { - strcpy(g->Message, "Logical error expanding array"); + safe_strcpy(g->Message, sizeof(g->Message), "Logical error expanding array"); throw 666; } // endif jvp @@ -1146,7 +1147,7 @@ PBVAL BCUTIL::GetRow(PGLOBAL g) } else if (row->Type == TYPE_JAR) { AddArrayValue(row, (nwr = NewVal(type))); } else { - strcpy(g->Message, "Wrong type when writing new row"); + safe_strcpy(g->Message, sizeof(g->Message), "Wrong type when writing new row"); nwr = NULL; } // endif's @@ -1255,7 +1256,7 @@ PTDB BSONDEF::GetTable(PGLOBAL g, MODE m) // Allocate the parse work memory G = PlugInit(NULL, (size_t)Lrecl * (Pretty < 0 ? 3 : 5)); } else { - strcpy(g->Message, "LRECL is not defined"); + safe_strcpy(g->Message, sizeof(g->Message), "LRECL is not defined"); return NULL; } // endif Lrecl @@ -1295,7 +1296,7 @@ PTDB BSONDEF::GetTable(PGLOBAL g, MODE m) } else if (m == MODE_INSERT) { txfp = new(g) ZIPFAM(this); } else { - strcpy(g->Message, "UPDATE/DELETE not supported for ZIP"); + safe_strcpy(g->Message, sizeof(g->Message), "UPDATE/DELETE not supported for ZIP"); return NULL; } // endif's m #else // !ZIP_SUPPORT @@ -1325,10 +1326,10 @@ PTDB BSONDEF::GetTable(PGLOBAL g, MODE m) if (m == MODE_READ || m == MODE_ANY || m == MODE_ALTER) { txfp = new(g) UNZFAM(this); } else if (m == MODE_INSERT) { - strcpy(g->Message, "INSERT supported only for zipped JSON when pretty=0"); + safe_strcpy(g->Message, sizeof(g->Message), "INSERT supported only for zipped JSON when pretty=0"); return NULL; } else { - strcpy(g->Message, "UPDATE/DELETE not supported for ZIP"); + safe_strcpy(g->Message, sizeof(g->Message), "UPDATE/DELETE not supported for ZIP"); return NULL; } // endif's m #else // !ZIP_SUPPORT @@ -1661,7 +1662,7 @@ bool TDBBSN::PrepareWriting(PGLOBAL g) strcat(s, ","); if ((signed)strlen(s) > Lrecl) { - strncpy(To_Line, s, Lrecl); + safe_strcpy(To_Line, Lrecl, s); snprintf(g->Message, sizeof(g->Message), "Line truncated (lrecl=%d)", Lrecl); return PushWarning(g, this); } else @@ -1764,7 +1765,7 @@ bool BSONCOL::CheckExpand(PGLOBAL g, int i, PSZ nm, bool b) Xpd = true; // Expandable object Nodes[i].Op = OP_EXP; } else if (b) { - strcpy(g->Message, "Cannot expand more than one branch"); + safe_strcpy(g->Message, sizeof(g->Message), "Cannot expand more than one branch"); return true; } // endif Xcol @@ -1975,7 +1976,7 @@ bool BSONCOL::ParseJpath(PGLOBAL g) if (SetArrayOptions(g, p, i, Nodes[i - 1].Key)) return true; else if (Xpd && Tbp->Mode == MODE_DELETE) { - strcpy(g->Message, "Cannot delete expanded columns"); + safe_strcpy(g->Message, sizeof(g->Message), "Cannot delete expanded columns"); return true; } // endif Xpd @@ -2096,7 +2097,7 @@ void BSONCOL::ReadColumn(PGLOBAL g) void BSONCOL::WriteColumn(PGLOBAL g) { if (Xpd && Tbp->Pretty < 2) { - strcpy(g->Message, "Cannot write expanded column when Pretty is not 2"); + safe_strcpy(g->Message, sizeof(g->Message), "Cannot write expanded column when Pretty is not 2"); throw 666; } // endif Xpd @@ -2126,7 +2127,7 @@ void BSONCOL::WriteColumn(PGLOBAL g) char *s = Value->GetCharValue(); if (!(jsp = Cp->ParseJson(g, s, strlen(s)))) { - strcpy(g->Message, s); + safe_strcpy(g->Message, sizeof(g->Message), s); throw 666; } // endif jsp @@ -2312,7 +2313,7 @@ int TDBBSON::MakeDocument(PGLOBAL g) if (!a && *p && *p != '[' && !IsNum(p)) { // obj is a key if (jsp->Type != TYPE_JOB) { - strcpy(g->Message, "Table path does not match the json file"); + safe_strcpy(g->Message, sizeof(g->Message), "Table path does not match the json file"); return RC_FX; } // endif Type @@ -2338,7 +2339,7 @@ int TDBBSON::MakeDocument(PGLOBAL g) } // endif p if (jsp->Type != TYPE_JAR) { - strcpy(g->Message, "Table path does not match the json file"); + safe_strcpy(g->Message, sizeof(g->Message), "Table path does not match the json file"); return RC_FX; } // endif Type @@ -2432,7 +2433,7 @@ void TDBBSON::ResetSize(void) int TDBBSON::MakeIndex(PGLOBAL g, PIXDEF pxdf, bool) { if (pxdf) { - strcpy(g->Message, "JSON not indexable when pretty = 2"); + safe_strcpy(g->Message, sizeof(g->Message), "JSON not indexable when pretty = 2"); return RC_FX; } else return RC_OK; diff --git a/storage/connect/tabdos.cpp b/storage/connect/tabdos.cpp index 797b988b309..4b50af0954e 100644 --- a/storage/connect/tabdos.cpp +++ b/storage/connect/tabdos.cpp @@ -62,6 +62,7 @@ #include "tabmul.h" #include "array.h" #include "blkfil.h" +#include "m_string.h" /***********************************************************************/ /* DB static variables. */ @@ -258,7 +259,7 @@ bool DOSDEF::DeleteIndexFile(PGLOBAL g, PIXDEF pxdf) sep = GetBoolCatInfo("SepIndex", false); if (!sep && pxdf) { - strcpy(g->Message, MSG(NO_RECOV_SPACE)); + safe_strcpy(g->Message, sizeof(g->Message), MSG(NO_RECOV_SPACE)); return true; } // endif sep @@ -293,7 +294,8 @@ bool DOSDEF::DeleteIndexFile(PGLOBAL g, PIXDEF pxdf) for (; pxdf; pxdf = pxdf->GetNext()) { _splitpath(Ofn, drive, direc, fname, NULL); - strcat(strcat(fname, "_"), pxdf->GetName()); + safe_strcat(fname, sizeof(fname), "_"); + safe_strcat(fname, sizeof(fname), pxdf->GetName()); _makepath(filename, drive, direc, fname, ftype); PlugSetPath(filename, filename, GetPath()); #if defined(_WIN32) @@ -312,7 +314,7 @@ bool DOSDEF::DeleteIndexFile(PGLOBAL g, PIXDEF pxdf) } else { // !sep // Drop all indexes, delete the common file PlugSetPath(filename, Ofn, GetPath()); - strcat(PlugRemoveType(filename, filename), ftype); + safe_strcat(PlugRemoveType(filename, filename), sizeof(filename), ftype); #if defined(_WIN32) if (!DeleteFile(filename)) rc = (GetLastError() != ERROR_FILE_NOT_FOUND); @@ -365,7 +367,7 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) if (mode == MODE_READ || mode == MODE_ANY || mode == MODE_ALTER) { txfp = new(g) UZDFAM(this); } else { - strcpy(g->Message, "Zipped DBF tables are read only"); + safe_strcpy(g->Message, sizeof(g->Message), "Zipped DBF tables are read only"); return NULL; } // endif's mode @@ -386,7 +388,7 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) } else if (mode == MODE_INSERT) { txfp = new(g) ZIPFAM(this); } else { - strcpy(g->Message, "UPDATE/DELETE not supported for ZIP"); + safe_strcpy(g->Message, sizeof(g->Message), "UPDATE/DELETE not supported for ZIP"); return NULL; } // endif's mode @@ -397,7 +399,7 @@ PTDB DOSDEF::GetTable(PGLOBAL g, MODE mode) } else if (mode == MODE_INSERT) { txfp = new(g) ZPXFAM(this); } else { - strcpy(g->Message, "UPDATE/DELETE not supported for ZIP"); + safe_strcpy(g->Message, sizeof(g->Message), "UPDATE/DELETE not supported for ZIP"); return NULL; } // endif's mode @@ -654,7 +656,7 @@ int TDBDOS::MakeBlockValues(PGLOBAL g) if ((nrec = defp->GetElemt()) < 2) { if (!To_Def->Partitioned()) { // This may be wrong to do in some cases - strcpy(g->Message, MSG(TABLE_NOT_OPT)); + safe_strcpy(g->Message, sizeof(g->Message), MSG(TABLE_NOT_OPT)); return RC_INFO; // Not to be optimized } else return RC_OK; @@ -674,7 +676,7 @@ int TDBDOS::MakeBlockValues(PGLOBAL g) if ((block = (int)((MaxSize + (int)nrec - 1) / (int)nrec)) < 2) { // This may be wrong to do in some cases defp->RemoveOptValues(g); - strcpy(g->Message, MSG(TABLE_NOT_OPT)); + safe_strcpy(g->Message, sizeof(g->Message), MSG(TABLE_NOT_OPT)); return RC_INFO; // Not to be optimized } // endif block @@ -757,7 +759,7 @@ int TDBDOS::MakeBlockValues(PGLOBAL g) // No optimised columns. Still useful for blocked variable tables. if (!colp && defp->Recfm != RECFM_VAR) { - strcpy(g->Message, "No optimised columns"); + safe_strcpy(g->Message, sizeof(g->Message), "No optimised columns"); return RC_INFO; } // endif colp @@ -787,7 +789,8 @@ int TDBDOS::MakeBlockValues(PGLOBAL g) /*********************************************************************/ char *p = (char *)PlugSubAlloc(g, NULL, 24 + strlen(Name)); - dup->Step = strcat(strcpy(p, MSG(OPTIMIZING)), Name); + snprintf(p, 24 + strlen(Name), "%s%s", MSG(OPTIMIZING), Name); + dup->Step = p; dup->ProgMax = GetProgMax(g); dup->ProgCur = 0; #endif // SOCKET_MODE || THREAD @@ -804,7 +807,7 @@ int TDBDOS::MakeBlockValues(PGLOBAL g) } else { if (++curnum >= nrec) { if (++curblk >= block) { - strcpy(g->Message, MSG(BAD_BLK_ESTIM)); + safe_strcpy(g->Message, sizeof(g->Message), MSG(BAD_BLK_ESTIM)); goto err; } else curnum = 0; @@ -832,7 +835,7 @@ int TDBDOS::MakeBlockValues(PGLOBAL g) #if defined(PROG_INFO) if (!dup->Step) { - strcpy(g->Message, MSG(OPT_CANCELLED)); + safe_strcpy(g->Message, sizeof(g->Message), MSG(OPT_CANCELLED)); goto err; } else dup->ProgCur = GetProgCur(); @@ -912,7 +915,8 @@ bool TDBDOS::SaveBlockValues(PGLOBAL g) if (!(opfile = fopen(filename, "wb"))) { snprintf(g->Message, sizeof(g->Message), MSG(OPEN_MODE_ERROR), "wb", (int)errno, filename); - strcat(strcat(g->Message, ": "), strerror(errno)); + safe_strcat(g->Message, sizeof(g->Message), ": "); + safe_strcat(g->Message, sizeof(g->Message), strerror(errno)); if (trace(1)) htrc("%s\n", g->Message); @@ -1227,7 +1231,8 @@ bool TDBDOS::GetDistinctColumnValues(PGLOBAL g, int nrec) /* Initialize progress information */ /*********************************************************************/ p = (char *)PlugSubAlloc(g, NULL, 48 + strlen(Name)); - dup->Step = strcat(strcpy(p, MSG(GET_DIST_VALS)), Name); + snprintf(p, 48 + strlen(Name), "%s%s", MSG(GET_DIST_VALS), Name); + dup->Step = p; dup->ProgMax = GetProgMax(g); dup->ProgCur = 0; @@ -1239,12 +1244,12 @@ bool TDBDOS::GetDistinctColumnValues(PGLOBAL g, int nrec) #if defined(SOCKET_MODE) if (SendProgress(dup)) { - strcpy(g->Message, MSG(OPT_CANCELLED)); + safe_strcpy(g->Message, sizeof(g->Message), MSG(OPT_CANCELLED)); return true; } else #elif defined(THREAD) if (!dup->Step) { - strcpy(g->Message, MSG(OPT_CANCELLED)); + safe_strcpy(g->Message, sizeof(g->Message), MSG(OPT_CANCELLED)); return true; } else #endif // THREAD @@ -1525,7 +1530,7 @@ PBF TDBDOS::CheckBlockFilari(PGLOBAL g, PXOB *arg, int op, bool *cnv) } else if (n == 8 || n == 14) { if (n == 8 && ctype != TYPE_LIST) { // Should never happen - strcpy(g->Message, "Block opt: bad constant"); + safe_strcpy(g->Message, sizeof(g->Message), "Block opt: bad constant"); throw 99; } // endif Conv @@ -1683,7 +1688,7 @@ int TDBDOS::MakeIndex(PGLOBAL g, PIXDEF pxdf, bool add) // Are we are called from CreateTable or CreateIndex? if (pxdf) { if (!add && dfp->GetIndx()) { - strcpy(g->Message, MSG(INDX_EXIST_YET)); + safe_strcpy(g->Message, sizeof(g->Message), MSG(INDX_EXIST_YET)); return RC_FX; } // endif To_Indx @@ -1795,7 +1800,7 @@ int TDBDOS::MakeIndex(PGLOBAL g, PIXDEF pxdf, bool add) htrc("Exception %d: %s\n", n, g->Message); rc = RC_FX; } catch (const char *msg) { - strcpy(g->Message, msg); + safe_strcpy(g->Message, sizeof(g->Message), msg); rc = RC_FX; } // end catch @@ -1829,7 +1834,7 @@ bool TDBDOS::InitialyzeIndex(PGLOBAL g, volatile PIXDEF xdp, bool sorted) PKPDEF kdp; if (!xdp && !(xdp = To_Xdp)) { - strcpy(g->Message, "NULL dynamic index"); + safe_strcpy(g->Message, sizeof(g->Message), "NULL dynamic index"); return true; } else dynamic = To_Filter && xdp->IsUnique() && xdp->IsDynamic(); @@ -1918,7 +1923,7 @@ bool TDBDOS::InitialyzeIndex(PGLOBAL g, volatile PIXDEF xdp, bool sorted) htrc("Exception %d: %s\n", n, g->Message); brc = true; } catch (const char *msg) { - strcpy(g->Message, msg); + safe_strcpy(g->Message, sizeof(g->Message), msg); brc = true; } // end catch @@ -2679,38 +2684,38 @@ void DOSCOL::WriteColumn(PGLOBAL g) if (Ldz || Nod || Dcm >= 0) { switch (Buf_Type) { case TYPE_SHORT: - strcpy(fmt, (Ldz) ? "%0*hd" : "%*.hd"); + safe_strcpy(fmt, sizeof(fmt), (Ldz) ? "%0*hd" : "%*.hd"); i = 0; if (Nod) for (; i < Dcm; i++) - strcat(fmt, "0"); + safe_strcat(fmt, sizeof(fmt), "0"); len = sprintf(Buf, fmt, field - i, Value->GetShortValue()); break; case TYPE_INT: - strcpy(fmt, (Ldz) ? "%0*d" : "%*.d"); + safe_strcpy(fmt, sizeof(fmt), (Ldz) ? "%0*d" : "%*.d"); i = 0; if (Nod) for (; i < Dcm; i++) - strcat(fmt, "0"); + safe_strcat(fmt,sizeof(fmt), "0"); len = sprintf(Buf, fmt, field - i, Value->GetIntValue()); break; case TYPE_TINY: - strcpy(fmt, (Ldz) ? "%0*d" : "%*.d"); + safe_strcpy(fmt, sizeof(fmt), (Ldz) ? "%0*d" : "%*.d"); i = 0; if (Nod) for (; i < Dcm; i++) - strcat(fmt, "0"); + safe_strcat(fmt, sizeof(fmt), "0"); len = sprintf(Buf, fmt, field - i, Value->GetTinyValue()); break; case TYPE_DOUBLE: case TYPE_DECIM: - strcpy(fmt, (Ldz) ? "%0*.*lf" : "%*.*lf"); + safe_strcpy(fmt, sizeof(fmt), (Ldz) ? "%0*.*lf" : "%*.*lf"); len = field + ((Nod && Dcm) ? 1 : 0); snprintf(Buf, len + 1, fmt, len, Dcm, Value->GetFloatValue()); len = strlen(Buf); diff --git a/storage/connect/tabext.cpp b/storage/connect/tabext.cpp index 01c55cca1cd..9d8a4aca077 100644 --- a/storage/connect/tabext.cpp +++ b/storage/connect/tabext.cpp @@ -65,7 +65,7 @@ int CONDFIL::Init(PGLOBAL g, PHC hc) while (alt) { if (!(p = strchr(alt, '='))) { - strcpy(g->Message, "Invalid alias list"); + safe_strcpy(g->Message, sizeof(g->Message), "Invalid alias list"); rc = RC_FX; break; } // endif !p @@ -126,7 +126,7 @@ EXTDEF::EXTDEF(void) bool EXTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) { if (g->Createas) { - strcpy(g->Message, + safe_strcpy(g->Message, sizeof(g->Message), "Multiple-table UPDATE/DELETE commands are not supported"); return true; } // endif multi @@ -349,7 +349,7 @@ bool TDBEXT::MakeSrcdef(PGLOBAL g) int n_placeholders = count_placeholders(Srcdef); if (n_placeholders < 0) { - strcpy(g->Message, "MakeSQL: Wrong place holders specification"); + safe_strcpy(g->Message, sizeof(g->Message), "MakeSQL: Wrong place holders specification"); return true; } @@ -372,7 +372,7 @@ bool TDBEXT::MakeSrcdef(PGLOBAL g) Query = new(g)STRING(g, strlen(Srcdef) + strlen(fil1) + strlen(fil2)); Query->SetLength(sprintf(Query->GetStr(), Srcdef, fil2, fil1)); } else { - strcpy(g->Message, "MakeSQL: Wrong place holders specification"); + safe_strcpy(g->Message, sizeof(g->Message), "MakeSQL: Wrong place holders specification"); return true; } // endif's ph @@ -513,7 +513,7 @@ bool TDBEXT::MakeSQL(PGLOBAL g, bool cnt) len += ((Mode == MODE_READX) ? 256 : 1); if (Query->IsTruncated()) { - strcpy(g->Message, "MakeSQL: Out of memory"); + safe_strcpy(g->Message, sizeof(g->Message), "MakeSQL: Out of memory"); return true; } else Query->Resize(len); @@ -574,6 +574,7 @@ bool TDBEXT::MakeCommand(PGLOBAL g) bool qtd = Quoted > 0; char q = qtd ? *Quote : ' '; int i = 0, k = 0; + size_t stmt_sz = 0; // Make a lower case copy of the originale query and change // back ticks to the data source identifier quoting character @@ -585,26 +586,30 @@ bool TDBEXT::MakeCommand(PGLOBAL g) p[7] = 0; // Remove where clause Qrystr[(p - qrystr) + 7] = 0; body = To_CondFil->Body; - stmt = (char*)PlugSubAlloc(g, NULL, strlen(qrystr) - + strlen(body) + 64); + stmt_sz = strlen(qrystr) + strlen(body) + 64; } else - stmt = (char*)PlugSubAlloc(g, NULL, strlen(Qrystr) + 64); + stmt_sz = strlen(Qrystr) + 64; + stmt = (char*)PlugSubAlloc(g, NULL, stmt_sz); // Check whether the table name is equal to a keyword // If so, it must be quoted in the original query - strlwr(strcat(strcat(strcpy(name, " "), Name), " ")); + snprintf(name, sizeof(name), " %s ", Name); + strlwr(name); if (strstr(" update delete low_priority ignore quick from ", name)) { if (Quote) { - strlwr(strcat(strcat(strcpy(name, Quote), Name), Quote)); + snprintf(name, sizeof(name), "%s%s%s", Quote, Name, Quote); + strlwr(name); k += 2; } else { - strcpy(g->Message, "Quoted must be specified"); + safe_strcpy(g->Message, sizeof(g->Message), "Quoted must be specified"); return true; } // endif Quote - } else - strlwr(strcpy(name, Name)); // Not a keyword + } else { + safe_strcpy(name, sizeof(name), Name); // Not a keyword + strlwr(name); + } if ((p = strstr(qrystr, name))) { for (i = 0; i < p - qrystr; i++) @@ -618,21 +623,29 @@ bool TDBEXT::MakeCommand(PGLOBAL g) schmp = Schema; if (qtd && *(p - 1) == ' ') { - if (schmp) - strcat(strcat(stmt, schmp), "."); + if (schmp) { + safe_strcat(stmt, stmt_sz, schmp); + safe_strcat(stmt, stmt_sz, "."); + } - strcat(strcat(strcat(stmt, Quote), TableName), Quote); + safe_strcat(stmt, stmt_sz, Quote); + safe_strcat(stmt, stmt_sz, TableName); + safe_strcat(stmt, stmt_sz, Quote); } else { if (schmp) { if (qtd && *(p - 1) != ' ') { stmt[i - 1] = 0; - strcat(strcat(strcat(stmt, schmp), "."), Quote); - } else - strcat(strcat(stmt, schmp), "."); + safe_strcat(stmt, stmt_sz, schmp); + safe_strcat(stmt, stmt_sz, "."); + safe_strcat(stmt, stmt_sz, Quote); + } else { + safe_strcat(stmt, stmt_sz, schmp); + safe_strcat(stmt, stmt_sz, "."); + } } // endif schmp - strcat(stmt, TableName); + safe_strcat(stmt, stmt_sz, TableName); } // endif's i = (int)strlen(stmt); @@ -644,7 +657,7 @@ bool TDBEXT::MakeCommand(PGLOBAL g) RemoveConst(g, stmt); if (body) - strcat(stmt, body); + safe_strcat(stmt, stmt_sz, body); } else { snprintf(g->Message, sizeof(g->Message), "Cannot use this %s command", diff --git a/storage/connect/tabfmt.cpp b/storage/connect/tabfmt.cpp index f20d9afb959..b93b7d8dc8b 100644 --- a/storage/connect/tabfmt.cpp +++ b/storage/connect/tabfmt.cpp @@ -62,6 +62,7 @@ #define NO_FUNC #include "plgcnx.h" // For DB types #include "resource.h" +#include "m_string.h" /***********************************************************************/ /* This should be an option. */ @@ -137,7 +138,7 @@ PQRYRES CSVColumns(PGLOBAL g, PCSZ dp, PTOS topt, bool info) ? strchr(tdp->Entry, '*') || strchr(tdp->Entry, '?') : GetBooleanTableOption(g, topt, "Mulentries", false); #else // !ZIP_SUPPORT - strcpy(g->Message, "ZIP not supported by this version"); + safe_strcpy(g->Message, sizeof(g->Message), "ZIP not supported by this version"); return NULL; #endif // !ZIP_SUPPORT } // endif // Zipped @@ -145,7 +146,7 @@ PQRYRES CSVColumns(PGLOBAL g, PCSZ dp, PTOS topt, bool info) fn = tdp->Fn = GetStringTableOption(g, topt, "Filename", NULL); if (!tdp->Fn) { - strcpy(g->Message, MSG(MISSING_FNAME)); + safe_strcpy(g->Message, sizeof(g->Message), MSG(MISSING_FNAME)); return NULL; } // endif Fn @@ -472,7 +473,7 @@ bool CSVDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff) if (Catfunc == FNC_NO) for (PCOLDEF cdp = To_Cols; cdp; cdp = cdp->GetNext()) if (cdp->GetOffset() < 1 && !cdp->IsSpecial()) { - strcpy(g->Message, MSG(BAD_OFFSET_VAL)); + safe_strcpy(g->Message, sizeof(g->Message), MSG(BAD_OFFSET_VAL)); return true; } // endif Offset @@ -528,11 +529,11 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode) } else if (mode == MODE_INSERT) { txfp = new(g) ZIPFAM(this); } else { - strcpy(g->Message, "UPDATE/DELETE not supported for ZIP"); + safe_strcpy(g->Message, sizeof(g->Message), "UPDATE/DELETE not supported for ZIP"); return NULL; } // endif's mode #else // !ZIP_SUPPORT - strcpy(g->Message, "ZIP not supported"); + safe_strcpy(g->Message, sizeof(g->Message), "ZIP not supported"); return NULL; #endif // !ZIP_SUPPORT } else if (map) { @@ -546,7 +547,7 @@ PTDB CSVDEF::GetTable(PGLOBAL g, MODE mode) txfp = new(g) ZLBFAM(this); #else // !GZ_SUPPORT - strcpy(g->Message, "Compress not supported"); + safe_strcpy(g->Message, sizeof(g->Message), "Compress not supported"); return NULL; #endif // !GZ_SUPPORT } else @@ -879,7 +880,7 @@ bool TDBCSV::SkipHeader(PGLOBAL g) if (q) To_Line[strlen(To_Line)] = Qot; - strcat(To_Line, cdp->GetName()); + safe_strcat(To_Line, Lrecl, cdp->GetName()); if (q) To_Line[strlen(To_Line)] = Qot; @@ -1049,14 +1050,16 @@ bool TDBCSV::PrepareWriting(PGLOBAL g) for (i = 0; i < Fields; i++) { if (i) - strcat(To_Line, sep); + safe_strcat(To_Line, Lrecl, sep); if (Field[i]) { if (!strlen(Field[i])) { // Generally null fields are not quoted - if (Quoted > 2) + if (Quoted > 2) { // Except if explicitly required - strcat(strcat(To_Line, qot), qot); + safe_strcat(To_Line, Lrecl, qot); + safe_strcat(To_Line, Lrecl, qot); + } } else if (Qot && (strchr(Field[i], Sep) || *Field[i] == Qot || Quoted > 1 || (Quoted == 1 && !Fldtyp[i]))) { @@ -1075,12 +1078,15 @@ bool TDBCSV::PrepareWriting(PGLOBAL g) To_Line[k++] = Qot; To_Line[k] = '\0'; - } else - strcat(strcat(strcat(To_Line, qot), Field[i]), qot); + } else { + safe_strcat(To_Line, Lrecl, qot); + safe_strcat(To_Line, Lrecl, Field[i]); + safe_strcat(To_Line, Lrecl, qot); + } } else - strcat(To_Line, Field[i]); + safe_strcat(To_Line, Lrecl, Field[i]); } } // endfor i @@ -1157,7 +1163,7 @@ int TDBCSV::CheckWrite(PGLOBAL g) } // endif } if ((nlen += n) > maxlen) { - strcpy(g->Message, MSG(LINE_TOO_LONG)); + safe_strcpy(g->Message, sizeof(g->Message), MSG(LINE_TOO_LONG)); return -1; } // endif nlen @@ -1267,7 +1273,7 @@ bool TDBFMT::OpenDB(PGLOBAL g) } // endif n FldFormat[i] = (PSZ)PlugSubAlloc(g, NULL, n + 5); - strcpy(FldFormat[i], pfm); + safe_strcpy(FldFormat[i], n + 5, pfm); if (!strcmp(pfm + n, "%m")) { // This is a field that can be missing. Flag it so it can @@ -1277,7 +1283,7 @@ bool TDBFMT::OpenDB(PGLOBAL g) } else if (i+1 < Fields && strcmp(pfm + n, "%n")) { // There are trailing characters after the field contents // add a marker for the next field start position. - strcat(FldFormat[i], "%n"); + safe_strcat(FldFormat[i], n + 5, "%n"); FmtTest[i] = 1; } // endif's diff --git a/storage/connect/tabjdbc.cpp b/storage/connect/tabjdbc.cpp index dc1f5e2fe5b..fe7609a1cac 100644 --- a/storage/connect/tabjdbc.cpp +++ b/storage/connect/tabjdbc.cpp @@ -277,7 +277,7 @@ PTDB JDBCDEF::GetTable(PGLOBAL g, MODE m) if (Multiple == 1) tdbp = new(g)TDBMUL(tdbp); else if (Multiple == 2) - strcpy(g->Message, "NO_JDBC_MUL"); + safe_strcpy(g->Message, sizeof(g->Message), "NO_JDBC_MUL"); } // endswitch Catfunc @@ -386,7 +386,7 @@ bool TDBJDBC::MakeInsert(PGLOBAL g) for (colp = Columns; colp; colp = colp->GetNext()) if (colp->IsSpecial()) { - strcpy(g->Message, "No JDBC special columns"); + safe_strcpy(g->Message, sizeof(g->Message), "No JDBC special columns"); return true; } else { // Column name can be encoded in UTF-8 @@ -460,7 +460,7 @@ bool TDBJDBC::MakeInsert(PGLOBAL g) } // endfor colp if ((Query->Append(") VALUES ("))) { - strcpy(g->Message, "MakeInsert: Out of memory"); + safe_strcpy(g->Message, sizeof(g->Message), "MakeInsert: Out of memory"); return true; } else // in case prepared statement fails pos = Query->GetLength(); @@ -470,7 +470,7 @@ bool TDBJDBC::MakeInsert(PGLOBAL g) Query->Append("?,"); if (Query->IsTruncated()) { - strcpy(g->Message, "MakeInsert: Out of memory"); + safe_strcpy(g->Message, sizeof(g->Message), "MakeInsert: Out of memory"); return true; } else Query->RepLast(')'); @@ -532,12 +532,15 @@ int TDBJDBC::Cardinality(PGLOBAL g) // Table name can be encoded in UTF-8 Decode(TableName, tbn, sizeof(tbn)); - strcpy(qry, "SELECT COUNT(*) FROM "); + safe_strcpy(qry, sizeof(qry), "SELECT COUNT(*) FROM "); - if (Quote) - strcat(strcat(strcat(qry, Quote), tbn), Quote); + if (Quote) { + safe_strcat(qry, sizeof(qry), Quote); + safe_strcat(qry, sizeof(qry), tbn); + safe_strcat(qry, sizeof(qry), Quote); + } else - strcat(qry, tbn); + safe_strcat(qry, sizeof(qry), tbn); // Allocate a Count(*) column (must not use the default constructor) Cnp = new(g)JDBCCOL; @@ -654,7 +657,7 @@ bool TDBJDBC::OpenDB(PGLOBAL g) if ((Qrp = Jcp->AllocateResult(g, this))) Memory = 2; // Must be filled else { - strcpy(g->Message, "Result set memory allocation failed"); + safe_strcpy(g->Message, sizeof(g->Message), "Result set memory allocation failed"); return true; } // endif n @@ -681,7 +684,7 @@ bool TDBJDBC::OpenDB(PGLOBAL g) #if 0 if (!(rc = MakeInsert(g))) { if (Nparm != Jcp->PrepareSQL(Query->GetStr())) { - strcpy(g->Message, MSG(PARM_CNT_MISS)); + safe_strcpy(g->Message, sizeof(g->Message), MSG(PARM_CNT_MISS)); rc = true; } else rc = BindParameters(g); @@ -733,12 +736,12 @@ bool TDBJDBC::SetRecpos(PGLOBAL g, int recpos) CurNum = recpos; Fpos = recpos; } else { - strcpy(g->Message, "Scrolling out of row set NIY"); + safe_strcpy(g->Message, sizeof(g->Message), "Scrolling out of row set NIY"); return true; } // endif recpos } else { - strcpy(g->Message, "This action requires a scrollable cursor"); + safe_strcpy(g->Message, sizeof(g->Message), "This action requires a scrollable cursor"); return true; } // endif's @@ -784,7 +787,7 @@ bool TDBJDBC::ReadKey(PGLOBAL g, OPVAL op, const key_range *kr) if (To_CondFil) if (Query->Append(" AND ") || Query->Append(To_CondFil->Body)) { - strcpy(g->Message, "Readkey: Out of memory"); + safe_strcpy(g->Message, sizeof(g->Message), "Readkey: Out of memory"); return true; } // endif Append @@ -917,7 +920,7 @@ int TDBJDBC::WriteDB(PGLOBAL g) } // endfor colp if (unlikely(Query->IsTruncated())) { - strcpy(g->Message, "WriteDB: Out of memory"); + safe_strcpy(g->Message, sizeof(g->Message), "WriteDB: Out of memory"); return RC_FX; } // endif Query @@ -1110,13 +1113,13 @@ PCMD TDBXJDC::MakeCMD(PGLOBAL g) (To_CondFil->Op == OP_EQ || To_CondFil->Op == OP_IN)) { xcmd = To_CondFil->Cmds; } else - strcpy(g->Message, "Invalid command specification filter"); + safe_strcpy(g->Message, sizeof(g->Message), "Invalid command specification filter"); } else - strcpy(g->Message, "No command column in select list"); + safe_strcpy(g->Message, sizeof(g->Message), "No command column in select list"); } else if (!Srcdef) - strcpy(g->Message, "No Srcdef default command"); + safe_strcpy(g->Message, sizeof(g->Message), "No Srcdef default command"); else xcmd = new(g) CMD(g, Srcdef); @@ -1149,7 +1152,7 @@ bool TDBXJDC::OpenDB(PGLOBAL g) this, Tdb_No, Use, Mode); if (Use == USE_OPEN) { - strcpy(g->Message, "Multiple execution is not allowed"); + safe_strcpy(g->Message, sizeof(g->Message), "Multiple execution is not allowed"); return true; } // endif use @@ -1171,7 +1174,7 @@ bool TDBXJDC::OpenDB(PGLOBAL g) Use = USE_OPEN; // Do it now in case we are recursively called if (Mode != MODE_READ && Mode != MODE_READX) { - strcpy(g->Message, "No INSERT/DELETE/UPDATE of XJDBC tables"); + safe_strcpy(g->Message, sizeof(g->Message), "No INSERT/DELETE/UPDATE of XJDBC tables"); return true; } // endif Mode @@ -1224,7 +1227,7 @@ int TDBXJDC::ReadDB(PGLOBAL g) /***********************************************************************/ int TDBXJDC::WriteDB(PGLOBAL g) { - strcpy(g->Message, "Execsrc tables are read only"); + safe_strcpy(g->Message, sizeof(g->Message), "Execsrc tables are read only"); return RC_FX; } // end of DeleteDB @@ -1233,7 +1236,7 @@ int TDBXJDC::WriteDB(PGLOBAL g) /***********************************************************************/ int TDBXJDC::DeleteDB(PGLOBAL g, int irc) { - strcpy(g->Message, "NO_XJDBC_DELETE"); + safe_strcpy(g->Message, sizeof(g->Message), "NO_XJDBC_DELETE"); return RC_FX; } // end of DeleteDB diff --git a/storage/connect/tabjson.cpp b/storage/connect/tabjson.cpp index 67eb5bbede0..4462541d712 100644 --- a/storage/connect/tabjson.cpp +++ b/storage/connect/tabjson.cpp @@ -85,7 +85,7 @@ PQRYRES JSONColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt, bool info) } // endif info if (GetIntegerTableOption(g, topt, "Multiple", 0)) { - strcpy(g->Message, "Cannot find column definition for multiple table"); + safe_strcpy(g->Message, sizeof(g->Message), "Cannot find column definition for multiple table"); return NULL; } // endif Multiple @@ -212,7 +212,7 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) tdp->Uri = (dsn && *dsn ? dsn : NULL); if (!tdp->Fn && !tdp->Uri) { - strcpy(g->Message, MSG(MISSING_FNAME)); + safe_strcpy(g->Message, sizeof(g->Message), MSG(MISSING_FNAME)); return 0; } else topt->subtype = NULL; @@ -320,7 +320,7 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) switch (tjnp->ReadDB(g)) { case RC_EF: - strcpy(g->Message, "Void json table"); + safe_strcpy(g->Message, sizeof(g->Message), "Void json table"); case RC_FX: goto err; default: @@ -333,7 +333,7 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt) } // endif pretty if (!(row = (jsp) ? jsp->GetObject() : NULL)) { - strcpy(g->Message, "Can only retrieve columns from object rows"); + safe_strcpy(g->Message, sizeof(g->Message), "Can only retrieve columns from object rows"); goto err; } // endif row @@ -417,7 +417,7 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j) if (jvp && jvp->DataType != TYPE_JSON) { if (JsonAllPath() && !fmt[bf]) - strcat(fmt, colname); + safe_strcat(fmt, sizeof(fmt), colname); jcol.Type = jvp->DataType; @@ -450,7 +450,7 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j) jcol.Cbn = true; } else if (j < lvl && !Stringified(strfy, colname)) { if (!fmt[bf]) - strcat(fmt, colname); + safe_strcat(fmt, sizeof(fmt), colname); p = fmt + strlen(fmt); jsp = jvp->GetJson(); @@ -520,11 +520,11 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j) } else if (lvl >= 0) { if (Stringified(strfy, colname)) { if (!fmt[bf]) - strcat(fmt, colname); + safe_strcat(fmt, sizeof(fmt), colname); - strcat(fmt, ".*"); + safe_strcat(fmt, sizeof(fmt), ".*"); } else if (JsonAllPath() && !fmt[bf]) - strcat(fmt, colname); + safe_strcat(fmt, sizeof(fmt), colname); jcol.Type = TYPE_STRG; jcol.Len = sz; @@ -735,7 +735,7 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) } else if (m == MODE_INSERT) { txfp = new(g) ZIPFAM(this); } else { - strcpy(g->Message, "UPDATE/DELETE not supported for ZIP"); + safe_strcpy(g->Message, sizeof(g->Message), "UPDATE/DELETE not supported for ZIP"); return NULL; } // endif's m #else // !ZIP_SUPPORT @@ -775,7 +775,7 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) #endif // 0 ((TDBJSN*)tdbp)->G = PlugInit(NULL, (size_t)Lrecl * (Pretty >= 0 ? 12 : 4)); } else { - strcpy(g->Message, "LRECL is not defined"); + safe_strcpy(g->Message, sizeof(g->Message), "LRECL is not defined"); return NULL; } // endif Lrecl @@ -785,10 +785,10 @@ PTDB JSONDEF::GetTable(PGLOBAL g, MODE m) if (m == MODE_READ || m == MODE_ANY || m == MODE_ALTER) { txfp = new(g) UNZFAM(this); } else if (m == MODE_INSERT) { - strcpy(g->Message, "INSERT supported only for zipped JSON when pretty=0"); + safe_strcpy(g->Message, sizeof(g->Message), "INSERT supported only for zipped JSON when pretty=0"); return NULL; } else { - strcpy(g->Message, "UPDATE/DELETE not supported for ZIP"); + safe_strcpy(g->Message, sizeof(g->Message), "UPDATE/DELETE not supported for ZIP"); return NULL; } // endif's m #else // !ZIP_SUPPORT @@ -1145,7 +1145,7 @@ int TDBJSN::ReadDB(PGLOBAL g) { M = 1; rc = RC_OK; } else if (Pretty != 1 || strcmp(To_Line, "]")) { - strcpy(g->Message, G->Message); + safe_strcpy(g->Message, sizeof(g->Message), G->Message); rc = RC_FX; } else rc = RC_EF; @@ -1258,7 +1258,7 @@ bool TDBJSN::PrepareWriting(PGLOBAL g) strcat(s, ","); if ((signed)strlen(s) > Lrecl) { - strncpy(To_Line, s, Lrecl); + safe_strcpy(To_Line, Lrecl, s); snprintf(g->Message, sizeof(g->Message), "Line truncated (lrecl=%d)", Lrecl); return PushWarning(g, this); } else @@ -1360,7 +1360,7 @@ bool JSONCOL::CheckExpand(PGLOBAL g, int i, PSZ nm, bool b) Xpd = true; // Expandable object Nodes[i].Op = OP_EXP; } else if (b) { - strcpy(g->Message, "Cannot expand more than one branch"); + safe_strcpy(g->Message, sizeof(g->Message), "Cannot expand more than one branch"); return true; } // endif Xcol @@ -1571,7 +1571,7 @@ bool JSONCOL::ParseJpath(PGLOBAL g) if (SetArrayOptions(g, p, i, Nodes[i - 1].Key)) return true; else if (Xpd && Tjp->Mode == MODE_DELETE) { - strcpy(g->Message, "Cannot delete expanded columns"); + safe_strcpy(g->Message, sizeof(g->Message), "Cannot delete expanded columns"); return true; } // endif Xpd @@ -1675,7 +1675,7 @@ PSZ JSONCOL::GetJpath(PGLOBAL g, bool proj) PVAL JSONCOL::MakeJson(PGLOBAL g, PJSON jsp, int n) { if (Value->IsTypeNum()) { - strcpy(g->Message, "Cannot make Json for a numeric column"); + safe_strcpy(g->Message, sizeof(g->Message), "Cannot make Json for a numeric column"); if (!Warned) { PushWarning(g, Tjp); @@ -1690,10 +1690,10 @@ PVAL JSONCOL::MakeJson(PGLOBAL g, PJSON jsp, int n) ulong len = Tjp->Lrecl ? Tjp->Lrecl : 500; PBSON bsp = JbinAlloc(g, NULL, len, jsp); - strcat(bsp->Msg, " column"); + safe_strcat(bsp->Msg, sizeof(bsp->Msg), " column"); ((BINVAL*)Value)->SetBinValue(bsp, sizeof(BSON)); } else { - strcpy(g->Message, "Column size too small"); + safe_strcpy(g->Message, sizeof(g->Message), "Column size too small"); Value->SetValue_char(NULL, 0); } // endif Clen #endif // 0 @@ -1937,7 +1937,7 @@ PVAL JSONCOL::ExpandArray(PGLOBAL g, PJAR arp, int n) } // endif ars if (!(jvp = arp->GetArrayValue((Nodes[n].Rx = Nodes[n].Nx)))) { - strcpy(g->Message, "Logical error expanding array"); + safe_strcpy(g->Message, sizeof(g->Message), "Logical error expanding array"); throw 666; } // endif jvp @@ -2125,7 +2125,7 @@ PJSON JSONCOL::GetRow(PGLOBAL g) ((PJAR)row)->AddArrayValue(G, new(G) JVALUE(nwr)); ((PJAR)row)->InitArray(G); } else { - strcpy(g->Message, "Wrong type when writing new row"); + safe_strcpy(g->Message, sizeof(g->Message), "Wrong type when writing new row"); nwr = NULL; } // endif's @@ -2146,7 +2146,7 @@ PJSON JSONCOL::GetRow(PGLOBAL g) void JSONCOL::WriteColumn(PGLOBAL g) { if (Xpd && Tjp->Pretty < 2) { - strcpy(g->Message, "Cannot write expanded column when Pretty is not 2"); + safe_strcpy(g->Message, sizeof(g->Message), "Cannot write expanded column when Pretty is not 2"); throw 666; } // endif Xpd @@ -2182,7 +2182,7 @@ void JSONCOL::WriteColumn(PGLOBAL g) if (s && *s) { if (!(jsp = ParseJson(G, s, strlen(s)))) { - strcpy(g->Message, s); + safe_strcpy(g->Message, sizeof(g->Message), s); throw 666; } // endif jsp @@ -2365,7 +2365,7 @@ int TDBJSON::MakeDocument(PGLOBAL g) if (!a && *p && *p != '[' && !IsNum(p)) { // obj is a key if (jsp->GetType() != TYPE_JOB) { - strcpy(g->Message, "Table path does not match the json file"); + safe_strcpy(g->Message, sizeof(g->Message), "Table path does not match the json file"); return RC_FX; } // endif Type @@ -2391,7 +2391,7 @@ int TDBJSON::MakeDocument(PGLOBAL g) } // endif p if (jsp->GetType() != TYPE_JAR) { - strcpy(g->Message, "Table path does not match the json file"); + safe_strcpy(g->Message, sizeof(g->Message), "Table path does not match the json file"); return RC_FX; } // endif Type @@ -2486,7 +2486,7 @@ void TDBJSON::ResetSize(void) int TDBJSON::MakeIndex(PGLOBAL g, PIXDEF pxdf, bool) { if (pxdf) { - strcpy(g->Message, "JSON not indexable when pretty = 2"); + safe_strcpy(g->Message, sizeof(g->Message), "JSON not indexable when pretty = 2"); return RC_FX; } else return RC_OK; -- cgit v1.2.1 From c6e58a8d173b7e9689952d07678ba79702ba8021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 21 Apr 2023 11:05:44 +0300 Subject: MDEV-30753 fixup: Unsafe buffer page restoration trx_purge_free_segment(): The buffer-fix only prevents a block from being freed completely from the buffer pool, but it will not prevent the block from being evicted. Recheck the page identifier after acquiring an exclusive page latch. If it has changed, backtrack and invoke buf_page_get_gen() to look up the page normally. --- storage/innobase/trx/trx0purge.cc | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 38438108480..b22a85f4646 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -367,19 +367,30 @@ void trx_purge_free_segment(mtr_t &mtr, trx_rseg_t* rseg, fil_addr_t hdr_addr) block->frame, &mtr)) { block->fix(); + const page_id_t id{block->page.id()}; mtr.commit(); /* NOTE: If the server is killed after the log that was produced up to this point was written, and before the log from the mtr.commit() in our caller is written, then the pages belonging to the undo log will become unaccessible garbage. - This does not matters when using multiple innodb_undo_tablespaces; + This does not matter when using multiple innodb_undo_tablespaces; innodb_undo_log_truncate=ON will be able to reclaim the space. */ log_free_check(); mtr.start(); ut_ad(rw_lock_s_lock_nowait(block->debug_latch, __FILE__, __LINE__)); rw_lock_x_lock(&block->lock); - mtr_memo_push(&mtr, block, MTR_MEMO_PAGE_X_FIX); + if (UNIV_UNLIKELY(block->page.id() != id)) + { + block->unfix(); + rw_lock_x_unlock(&block->lock); + ut_d(rw_lock_s_unlock(block->debug_latch)); + block= buf_page_get(id, 0, RW_X_LATCH, &mtr); + if (!block) + return; + } + else + mtr_memo_push(&mtr, block, MTR_MEMO_PAGE_X_FIX); } while (!fseg_free_step(TRX_UNDO_SEG_HDR + TRX_UNDO_FSEG_HEADER + -- cgit v1.2.1 From e55e761eae707e47e86af1899335c777e194fdd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 21 Apr 2023 16:49:59 +0300 Subject: MDEV-31084 assert(waiting) failed in TP_connection_generic::wait_end buf_flush_wait_flushed(): Correct the logic for registering a wait around buf_flush_wait() that commit a091d6ac4e7d2d7873749e685943b3032ccfda57 recently broke. This should be easily repeatable when using a non-default startup parameter: thread-handling=pool-of-threads --- storage/innobase/buf/buf0flu.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 3ef70741da1..75286d9d33b 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1905,9 +1905,13 @@ ATTRIBUTE_COLD void buf_flush_wait_flushed(lsn_t sync_lsn) } else #endif + { + thd_wait_begin(nullptr, THD_WAIT_DISKIO); + tpool::tpool_wait_begin(); buf_flush_wait(sync_lsn); - - thd_wait_end(nullptr); + tpool::tpool_wait_end(); + thd_wait_end(nullptr); + } } mysql_mutex_unlock(&buf_pool.flush_list_mutex); -- cgit v1.2.1 From 46af63bfe202b62a5665e1aa09d069b5b5613249 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 10 Apr 2023 11:35:38 +0400 Subject: MDEV-31018 Replica of 10.3, 10.4, <10.5.19 and <10.6.12 to 10.11 will not work when using non-default charset MDEV-28769 earlier disabled the use if IDs with non-default collations in statements like: SET character_set_results=2/*latin2_czech_cs*/; SET character_set_client=2/*latin2_czech_cs*/; SET character_set_server=2/*latin2_czech_cs*/; SET character_set_connection=2/*latin2_czech_cs*/; MDEV-30824 later fixed "mysqlbinlog" to dump character set names instead of IDs in these statements: < SET @@session.character_set_client=33, ... /*!*/; > SET @@session.character_set_client=utf8mb3, ... /*!*/; However, mysqlbinlog from old (pre MDEV-30824) distributions can still produce incorrect statements with numeric non-default collation IDs. New servers should still be able to load old dumps. Allowing the use of "SET @@character_set_xxx=ID" with numeric non-default collation IDs but only if: - the current THD is a true slave thread or - the current THD a pseudo slave thread (loading a mysqlbinlog output). --- .../sys_vars/r/character_set_client_basic.result | 11 +++++++++++ .../r/character_set_connection_basic.result | 16 +++++++++++++++ .../sys_vars/r/character_set_results_basic.result | Bin 16427 -> 16895 bytes .../sys_vars/r/character_set_server_basic.result | 16 +++++++++++++++ .../sys_vars/t/character_set_client_basic.test | 9 +++++++++ .../sys_vars/t/character_set_connection_basic.test | 13 ++++++++++++ .../sys_vars/t/character_set_results_basic.test | 12 +++++++++++ .../sys_vars/t/character_set_server_basic.test | 12 +++++++++++ sql/sys_vars.cc | 22 +++++++++++++++++---- 9 files changed, 107 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/sys_vars/r/character_set_client_basic.result b/mysql-test/suite/sys_vars/r/character_set_client_basic.result index d62d88027c7..14c8bcfd058 100644 --- a/mysql-test/suite/sys_vars/r/character_set_client_basic.result +++ b/mysql-test/suite/sys_vars/r/character_set_client_basic.result @@ -500,5 +500,16 @@ res # SET GLOBAL character_set_client=2; ERROR 42000: Unknown character set: '2' +# +# MDEV-31018 Replica of 10.3, 10.4, <10.5.19 and <10.6.12 to 10.11 will not work when using non-default charset +# +SET @@pseudo_slave_mode=1; +SET character_set_client=2/*latin2_czech_cs*/; +SHOW VARIABLES LIKE 'character_set_client'; +Variable_name Value +character_set_client latin2 +SET @@pseudo_slave_mode=0; +Warnings: +Warning 1231 Slave applier execution mode not active, statement ineffective. SET @@global.character_set_client = @global_start_value; SET @@session.character_set_client = @session_start_value; diff --git a/mysql-test/suite/sys_vars/r/character_set_connection_basic.result b/mysql-test/suite/sys_vars/r/character_set_connection_basic.result index e356d62ed2b..bbb4f4e5f75 100644 --- a/mysql-test/suite/sys_vars/r/character_set_connection_basic.result +++ b/mysql-test/suite/sys_vars/r/character_set_connection_basic.result @@ -494,5 +494,21 @@ SELECT @@session.character_set_connection = WHERE VARIABLE_NAME='character_set_connection') AS res; res 1 +# +# MDEV-31018 Replica of 10.3, 10.4, <10.5.19 and <10.6.12 to 10.11 will not work when using non-default charset +# +SET character_set_connection=2/*latin2_czech_cs*/; +ERROR 42000: Unknown character set: '2' +SET @@pseudo_slave_mode=1; +SET character_set_connection=2/*latin2_czech_cs*/; +SHOW VARIABLES LIKE 'character_set_connection'; +Variable_name Value +character_set_connection latin2 +SHOW VARIABLES LIKE 'collation_connection'; +Variable_name Value +collation_connection latin2_general_ci +SET @@pseudo_slave_mode=0; +Warnings: +Warning 1231 Slave applier execution mode not active, statement ineffective. SET @@global.character_set_connection = @global_start_value; SET @@global.character_set_client = @save_character_set_client; diff --git a/mysql-test/suite/sys_vars/r/character_set_results_basic.result b/mysql-test/suite/sys_vars/r/character_set_results_basic.result index d1c6a52ba17..1346c8b42ca 100644 Binary files a/mysql-test/suite/sys_vars/r/character_set_results_basic.result and b/mysql-test/suite/sys_vars/r/character_set_results_basic.result differ diff --git a/mysql-test/suite/sys_vars/r/character_set_server_basic.result b/mysql-test/suite/sys_vars/r/character_set_server_basic.result index e0a13c729fe..7445be6aef1 100644 --- a/mysql-test/suite/sys_vars/r/character_set_server_basic.result +++ b/mysql-test/suite/sys_vars/r/character_set_server_basic.result @@ -486,5 +486,21 @@ SELECT @@session.character_set_server = WHERE VARIABLE_NAME='character_set_server') AS res; res 1 +# +# MDEV-31018 Replica of 10.3, 10.4, <10.5.19 and <10.6.12 to 10.11 will not work when using non-default charset +# +SET character_set_server=2/*latin2_czech_cs*/; +ERROR 42000: Unknown character set: '2' +SET @@pseudo_slave_mode=1; +SET character_set_server=2/*latin2_czech_cs*/; +SHOW VARIABLES LIKE 'character_set_server'; +Variable_name Value +character_set_server latin2 +SHOW VARIABLES LIKE 'collation_server'; +Variable_name Value +collation_server latin2_general_ci +SET @@pseudo_slave_mode=0; +Warnings: +Warning 1231 Slave applier execution mode not active, statement ineffective. SET @@global.character_set_server = @global_start_value; SET @@session.character_set_server = @session_start_value; diff --git a/mysql-test/suite/sys_vars/t/character_set_client_basic.test b/mysql-test/suite/sys_vars/t/character_set_client_basic.test index 09f758a0316..44c0d240a9d 100644 --- a/mysql-test/suite/sys_vars/t/character_set_client_basic.test +++ b/mysql-test/suite/sys_vars/t/character_set_client_basic.test @@ -350,6 +350,15 @@ SELECT @@session.character_set_client = --error ER_UNKNOWN_CHARACTER_SET SET GLOBAL character_set_client=2; +--echo # +--echo # MDEV-31018 Replica of 10.3, 10.4, <10.5.19 and <10.6.12 to 10.11 will not work when using non-default charset +--echo # + +SET @@pseudo_slave_mode=1; +SET character_set_client=2/*latin2_czech_cs*/; +SHOW VARIABLES LIKE 'character_set_client'; +SET @@pseudo_slave_mode=0; + #################################### # Restore initial value # #################################### diff --git a/mysql-test/suite/sys_vars/t/character_set_connection_basic.test b/mysql-test/suite/sys_vars/t/character_set_connection_basic.test index 3d9094fca04..46bdfcb52e2 100644 --- a/mysql-test/suite/sys_vars/t/character_set_connection_basic.test +++ b/mysql-test/suite/sys_vars/t/character_set_connection_basic.test @@ -275,6 +275,19 @@ SELECT @@session.character_set_connection = (SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.SESSION_VARIABLES WHERE VARIABLE_NAME='character_set_connection') AS res; +--echo # +--echo # MDEV-31018 Replica of 10.3, 10.4, <10.5.19 and <10.6.12 to 10.11 will not work when using non-default charset +--echo # + +--error ER_UNKNOWN_CHARACTER_SET +SET character_set_connection=2/*latin2_czech_cs*/; +SET @@pseudo_slave_mode=1; +SET character_set_connection=2/*latin2_czech_cs*/; +SHOW VARIABLES LIKE 'character_set_connection'; +SHOW VARIABLES LIKE 'collation_connection'; +SET @@pseudo_slave_mode=0; + + #################################### # Restore initial value # #################################### diff --git a/mysql-test/suite/sys_vars/t/character_set_results_basic.test b/mysql-test/suite/sys_vars/t/character_set_results_basic.test index 617332b9780..41090be6657 100644 --- a/mysql-test/suite/sys_vars/t/character_set_results_basic.test +++ b/mysql-test/suite/sys_vars/t/character_set_results_basic.test @@ -272,6 +272,18 @@ SELECT @@session.character_set_results = (SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.SESSION_VARIABLES WHERE VARIABLE_NAME='character_set_results') AS res; +--echo # +--echo # MDEV-31018 Replica of 10.3, 10.4, <10.5.19 and <10.6.12 to 10.11 will not work when using non-default charset +--echo # + +--error ER_UNKNOWN_CHARACTER_SET +SET character_set_results=2/*latin2_czech_cs*/; +SET @@pseudo_slave_mode=1; +SET character_set_results=2; +SHOW VARIABLES LIKE 'character_set_results'; +SET @@pseudo_slave_mode=0; + + #################################### # Restore initial value # #################################### diff --git a/mysql-test/suite/sys_vars/t/character_set_server_basic.test b/mysql-test/suite/sys_vars/t/character_set_server_basic.test index b2f4788fdae..929dbfaf143 100644 --- a/mysql-test/suite/sys_vars/t/character_set_server_basic.test +++ b/mysql-test/suite/sys_vars/t/character_set_server_basic.test @@ -266,6 +266,18 @@ SELECT @@session.character_set_server = (SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.SESSION_VARIABLES WHERE VARIABLE_NAME='character_set_server') AS res; +--echo # +--echo # MDEV-31018 Replica of 10.3, 10.4, <10.5.19 and <10.6.12 to 10.11 will not work when using non-default charset +--echo # + +--error ER_UNKNOWN_CHARACTER_SET +SET character_set_server=2/*latin2_czech_cs*/; +SET @@pseudo_slave_mode=1; +SET character_set_server=2/*latin2_czech_cs*/; +SHOW VARIABLES LIKE 'character_set_server'; +SHOW VARIABLES LIKE 'collation_server'; +SET @@pseudo_slave_mode=0; + #################################### # Restore initial value # #################################### diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 10f9de106f8..45b5fe20ff6 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -791,12 +791,26 @@ static bool check_charset(sys_var *self, THD *thd, set_var *var) { int csno= (int)var->value->val_int(); CHARSET_INFO *cs; - if (!(var->save_result.ptr= cs= get_charset(csno, MYF(0))) || - !(cs->state & MY_CS_PRIMARY)) + if ((var->save_result.ptr= cs= get_charset(csno, MYF(0)))) { - my_error(ER_UNKNOWN_CHARACTER_SET, MYF(0), llstr(csno, buff)); - return true; + /* + Backward compatibility: pre MDEV-30824 servers + can write non-default collation IDs to binary log: + SET character_set_client=83; -- utf8mb3_bin + Convert a non-default collation to the compiled default collation, + e.g. utf8mb3_bin to utf8mb3_general_ci, but only if + - THD is a slave thread or + - is processing a mysqlbinlog output. + */ + if ((cs->state & MY_CS_PRIMARY) || + ((thd->variables.pseudo_slave_mode || thd->slave_thread) && + (var->save_result.ptr= + Lex_exact_charset_opt_extended_collate(cs, true). + find_default_collation()))) + return false; } + my_error(ER_UNKNOWN_CHARACTER_SET, MYF(0), llstr(csno, buff)); + return true; } return false; } -- cgit v1.2.1 From da1c91fb9232fbea88d3f9a27f81b39f85cfc468 Mon Sep 17 00:00:00 2001 From: Daniel Black Date: Tue, 28 Feb 2023 10:43:39 +1100 Subject: MDEV-30713 field length handling for CONNECT engine fp->field_length was unsigned and therefore the negative condition around it. Backport of cc182aca9352 fixes it, however to correct the consistent use of types pcf->Length needs to be unsigned too. At one point pcf->Precision is assigned from pcf->Length so that's also unsigned. GetTypeSize is assigned to length and has a length argument. A -1 default value seemed dangerious to case, so at least 0 should assert if every hit. --- storage/connect/catalog.h | 4 ++-- storage/connect/ha_connect.cc | 5 +---- storage/connect/tabext.cpp | 2 +- storage/connect/value.cpp | 8 ++++---- storage/connect/value.h | 2 +- 5 files changed, 9 insertions(+), 12 deletions(-) diff --git a/storage/connect/catalog.h b/storage/connect/catalog.h index 2649a50cf76..a46615f5d6e 100644 --- a/storage/connect/catalog.h +++ b/storage/connect/catalog.h @@ -39,9 +39,9 @@ typedef struct _colinfo { PCSZ Name; int Type; int Offset; - int Length; + unsigned Length; int Key; - int Precision; + unsigned Precision; int Scale; int Opt; int Freq; diff --git a/storage/connect/ha_connect.cc b/storage/connect/ha_connect.cc index 81960332cbe..0a0a206c9c8 100644 --- a/storage/connect/ha_connect.cc +++ b/storage/connect/ha_connect.cc @@ -1618,10 +1618,7 @@ void *ha_connect::GetColumnOption(PGLOBAL g, void *field, PCOLINFO pcf) pcf->Scale= 0; pcf->Opt= (fop) ? (int)fop->opt : 0; - if (fp->field_length >= 0) - pcf->Length= fp->field_length; - else - pcf->Length= 256; // BLOB? + pcf->Length= fp->field_length; pcf->Precision= pcf->Length; diff --git a/storage/connect/tabext.cpp b/storage/connect/tabext.cpp index 9d8a4aca077..6903e112238 100644 --- a/storage/connect/tabext.cpp +++ b/storage/connect/tabext.cpp @@ -466,7 +466,7 @@ bool TDBEXT::MakeSQL(PGLOBAL g, bool cnt) if (Quote) { // Tabname can have both database and table identifiers, we need to parse - if (res= strstr(buf, ".")) + if ((res= strstr(buf, "."))) { // Parse schema my_len= res - buf + 1; diff --git a/storage/connect/value.cpp b/storage/connect/value.cpp index 498ec71a87f..7265b2ed0ca 100644 --- a/storage/connect/value.cpp +++ b/storage/connect/value.cpp @@ -163,9 +163,9 @@ PCSZ GetTypeName(int type) /***********************************************************************/ /* GetTypeSize: returns the PlugDB internal type size. */ /***********************************************************************/ -int GetTypeSize(int type, int len) - { - switch (type) { +unsigned GetTypeSize(int type, unsigned len) +{ + switch (type) { case TYPE_DECIM: case TYPE_BIN: case TYPE_STRING: len = len * sizeof(char); break; @@ -176,7 +176,7 @@ int GetTypeSize(int type, int len) case TYPE_DOUBLE: len = sizeof(double); break; case TYPE_TINY: len = sizeof(char); break; case TYPE_PCHAR: len = sizeof(char*); break; - default: len = -1; + default: len = 0; } // endswitch type return len; diff --git a/storage/connect/value.h b/storage/connect/value.h index a0d947347c3..7eb0dec29f2 100644 --- a/storage/connect/value.h +++ b/storage/connect/value.h @@ -41,7 +41,7 @@ typedef struct _datpar *PDTP; // For DTVAL /***********************************************************************/ // Exported functions DllExport PCSZ GetTypeName(int); -DllExport int GetTypeSize(int, int); +DllExport unsigned GetTypeSize(int, unsigned); #ifdef ODBC_SUPPORT /* This function is exported for use in OEM table type DLLs */ DllExport int TranslateSQLType(int stp, int prec, -- cgit v1.2.1 From 40eff3f868108a202d88e4da339fc539cbf29e85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 21 Apr 2023 17:52:47 +0300 Subject: MDEV-26827 fixup: hangs and !os_aio_pending_writes() assertion failures buf_LRU_get_free_block(): Always wake up the page cleaner if needed before exiting the inner loop. srv_prepare_to_delete_redo_log_file(): Replace a debug assertion with a wait in debug builds. Starting with commit 7e31a8e7fa97a87fc164381588d172bf0e76abb6 the debug assertion ut_ad(!os_aio_pending_writes()) could occasionally fail, while it would hold in core dumps of crashes. The failure can be reproduced more easily by adding a sleep to the write completion callback function, right before releasing to write_slots. srv_start(): Remove a bogus debug assertion ut_ad(!os_aio_pending_writes()) that could fail in mariadb-backup --prepare. In an rr replay trace, we had buf_pool.flush_list.count==0 but write_slots->m_cache.m_pos==1 and buf_page_t::write_complete() was executing u_unlock(). --- storage/innobase/buf/buf0lru.cc | 6 +++--- storage/innobase/srv/srv0start.cc | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 844e288843b..feb15fc226c 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -448,15 +448,15 @@ got_block: mysql_mutex_unlock(&buf_pool.mutex); mysql_mutex_lock(&buf_pool.flush_list_mutex); const auto n_flush = buf_pool.n_flush(); + if (!buf_pool.try_LRU_scan) { + buf_pool.page_cleaner_wakeup(true); + } mysql_mutex_unlock(&buf_pool.flush_list_mutex); mysql_mutex_lock(&buf_pool.mutex); if (!n_flush) { goto not_found; } if (!buf_pool.try_LRU_scan) { - mysql_mutex_lock(&buf_pool.flush_list_mutex); - buf_pool.page_cleaner_wakeup(true); - mysql_mutex_unlock(&buf_pool.flush_list_mutex); my_cond_wait(&buf_pool.done_free, &buf_pool.mutex.m_mutex); } diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 707804f2206..5f6b4b02e16 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -970,10 +970,10 @@ same_size: ut_ad(flushed_lsn == log_sys.get_lsn()); ut_ad(!os_aio_pending_reads()); - ut_ad(!os_aio_pending_writes()); ut_d(mysql_mutex_lock(&buf_pool.flush_list_mutex)); ut_ad(!buf_pool.get_oldest_modification(0)); ut_d(mysql_mutex_unlock(&buf_pool.flush_list_mutex)); + ut_d(os_aio_wait_until_no_pending_writes()); DBUG_RETURN(flushed_lsn); } @@ -1608,7 +1608,6 @@ file_checked: ut_ad(recv_no_log_write); err = fil_write_flushed_lsn(log_sys.get_lsn()); ut_ad(!os_aio_pending_reads()); - ut_ad(!os_aio_pending_writes()); ut_d(mysql_mutex_lock(&buf_pool.flush_list_mutex)); ut_ad(!buf_pool.get_oldest_modification(0)); ut_d(mysql_mutex_unlock(&buf_pool.flush_list_mutex)); -- cgit v1.2.1 From 86767bcc0f121db3ad83a74647a642754a0ce57f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 21 Apr 2023 17:58:09 +0300 Subject: MDEV-29593 Purge misses a chance to free not-yet-reused undo pages trx_purge_truncate_rseg_history(): If all other conditions for invoking trx_purge_remove_log_hdr() hold, but the state is TRX_UNDO_CACHED instead of TRX_UNDO_TO_PURGE, detach and free it. Tested by: Matthias Leich --- storage/innobase/include/mtr0mtr.h | 2 +- storage/innobase/trx/trx0purge.cc | 231 ++++++++++++++++++------------------- 2 files changed, 116 insertions(+), 117 deletions(-) diff --git a/storage/innobase/include/mtr0mtr.h b/storage/innobase/include/mtr0mtr.h index 1c044319ca0..5ce297ba5c9 100644 --- a/storage/innobase/include/mtr0mtr.h +++ b/storage/innobase/include/mtr0mtr.h @@ -343,7 +343,7 @@ public: { mtr_memo_slot_t &slot= m_memo[savepoint]; ut_ad(slot.type <= MTR_MEMO_BUF_FIX); - ut_ad(type <= MTR_MEMO_BUF_FIX); + ut_ad(type < MTR_MEMO_S_LOCK); slot.type= type; } diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index ded4d63d705..4a2fa214b37 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -275,7 +275,7 @@ trx_purge_add_undo_to_history(const trx_t* trx, trx_undo_t*& undo, mtr_t* mtr) if (undo->state != TRX_UNDO_CACHED) { /* The undo log segment will not be reused */ ut_a(undo->id < TRX_RSEG_N_SLOTS); - compile_time_assert(FIL_NULL == 0xffffffff); + static_assert(FIL_NULL == 0xffffffff, ""); mtr->memset(rseg_header, TRX_RSEG + TRX_RSEG_UNDO_SLOTS + undo->id * TRX_RSEG_SLOT_SIZE, 4, 0xff); @@ -385,45 +385,11 @@ static dberr_t trx_purge_remove_log_hdr(buf_block_t *rseg, buf_block_t* log, uint16_t(offset + TRX_UNDO_HISTORY_NODE), mtr); } -MY_ATTRIBUTE((nonnull, warn_unused_result)) -/** Free an undo log segment, and remove the header from the history list. -@param[in,out] mtr mini-transaction -@param[in,out] rseg rollback segment -@param[in] hdr_addr file address of log_hdr -@return error code */ -static dberr_t -trx_purge_free_segment(mtr_t &mtr, trx_rseg_t* rseg, fil_addr_t hdr_addr) +/** Free an undo log segment. +@param block rollback segment header page +@param mtr mini-transaction */ +static void trx_purge_free_segment(buf_block_t *block, mtr_t &mtr) { - mtr.commit(); - log_free_check(); - mtr.start(); - - const page_id_t hdr_page_id{rseg->space->id, hdr_addr.page}; - dberr_t err; - buf_block_t *rseg_hdr= rseg->get(&mtr, &err); - if (!rseg_hdr) - return err; - buf_block_t *block= buf_page_get_gen(hdr_page_id, 0, RW_X_LATCH, - nullptr, BUF_GET_POSSIBLY_FREED, - &mtr, &err); - if (!block) - return err; - - const uint32_t seg_size= - flst_get_len(TRX_UNDO_SEG_HDR + TRX_UNDO_PAGE_LIST + block->page.frame); - - err= trx_purge_remove_log_hdr(rseg_hdr, block, hdr_addr.boffset, &mtr); - if (UNIV_UNLIKELY(err != DB_SUCCESS)) - return err; - - ut_ad(rseg->curr_size >= seg_size); - rseg->curr_size-= seg_size; - rseg->history_size--; - - byte *hist= TRX_RSEG + TRX_RSEG_HISTORY_SIZE + rseg_hdr->page.frame; - ut_ad(mach_read_from_4(hist) >= seg_size); - mtr.write<4>(*rseg_hdr, hist, mach_read_from_4(hist) - seg_size); - while (!fseg_free_step_not_header(TRX_UNDO_SEG_HDR + TRX_UNDO_FSEG_HEADER + block->page.frame, &mtr)) { @@ -444,9 +410,9 @@ trx_purge_free_segment(mtr_t &mtr, trx_rseg_t* rseg, fil_addr_t hdr_addr) { block->unfix(); block->page.lock.x_unlock(); - block= buf_page_get_gen(id, 0, RW_X_LATCH, nullptr, BUF_GET, &mtr, &err); + block= buf_page_get_gen(id, 0, RW_X_LATCH, nullptr, BUF_GET, &mtr); if (!block) - return err; + return; } else mtr.memo_push(block, MTR_MEMO_PAGE_X_MODIFY); @@ -454,102 +420,135 @@ trx_purge_free_segment(mtr_t &mtr, trx_rseg_t* rseg, fil_addr_t hdr_addr) while (!fseg_free_step(TRX_UNDO_SEG_HDR + TRX_UNDO_FSEG_HEADER + block->page.frame, &mtr)); - return DB_SUCCESS; } /** Remove unnecessary history data from a rollback segment. @param[in,out] rseg rollback segment @param[in] limit truncate anything before this @return error code */ -static -dberr_t -trx_purge_truncate_rseg_history( - trx_rseg_t& rseg, - const purge_sys_t::iterator& limit) +static dberr_t +trx_purge_truncate_rseg_history(trx_rseg_t& rseg, + const purge_sys_t::iterator& limit) { - fil_addr_t hdr_addr; - mtr_t mtr; + fil_addr_t hdr_addr; + mtr_t mtr; - mtr.start(); + log_free_check(); + mtr.start(); - dberr_t err; - buf_block_t* rseg_hdr = rseg.get(&mtr, &err); - if (!rseg_hdr) { - goto func_exit; - } + dberr_t err; +reget: + buf_block_t *rseg_hdr= rseg.get(&mtr, &err); + if (!rseg_hdr) + { +func_exit: + mtr.commit(); + return err; + } - hdr_addr = flst_get_last(TRX_RSEG + TRX_RSEG_HISTORY - + rseg_hdr->page.frame); - hdr_addr.boffset = static_cast(hdr_addr.boffset - - TRX_UNDO_HISTORY_NODE); + hdr_addr= flst_get_last(TRX_RSEG + TRX_RSEG_HISTORY + rseg_hdr->page.frame); + hdr_addr.boffset= static_cast(hdr_addr.boffset - + TRX_UNDO_HISTORY_NODE); loop: - if (hdr_addr.page == FIL_NULL) { -func_exit: - mtr.commit(); - return err; - } + if (hdr_addr.page == FIL_NULL) + goto func_exit; - buf_block_t* block = buf_page_get_gen(page_id_t(rseg.space->id, - hdr_addr.page), - 0, RW_X_LATCH, nullptr, - BUF_GET_POSSIBLY_FREED, - &mtr, &err); - if (!block) { - goto func_exit; - } + buf_block_t *b= + buf_page_get_gen(page_id_t(rseg.space->id, hdr_addr.page), + 0, RW_X_LATCH, nullptr, BUF_GET_POSSIBLY_FREED, + &mtr, &err); + if (!b) + goto func_exit; - const trx_id_t undo_trx_no = mach_read_from_8( - block->page.frame + hdr_addr.boffset + TRX_UNDO_TRX_NO); + const trx_id_t undo_trx_no= + mach_read_from_8(b->page.frame + hdr_addr.boffset + TRX_UNDO_TRX_NO); - if (undo_trx_no >= limit.trx_no) { - if (undo_trx_no == limit.trx_no) { - err = trx_undo_truncate_start( - &rseg, hdr_addr.page, - hdr_addr.boffset, limit.undo_no); - } + if (undo_trx_no >= limit.trx_no) + { + if (undo_trx_no == limit.trx_no) + err = trx_undo_truncate_start(&rseg, hdr_addr.page, + hdr_addr.boffset, limit.undo_no); + goto func_exit; + } - goto func_exit; - } + fil_addr_t prev_hdr_addr= + flst_get_prev_addr(b->page.frame + hdr_addr.boffset + + TRX_UNDO_HISTORY_NODE); + prev_hdr_addr.boffset= static_cast(prev_hdr_addr.boffset - + TRX_UNDO_HISTORY_NODE); + err= trx_purge_remove_log_hdr(rseg_hdr, b, hdr_addr.boffset, &mtr); + if (UNIV_UNLIKELY(err != DB_SUCCESS)) + goto func_exit; - fil_addr_t prev_hdr_addr = flst_get_prev_addr( - block->page.frame + hdr_addr.boffset + TRX_UNDO_HISTORY_NODE); - prev_hdr_addr.boffset = static_cast(prev_hdr_addr.boffset - - TRX_UNDO_HISTORY_NODE); - - if (!rseg.is_referenced() - && rseg.needs_purge <= (purge_sys.head.trx_no - ? purge_sys.head.trx_no - : purge_sys.tail.trx_no) - && mach_read_from_2(TRX_UNDO_SEG_HDR + TRX_UNDO_STATE - + block->page.frame) - == TRX_UNDO_TO_PURGE - && !mach_read_from_2(block->page.frame + hdr_addr.boffset - + TRX_UNDO_NEXT_LOG)) { - /* We can free the whole log segment. - This will call trx_purge_remove_log_hdr(). */ - err = trx_purge_free_segment(mtr, &rseg, hdr_addr); - } else { - /* Remove the log hdr from the rseg history. */ - rseg.history_size--; - err = trx_purge_remove_log_hdr(rseg_hdr, block, - hdr_addr.boffset, &mtr); - } + rseg_hdr->fix(); - mtr.commit(); - if (err != DB_SUCCESS) { - return err; - } - mtr.start(); + if (mach_read_from_2(b->page.frame + hdr_addr.boffset + TRX_UNDO_NEXT_LOG) || + rseg.is_referenced() || + rseg.needs_purge > (purge_sys.head.trx_no + ? purge_sys.head.trx_no + : purge_sys.tail.trx_no)) + /* We cannot free the entire undo page. */; + else + { + const uint32_t seg_size= + flst_get_len(TRX_UNDO_SEG_HDR + TRX_UNDO_PAGE_LIST + b->page.frame); + switch (mach_read_from_2(TRX_UNDO_SEG_HDR + TRX_UNDO_STATE + + b->page.frame)) { + case TRX_UNDO_TO_PURGE: + { + byte *hist= TRX_RSEG + TRX_RSEG_HISTORY_SIZE + rseg_hdr->page.frame; + ut_ad(mach_read_from_4(hist) >= seg_size); + mtr.write<4>(*rseg_hdr, hist, mach_read_from_4(hist) - seg_size); + } + free_segment: + ut_ad(rseg.curr_size >= seg_size); + rseg.curr_size-= seg_size; + trx_purge_free_segment(b, mtr); + break; + case TRX_UNDO_CACHED: + /* rseg.undo_cached must point to this page */ + trx_undo_t *undo= UT_LIST_GET_FIRST(rseg.undo_cached); + for (; undo; undo= UT_LIST_GET_NEXT(undo_list, undo)) + if (undo->hdr_page_no == hdr_addr.page) + goto found_cached; + ut_ad("inconsistent undo logs" == 0); + break; + found_cached: + UT_LIST_REMOVE(rseg.undo_cached, undo); + static_assert(FIL_NULL == 0xffffffff, ""); + if (UNIV_UNLIKELY(mach_read_from_4(TRX_RSEG + TRX_RSEG_FORMAT + + rseg_hdr->page.frame))) + trx_rseg_format_upgrade(rseg_hdr, &mtr); + mtr.memset(rseg_hdr, TRX_RSEG + TRX_RSEG_UNDO_SLOTS + + undo->id * TRX_RSEG_SLOT_SIZE, 4, 0xff); + ut_free(undo); + mtr.write<8,mtr_t::MAYBE_NOP>(*rseg_hdr, TRX_RSEG + TRX_RSEG_MAX_TRX_ID + + rseg_hdr->page.frame, + trx_sys.get_max_trx_id() - 1); + MONITOR_DEC(MONITOR_NUM_UNDO_SLOT_CACHED); + MONITOR_DEC(MONITOR_NUM_UNDO_SLOT_USED); + goto free_segment; + } + } - hdr_addr = prev_hdr_addr; + hdr_addr= prev_hdr_addr; - rseg_hdr = rseg.get(&mtr, &err); - if (!rseg_hdr) { - goto func_exit; - } + mtr.commit(); + ut_ad(rseg.history_size > 0); + rseg.history_size--; + log_free_check(); + mtr.start(); + rseg_hdr->page.lock.x_lock(); + if (UNIV_UNLIKELY(rseg_hdr->page.id() != rseg.page_id())) + { + rseg_hdr->unfix(); + rseg_hdr->page.lock.x_unlock(); + goto reget; + } + mtr.memo_push(rseg_hdr, MTR_MEMO_PAGE_X_MODIFY); - goto loop; + goto loop; } /** Cleanse purge queue to remove the rseg that reside in undo-tablespace -- cgit v1.2.1 From 204e7225dce32130ac2c96f469611d2cb421241e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 21 Apr 2023 17:58:18 +0300 Subject: Cleanup: MONITOR_EXISTING trx_undo_slots_used, trx_undo_slots_cached Let us remove explicit updates of MONITOR_NUM_UNDO_SLOT_USED and MONITOR_NUM_UNDO_SLOT_CACHED, and let us compute the rough values from trx_sys.rseg_array[] on demand. --- .../innodb/r/innodb_skip_innodb_is_tables.result | 4 +-- storage/innobase/srv/srv0mon.cc | 34 ++++++++++++++++++---- storage/innobase/trx/trx0purge.cc | 5 ---- storage/innobase/trx/trx0rseg.cc | 2 -- storage/innobase/trx/trx0undo.cc | 6 ---- .../rocksdb/r/innodb_i_s_tables_disabled.result | 4 +-- 6 files changed, 33 insertions(+), 22 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb_skip_innodb_is_tables.result b/mysql-test/suite/innodb/r/innodb_skip_innodb_is_tables.result index 9bdb546482e..9810e49bd20 100644 --- a/mysql-test/suite/innodb/r/innodb_skip_innodb_is_tables.result +++ b/mysql-test/suite/innodb/r/innodb_skip_innodb_is_tables.result @@ -164,8 +164,8 @@ trx_commits_insert_update transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NUL trx_rollbacks transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of transactions rolled back trx_rollbacks_savepoint transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of transactions rolled back to savepoint trx_rseg_history_len transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 value Length of the TRX_RSEG_HISTORY list -trx_undo_slots_used transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of undo slots used -trx_undo_slots_cached transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of undo slots cached +trx_undo_slots_used transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 value Number of undo slots used +trx_undo_slots_cached transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 value Number of undo slots cached trx_rseg_current_size transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 value Current rollback segment size in pages purge_del_mark_records purge 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of delete-marked rows purged purge_upd_exist_or_extern_records purge 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of purges on updates of existing records and updates on delete marked record with externally stored field diff --git a/storage/innobase/srv/srv0mon.cc b/storage/innobase/srv/srv0mon.cc index b6496d03908..2a3720641bc 100644 --- a/storage/innobase/srv/srv0mon.cc +++ b/storage/innobase/srv/srv0mon.cc @@ -704,16 +704,18 @@ static monitor_info_t innodb_counter_info[] = {"trx_rseg_history_len", "transaction", "Length of the TRX_RSEG_HISTORY list", static_cast( - MONITOR_EXISTING | MONITOR_DISPLAY_CURRENT | MONITOR_DEFAULT_ON), + MONITOR_EXISTING | MONITOR_DISPLAY_CURRENT), MONITOR_DEFAULT_START, MONITOR_RSEG_HISTORY_LEN}, {"trx_undo_slots_used", "transaction", "Number of undo slots used", - MONITOR_NONE, + static_cast( + MONITOR_EXISTING | MONITOR_DISPLAY_CURRENT), MONITOR_DEFAULT_START, MONITOR_NUM_UNDO_SLOT_USED}, {"trx_undo_slots_cached", "transaction", "Number of undo slots cached", - MONITOR_NONE, + static_cast( + MONITOR_EXISTING | MONITOR_DISPLAY_CURRENT | MONITOR_DEFAULT_ON), MONITOR_DEFAULT_START, MONITOR_NUM_UNDO_SLOT_CACHED}, {"trx_rseg_current_size", "transaction", @@ -1383,6 +1385,24 @@ TPOOL_SUPPRESS_TSAN static ulint srv_mon_get_rseg_size() return size; } +/** @return number of used undo log slots */ +TPOOL_SUPPRESS_TSAN static ulint srv_mon_get_rseg_used() +{ + ulint size= 0; + for (const auto &rseg : trx_sys.rseg_array) + size+= UT_LIST_GET_LEN(rseg.undo_list); + return size; +} + +/** @return number of cached undo log slots */ +TPOOL_SUPPRESS_TSAN static ulint srv_mon_get_rseg_cached() +{ + ulint size= 0; + for (const auto &rseg : trx_sys.rseg_array) + size+= UT_LIST_GET_LEN(rseg.undo_cached); + return size; +} + /****************************************************************//** This function consolidates some existing server counters used by "system status variables". These existing system variables do not have @@ -1690,7 +1710,12 @@ srv_mon_process_existing_counter( case MONITOR_RSEG_CUR_SIZE: value = srv_mon_get_rseg_size(); break; - + case MONITOR_NUM_UNDO_SLOT_USED: + value = srv_mon_get_rseg_used(); + break; + case MONITOR_NUM_UNDO_SLOT_CACHED: + value = srv_mon_get_rseg_cached(); + break; case MONITOR_OVLD_N_FILE_OPENED: value = fil_system.n_open; break; @@ -1812,7 +1837,6 @@ srv_mon_process_existing_counter( case MONITOR_TIMEOUT: value = lock_sys.timeouts; break; - default: ut_error; } diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 4a2fa214b37..7a63b1155b6 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -280,8 +280,6 @@ trx_purge_add_undo_to_history(const trx_t* trx, trx_undo_t*& undo, mtr_t* mtr) TRX_RSEG + TRX_RSEG_UNDO_SLOTS + undo->id * TRX_RSEG_SLOT_SIZE, 4, 0xff); - MONITOR_DEC(MONITOR_NUM_UNDO_SLOT_USED); - uint32_t hist_size = mach_read_from_4( TRX_RSEG_HISTORY_SIZE + TRX_RSEG + rseg_header->page.frame); @@ -363,7 +361,6 @@ trx_purge_add_undo_to_history(const trx_t* trx, trx_undo_t*& undo, mtr_t* mtr) if (undo->state == TRX_UNDO_CACHED) { UT_LIST_ADD_FIRST(rseg->undo_cached, undo); - MONITOR_INC(MONITOR_NUM_UNDO_SLOT_CACHED); } else { ut_ad(undo->state == TRX_UNDO_TO_PURGE); ut_free(undo); @@ -526,8 +523,6 @@ loop: mtr.write<8,mtr_t::MAYBE_NOP>(*rseg_hdr, TRX_RSEG + TRX_RSEG_MAX_TRX_ID + rseg_hdr->page.frame, trx_sys.get_max_trx_id() - 1); - MONITOR_DEC(MONITOR_NUM_UNDO_SLOT_CACHED); - MONITOR_DEC(MONITOR_NUM_UNDO_SLOT_USED); goto free_segment; } } diff --git a/storage/innobase/trx/trx0rseg.cc b/storage/innobase/trx/trx0rseg.cc index 6d95dcf06f1..0d7b96e9280 100644 --- a/storage/innobase/trx/trx0rseg.cc +++ b/storage/innobase/trx/trx0rseg.cc @@ -394,7 +394,6 @@ void trx_rseg_t::reinit(uint32_t page) { next= UT_LIST_GET_NEXT(undo_list, undo); UT_LIST_REMOVE(undo_cached, undo); - MONITOR_DEC(MONITOR_NUM_UNDO_SLOT_CACHED); ut_free(undo); } @@ -425,7 +424,6 @@ static dberr_t trx_undo_lists_init(trx_rseg_t *rseg, if (!undo) return DB_CORRUPTION; rseg->curr_size+= undo->size; - MONITOR_INC(MONITOR_NUM_UNDO_SLOT_USED); } } diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc index 33b1f93ff65..bccb6538e63 100644 --- a/storage/innobase/trx/trx0undo.cc +++ b/storage/innobase/trx/trx0undo.cc @@ -535,8 +535,6 @@ trx_undo_seg_create(fil_space_t *space, buf_block_t *rseg_hdr, ulint *id, + slot_no * TRX_RSEG_SLOT_SIZE + rseg_hdr->page.frame, block->page.id().page_no()); - MONITOR_INC(MONITOR_NUM_UNDO_SLOT_USED); - *err = DB_SUCCESS; return block; } @@ -996,7 +994,6 @@ static void trx_undo_seg_free(const trx_undo_t *undo) static_assert(FIL_NULL == 0xffffffff, "compatibility"); mtr.memset(rseg_header, TRX_RSEG + TRX_RSEG_UNDO_SLOTS + undo->id * TRX_RSEG_SLOT_SIZE, 4, 0xff); - MONITOR_DEC(MONITOR_NUM_UNDO_SLOT_USED); } } @@ -1155,7 +1152,6 @@ corrupted_type: UT_LIST_ADD_LAST(rseg->undo_list, undo); } else { UT_LIST_ADD_LAST(rseg->undo_cached, undo); - MONITOR_INC(MONITOR_NUM_UNDO_SLOT_CACHED); } mtr.commit(); @@ -1333,7 +1329,6 @@ trx_undo_reuse_cached(trx_t* trx, trx_rseg_t* rseg, trx_undo_t** pundo, } UT_LIST_REMOVE(rseg->undo_cached, undo); - MONITOR_DEC(MONITOR_NUM_UNDO_SLOT_CACHED); *pundo = undo; @@ -1546,7 +1541,6 @@ void trx_undo_commit_cleanup(trx_undo_t *undo) if (undo->state == TRX_UNDO_CACHED) { UT_LIST_ADD_FIRST(rseg->undo_cached, undo); - MONITOR_INC(MONITOR_NUM_UNDO_SLOT_CACHED); undo = nullptr; } else { ut_ad(undo->state == TRX_UNDO_TO_PURGE); diff --git a/storage/rocksdb/mysql-test/rocksdb/r/innodb_i_s_tables_disabled.result b/storage/rocksdb/mysql-test/rocksdb/r/innodb_i_s_tables_disabled.result index d3f0ee3bcd9..064019a3bfc 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/innodb_i_s_tables_disabled.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/innodb_i_s_tables_disabled.result @@ -146,8 +146,8 @@ trx_commits_insert_update transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NUL trx_rollbacks transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of transactions rolled back trx_rollbacks_savepoint transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of transactions rolled back to savepoint trx_rseg_history_len transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 value Length of the TRX_RSEG_HISTORY list -trx_undo_slots_used transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of undo slots used -trx_undo_slots_cached transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of undo slots cached +trx_undo_slots_used transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 value Number of undo slots used +trx_undo_slots_cached transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 value Number of undo slots cached trx_rseg_current_size transaction 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 value Current rollback segment size in pages purge_del_mark_records purge 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of delete-marked rows purged purge_upd_exist_or_extern_records purge 0 NULL NULL NULL 0 NULL NULL NULL NULL NULL NULL NULL 0 counter Number of purges on updates of existing records and updates on delete marked record with externally stored field -- cgit v1.2.1 From 51e62cb3b3b5f28a67c3c5862b3cb60ecf87180a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 21 Apr 2023 17:58:26 +0300 Subject: MDEV-26782 InnoDB temporary tablespace: reclaiming of free space does not work The motivation of this change is to allow undo pages for temporary tables to be marked free as often as possible, so that we can avoid buf_pool.LRU eviction (and writes) of undo pages that contain data that is no longer needed. For temporary tables, no MVCC or purge of history is needed, and reusing cached undo log pages might not help that much. It is possible that this may cause some performance regression due to more frequent allocation and freeing of undo log pages, but I only measured a performance improvement. trx_write_serialisation_history(): Never cache temporary undo log pages. trx_undo_reuse_cached(): Assert that the rollback segment is persistent. trx_undo_assign_low(): Add template. Never invoke trx_undo_reuse_cached() for temporary tables. Tested by: Matthias Leich --- storage/innobase/include/trx0undo.h | 10 ++-- storage/innobase/trx/trx0rec.cc | 10 ++-- storage/innobase/trx/trx0trx.cc | 8 ++- storage/innobase/trx/trx0undo.cc | 114 +++++++++++++++++++----------------- 4 files changed, 78 insertions(+), 64 deletions(-) diff --git a/storage/innobase/include/trx0undo.h b/storage/innobase/include/trx0undo.h index 3474a903f6c..670fe00c25b 100644 --- a/storage/innobase/include/trx0undo.h +++ b/storage/innobase/include/trx0undo.h @@ -203,16 +203,18 @@ trx_undo_assign(trx_t* trx, dberr_t* err, mtr_t* mtr) MY_ATTRIBUTE((nonnull)); /** Assign an undo log for a transaction. A new undo log is created or a cached undo log reused. +@tparam is_temp whether this is temporary undo log @param[in,out] trx transaction @param[in] rseg rollback segment @param[out] undo the undo log -@param[out] err error code @param[in,out] mtr mini-transaction +@param[out] err error code @return the undo log block -@retval NULL on error */ +@retval nullptr on error */ +template buf_block_t* -trx_undo_assign_low(trx_t* trx, trx_rseg_t* rseg, trx_undo_t** undo, - dberr_t* err, mtr_t* mtr) +trx_undo_assign_low(trx_t *trx, trx_rseg_t *rseg, trx_undo_t **undo, + mtr_t *mtr, dberr_t *err) MY_ATTRIBUTE((nonnull, warn_unused_result)); /******************************************************************//** Sets the state of the undo log segment at a transaction finish. diff --git a/storage/innobase/trx/trx0rec.cc b/storage/innobase/trx/trx0rec.cc index dc24f083d05..d3f64754d89 100644 --- a/storage/innobase/trx/trx0rec.cc +++ b/storage/innobase/trx/trx0rec.cc @@ -1868,26 +1868,28 @@ trx_undo_report_row_operation( } mtr_t mtr; + dberr_t err; mtr.start(); trx_undo_t** pundo; trx_rseg_t* rseg; const bool is_temp = index->table->is_temporary(); + buf_block_t* undo_block; if (is_temp) { mtr.set_log_mode(MTR_LOG_NO_REDO); - rseg = trx->get_temp_rseg(); pundo = &trx->rsegs.m_noredo.undo; + undo_block = trx_undo_assign_low(trx, rseg, pundo, + &mtr, &err); } else { ut_ad(!trx->read_only); ut_ad(trx->id); pundo = &trx->rsegs.m_redo.undo; rseg = trx->rsegs.m_redo.rseg; + undo_block = trx_undo_assign_low(trx, rseg, pundo, + &mtr, &err); } - dberr_t err; - buf_block_t* undo_block = trx_undo_assign_low(trx, rseg, pundo, - &err, &mtr); trx_undo_t* undo = *pundo; ut_ad((err == DB_SUCCESS) == (undo_block != NULL)); if (UNIV_UNLIKELY(undo_block == NULL)) { diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 88e42b2ebad..b96ababb1f3 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -1023,7 +1023,13 @@ trx_write_serialisation_history( mtr_t temp_mtr; temp_mtr.start(); temp_mtr.set_log_mode(MTR_LOG_NO_REDO); - trx_undo_set_state_at_finish(undo, &temp_mtr); + buf_block_t* block= buf_page_get(page_id_t(SRV_TMP_SPACE_ID, + undo->hdr_page_no), + 0, RW_X_LATCH, mtr); + ut_a(block); + temp_mtr.write<2>(*block, TRX_UNDO_SEG_HDR + TRX_UNDO_STATE + + block->page.frame, TRX_UNDO_TO_PURGE); + undo->state = TRX_UNDO_TO_PURGE; temp_mtr.commit(); } diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc index bccb6538e63..20434d9fb9c 100644 --- a/storage/innobase/trx/trx0undo.cc +++ b/storage/innobase/trx/trx0undo.cc @@ -1290,27 +1290,25 @@ trx_undo_create(trx_t* trx, trx_rseg_t* rseg, trx_undo_t** undo, @param[in,out] rseg rollback segment @param[out] pundo the undo log memory object @param[in,out] mtr mini-transaction +@param[out] err error code @return the undo log block @retval NULL if none cached */ static buf_block_t* trx_undo_reuse_cached(trx_t* trx, trx_rseg_t* rseg, trx_undo_t** pundo, - mtr_t* mtr) + mtr_t* mtr, dberr_t *err) { - if (rseg->is_persistent()) { - ut_ad(rseg->is_referenced()); - if (rseg->needs_purge <= trx->id) { - /* trx_purge_truncate_history() compares - rseg->needs_purge <= head.trx_no - so we need to compensate for that. - The rseg->needs_purge after crash - recovery would be at least trx->id + 1, - because that is the minimum possible value - assigned by trx_serialise() on commit. */ - rseg->needs_purge = trx->id + 1; - } - } else { - ut_ad(!rseg->is_referenced()); + ut_ad(rseg->is_persistent()); + ut_ad(rseg->is_referenced()); + if (rseg->needs_purge <= trx->id) { + /* trx_purge_truncate_history() compares + rseg->needs_purge <= head.trx_no + so we need to compensate for that. + The rseg->needs_purge after crash + recovery would be at least trx->id + 1, + because that is the minimum possible value + assigned by trx_serialise() on commit. */ + rseg->needs_purge = trx->id + 1; } trx_undo_t* undo = UT_LIST_GET_FIRST(rseg->undo_cached); @@ -1321,9 +1319,10 @@ trx_undo_reuse_cached(trx_t* trx, trx_rseg_t* rseg, trx_undo_t** pundo, ut_ad(undo->size == 1); ut_ad(undo->id < TRX_RSEG_N_SLOTS); - buf_block_t* block = buf_page_get(page_id_t(undo->rseg->space->id, - undo->hdr_page_no), - 0, RW_X_LATCH, mtr); + buf_block_t* block = buf_page_get_gen(page_id_t(undo->rseg->space->id, + undo->hdr_page_no), + 0, RW_X_LATCH, nullptr, BUF_GET, + mtr, err); if (!block) { return NULL; } @@ -1374,11 +1373,12 @@ trx_undo_assign(trx_t* trx, dberr_t* err, mtr_t* mtr) BUF_GET, mtr, err); } + *err = DB_SUCCESS; trx_rseg_t* rseg = trx->rsegs.m_redo.rseg; rseg->latch.wr_lock(SRW_LOCK_CALL); buf_block_t* block = trx_undo_reuse_cached( - trx, rseg, &trx->rsegs.m_redo.undo, mtr); + trx, rseg, &trx->rsegs.m_redo.undo, mtr, err); if (!block) { block = trx_undo_create(trx, rseg, &trx->rsegs.m_redo.undo, @@ -1387,8 +1387,6 @@ trx_undo_assign(trx_t* trx, dberr_t* err, mtr_t* mtr) if (!block) { goto func_exit; } - } else { - *err = DB_SUCCESS; } UT_LIST_ADD_FIRST(rseg->undo_list, trx->rsegs.m_redo.undo); @@ -1400,18 +1398,20 @@ func_exit: /** Assign an undo log for a transaction. A new undo log is created or a cached undo log reused. +@tparam is_temp whether this is temporary undo log @param[in,out] trx transaction @param[in] rseg rollback segment @param[out] undo the undo log -@param[out] err error code @param[in,out] mtr mini-transaction +@param[out] err error code @return the undo log block -@retval NULL on error */ +@retval nullptr on error */ +template buf_block_t* -trx_undo_assign_low(trx_t* trx, trx_rseg_t* rseg, trx_undo_t** undo, - dberr_t* err, mtr_t* mtr) +trx_undo_assign_low(trx_t *trx, trx_rseg_t *rseg, trx_undo_t **undo, + mtr_t *mtr, dberr_t *err) { - ut_d(const bool is_temp = rseg == trx->rsegs.m_noredo.rseg); + ut_ad(is_temp == (rseg == trx->rsegs.m_noredo.rseg)); ut_ad(is_temp || rseg == trx->rsegs.m_redo.rseg); ut_ad(undo == (is_temp ? &trx->rsegs.m_noredo.undo @@ -1431,19 +1431,24 @@ trx_undo_assign_low(trx_t* trx, trx_rseg_t* rseg, trx_undo_t** undo, *err = DB_TOO_MANY_CONCURRENT_TRXS; return NULL; ); + *err = DB_SUCCESS; rseg->latch.wr_lock(SRW_LOCK_CALL); - buf_block_t* block = trx_undo_reuse_cached(trx, rseg, undo, mtr); - - if (!block) { - block = trx_undo_create(trx, rseg, undo, err, mtr); - ut_ad(!block == (*err != DB_SUCCESS)); - if (!block) { - goto func_exit; - } + buf_block_t* block; + if (is_temp) { + ut_ad(!UT_LIST_GET_LEN(rseg->undo_cached)); } else { - *err = DB_SUCCESS; + block = trx_undo_reuse_cached(trx, rseg, undo, mtr, err); + if (block) { + goto got_block; + } + } + block = trx_undo_create(trx, rseg, undo, err, mtr); + ut_ad(!block == (*err != DB_SUCCESS)); + if (!block) { + goto func_exit; } +got_block: UT_LIST_ADD_FIRST(rseg->undo_list, *undo); func_exit: @@ -1451,6 +1456,13 @@ func_exit: return block; } +template buf_block_t* +trx_undo_assign_low(trx_t *trx, trx_rseg_t *rseg, trx_undo_t **undo, + mtr_t *mtr, dberr_t *err); +template buf_block_t* +trx_undo_assign_low(trx_t *trx, trx_rseg_t *rseg, trx_undo_t **undo, + mtr_t *mtr, dberr_t *err); + /******************************************************************//** Sets the state of the undo log segment at a transaction finish. @return undo log segment header page, x-latched */ @@ -1461,6 +1473,7 @@ trx_undo_set_state_at_finish( mtr_t* mtr) /*!< in: mtr */ { ut_ad(undo->id < TRX_RSEG_N_SLOTS); + ut_ad(undo->rseg->is_persistent()); buf_block_t *block= buf_page_get(page_id_t(undo->rseg->space->id, undo->hdr_page_no), 0, @@ -1532,28 +1545,19 @@ the data can be discarded. @param undo temporary undo log */ void trx_undo_commit_cleanup(trx_undo_t *undo) { - trx_rseg_t* rseg = undo->rseg; - ut_ad(rseg->space == fil_system.temp_space); - - rseg->latch.wr_lock(SRW_LOCK_CALL); - - UT_LIST_REMOVE(rseg->undo_list, undo); - - if (undo->state == TRX_UNDO_CACHED) { - UT_LIST_ADD_FIRST(rseg->undo_cached, undo); - undo = nullptr; - } else { - ut_ad(undo->state == TRX_UNDO_TO_PURGE); + trx_rseg_t *rseg= undo->rseg; + ut_ad(rseg->space == fil_system.temp_space); + rseg->latch.wr_lock(SRW_LOCK_CALL); - /* Delete first the undo log segment in the file */ - trx_undo_seg_free(undo); + UT_LIST_REMOVE(rseg->undo_list, undo); + ut_ad(undo->state == TRX_UNDO_TO_PURGE); + /* Delete first the undo log segment in the file */ + trx_undo_seg_free(undo); + ut_ad(rseg->curr_size > undo->size); + rseg->curr_size-= undo->size; - ut_ad(rseg->curr_size > undo->size); - rseg->curr_size -= undo->size; - } - - rseg->latch.wr_unlock(); - ut_free(undo); + rseg->latch.wr_unlock(); + ut_free(undo); } /** At shutdown, frees the undo logs of a transaction. */ -- cgit v1.2.1 From 9f98a2acd71dcfbdb32a08e72a4737359ed9be40 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Thu, 13 Apr 2023 15:42:53 +0400 Subject: MDEV-30968 mariadb-backup does not copy Aria logs if aria_log_dir_path is used - `mariadb-backup --backup` was fixed to fetch the value of the @@aria_log_dir_path server variable and copy aria_log* files from @@aria_log_dir_path directory to the backup directory. Absolute and relative (to --datadir) paths are supported. Before this change aria_log* files were copied to the backup only if they were in the default location in @@datadir. - `mariadb-backup --copy-back` now understands a new my.cnf and command line parameter --aria-log-dir-path. `mariadb-backup --copy-back` in the main loop in copy_back() (when copying back from the backup directory to --datadir) was fixed to ignore all aria_log* files. A new function copy_back_aria_logs() was added. It consists of a separate loop copying back aria_log* files from the backup directory to the directory specified in --aria-log-dir-path. Absolute and relative (to --datadir) paths are supported. If --aria-log-dir-path is not specified, aria_log* files are copied to --datadir by default. - The function is_absolute_path() was fixed to understand MTR style paths on Windows with forward slashes, e.g. --aria-log-dir-path=D:/Buildbot/amd64-windows/build/mysql-test/var/... --- extra/mariabackup/backup_copy.cc | 70 ++++++++++++-- extra/mariabackup/backup_mysql.cc | 7 ++ extra/mariabackup/xtrabackup.cc | 14 ++- extra/mariabackup/xtrabackup.h | 1 + .../suite/mariabackup/aria_log_dir_path.result | 41 ++++++++ .../suite/mariabackup/aria_log_dir_path.test | 105 +++++++++++++++++++++ .../suite/mariabackup/aria_log_dir_path_rel.result | 41 ++++++++ .../suite/mariabackup/aria_log_dir_path_rel.test | 4 + storage/innobase/include/os0file.h | 6 +- 9 files changed, 281 insertions(+), 8 deletions(-) create mode 100644 mysql-test/suite/mariabackup/aria_log_dir_path.result create mode 100644 mysql-test/suite/mariabackup/aria_log_dir_path.test create mode 100644 mysql-test/suite/mariabackup/aria_log_dir_path_rel.result create mode 100644 mysql-test/suite/mariabackup/aria_log_dir_path_rel.test diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index ffaf6dc98e4..8ab52fa983b 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -130,7 +130,9 @@ struct datadir_thread_ctxt_t { bool ret; }; -static bool backup_files_from_datadir(ds_ctxt *ds_data, const char *dir_path); +static bool backup_files_from_datadir(ds_ctxt_t *ds_data, + const char *dir_path, + const char *prefix); /************************************************************************ Retirn true if character if file separator */ @@ -1499,7 +1501,11 @@ bool backup_start(ds_ctxt *ds_data, ds_ctxt *ds_meta, return(false); } - if (!backup_files_from_datadir(ds_data, fil_path_to_mysql_datadir)) { + if (!backup_files_from_datadir(ds_data, fil_path_to_mysql_datadir, + "aws-kms-key") || + !backup_files_from_datadir(ds_data, + aria_log_dir_path, + "aria_log")) { return false; } @@ -1714,7 +1720,12 @@ ibx_copy_incremental_over_full() } } - if (!(ret = backup_files_from_datadir(ds_data, xtrabackup_incremental_dir))) + if (!(ret = backup_files_from_datadir(ds_data, + xtrabackup_incremental_dir, + "aws-kms-key")) || + !(ret = backup_files_from_datadir(ds_data, + xtrabackup_incremental_dir, + "aria_log"))) goto cleanup; /* copy supplementary files */ @@ -1829,6 +1840,41 @@ public: } }; + +static inline bool +is_aria_log_dir_file(const datadir_node_t &node) +{ + return starts_with(node.filepath_rel, "aria_log"); +} + + +bool +copy_back_aria_logs() +{ + Copy_back_dst_dir dst_dir_buf; + const char *dstdir= dst_dir_buf.make(aria_log_dir_path); + std::unique_ptr + ds_ctxt_aria_log_dir_path(ds_create(dstdir, DS_TYPE_LOCAL), ds_destroy); + + datadir_node_t node; + datadir_node_init(&node); + datadir_iter_t *it = datadir_iter_new(".", false); + + while (datadir_iter_next(it, &node)) + { + if (!is_aria_log_dir_file(node)) + continue; + if (!copy_or_move_file(ds_ctxt_aria_log_dir_path.get(), + node.filepath, node.filepath_rel, + dstdir, 1)) + return false; + } + datadir_node_free(&node); + datadir_iter_free(it); + return true; +} + + bool copy_back() { @@ -1861,6 +1907,10 @@ copy_back() && !directory_exists(srv_log_group_home_dir, true)) { return(false); } + if (aria_log_dir_path && *aria_log_dir_path + && !directory_exists(aria_log_dir_path, true)) { + return false; + } /* cd to backup directory */ if (my_setwd(xtrabackup_target_dir, MYF(MY_WME))) @@ -1869,6 +1919,9 @@ copy_back() return(false); } + if (!copy_back_aria_logs()) + return false; + /* parse data file path */ if (!innobase_data_file_path) { @@ -1973,6 +2026,10 @@ copy_back() int i_tmp; bool is_ibdata_file; + /* Skip aria log files */ + if (is_aria_log_dir_file(node)) + continue; + if (strstr(node.filepath,"/" ROCKSDB_BACKUP_DIR "/") #ifdef _WIN32 || strstr(node.filepath,"\\" ROCKSDB_BACKUP_DIR "\\") @@ -2209,7 +2266,9 @@ decrypt_decompress() Do not copy the Innodb files (ibdata1, redo log files), as this is done in a separate step. */ -static bool backup_files_from_datadir(ds_ctxt *ds_data, const char *dir_path) +static bool backup_files_from_datadir(ds_ctxt_t *ds_data, + const char *dir_path, + const char *prefix) { os_file_dir_t dir = os_file_opendir(dir_path); if (dir == IF_WIN(INVALID_HANDLE_VALUE, nullptr)) return false; @@ -2225,8 +2284,7 @@ static bool backup_files_from_datadir(ds_ctxt *ds_data, const char *dir_path) if (!pname) pname = info.name; - if (!starts_with(pname, "aws-kms-key") && - !starts_with(pname, "aria_log")) + if (!starts_with(pname, prefix)) /* For ES exchange the above line with the following code: (!xtrabackup_prepare || !xtrabackup_incremental_dir || !starts_with(pname, "aria_log"))) diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index 6003bfb36c4..1831485e957 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -367,6 +367,7 @@ bool get_mysql_vars(MYSQL *connection) char *innodb_undo_directory_var= NULL; char *innodb_page_size_var= NULL; char *innodb_undo_tablespaces_var= NULL; + char *aria_log_dir_path_var= NULL; char *page_zip_level_var= NULL; char *ignore_db_dirs= NULL; char *endptr; @@ -397,6 +398,7 @@ bool get_mysql_vars(MYSQL *connection) {"innodb_undo_tablespaces", &innodb_undo_tablespaces_var}, {"innodb_compression_level", &page_zip_level_var}, {"ignore_db_dirs", &ignore_db_dirs}, + {"aria_log_dir_path", &aria_log_dir_path_var}, {NULL, NULL}}; read_mysql_variables(connection, "SHOW VARIABLES", mysql_vars, true); @@ -538,6 +540,11 @@ bool get_mysql_vars(MYSQL *connection) ut_ad(*endptr == 0); } + if (aria_log_dir_path_var) + { + aria_log_dir_path= my_strdup(aria_log_dir_path_var, MYF(MY_FAE)); + } + if (page_zip_level_var != NULL) { page_zip_level= strtoul(page_zip_level_var, &endptr, 10); diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index ee12034c910..96c90b5afad 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -266,6 +266,8 @@ my_bool innobase_locks_unsafe_for_binlog; my_bool innobase_rollback_on_timeout; my_bool innobase_create_status_file; +char *aria_log_dir_path; + /* The following counter is used to convey information to InnoDB about server activity: in selects it is not sensible to call srv_active_wake_master_thread after each fetch or search, we only do @@ -1105,7 +1107,8 @@ enum options_xtrabackup OPT_XTRA_CHECK_PRIVILEGES, OPT_XTRA_MYSQLD_ARGS, OPT_XB_IGNORE_INNODB_PAGE_CORRUPTION, - OPT_INNODB_FORCE_RECOVERY + OPT_INNODB_FORCE_RECOVERY, + OPT_ARIA_LOG_DIR_PATH }; struct my_option xb_client_options[]= { @@ -1696,6 +1699,11 @@ struct my_option xb_server_options[] = &innodb_log_checksums, &innodb_log_checksums, 0, GET_BOOL, REQUIRED_ARG, 1, 0, 0, 0, 0, 0 }, + {"aria_log_dir_path", OPT_ARIA_LOG_DIR_PATH, + "Path to individual files and their sizes.", + &aria_log_dir_path, &aria_log_dir_path, + 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"open_files_limit", OPT_OPEN_FILES_LIMIT, "the maximum number of file " "descriptors to reserve with setrlimit().", (G_PTR*) &xb_open_files_limit, (G_PTR*) &xb_open_files_limit, 0, GET_ULONG, @@ -2012,6 +2020,10 @@ xb_get_one_option(int optid, } break; + case OPT_ARIA_LOG_DIR_PATH: + ADD_PRINT_PARAM_OPT(aria_log_dir_path); + break; + case OPT_XTRA_TARGET_DIR: strmake(xtrabackup_real_target_dir,argument, sizeof(xtrabackup_real_target_dir)-1); xtrabackup_target_dir= xtrabackup_real_target_dir; diff --git a/extra/mariabackup/xtrabackup.h b/extra/mariabackup/xtrabackup.h index de3a96443a3..df2f766aedb 100644 --- a/extra/mariabackup/xtrabackup.h +++ b/extra/mariabackup/xtrabackup.h @@ -74,6 +74,7 @@ extern char *xtrabackup_incremental_dir; extern char *xtrabackup_incremental_basedir; extern char *innobase_data_home_dir; extern char *innobase_buffer_pool_filename; +extern char *aria_log_dir_path; extern char *xb_plugin_dir; extern char *xb_rocksdb_datadir; extern my_bool xb_backup_rocksdb; diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path.result b/mysql-test/suite/mariabackup/aria_log_dir_path.result new file mode 100644 index 00000000000..1a877321bbe --- /dev/null +++ b/mysql-test/suite/mariabackup/aria_log_dir_path.result @@ -0,0 +1,41 @@ +# +# MDEV-30968 mariadb-backup does not copy Aria logs if aria_log_dir_path is used +# +# Restart mariadbd with the test specific parameters +# restart: --aria-log-file-size=8388608 --aria-log-purge-type=external --loose-aria-log-dir-path=MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path +# Create and populate an Aria table (and Aria logs) +CREATE TABLE t1 (id INT, txt LONGTEXT) ENGINE=Aria; +BEGIN NOT ATOMIC +FOR id IN 0..9 DO +INSERT INTO test.t1 (id, txt) VALUES (id, REPEAT(id,1024*1024)); +END FOR; +END; +$$ +# Testing aria log files before --backup +SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; +SHOW ENGINE aria logs; +Type Name Status +Aria aria_log.00000001 free +Aria aria_log.00000002 in use +# mariadb-backup --backup +# mariadb-backup --prepare +# shutdown server +# remove datadir +# remove aria-log-dir-path +# mariadb-backup --copy-back +# with parameters: --defaults-file=MYSQLTEST_VARDIR/my.cnf --copy-back --datadir=MYSQLTEST_VARDIR/mysqld.1/data/ --target-dir=MYSQLTEST_VARDIR/tmp/backup --parallel=2 --throttle=1 --aria-log-dir-path=MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path +# starting server +# restart: --aria-log-file-size=8388608 --aria-log-purge-type=external --loose-aria-log-dir-path=MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path +# Check that the table is there after --copy-back +SELECT COUNT(*) from t1; +COUNT(*) +10 +DROP TABLE t1; +# Testing aria log files after --copy-back +SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; +SHOW ENGINE aria logs; +Type Name Status +Aria aria_log.00000001 free +Aria aria_log.00000002 in use +# Restarting mariadbd with default parameters +# restart diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path.test b/mysql-test/suite/mariabackup/aria_log_dir_path.test new file mode 100644 index 00000000000..0178cd4eae5 --- /dev/null +++ b/mysql-test/suite/mariabackup/aria_log_dir_path.test @@ -0,0 +1,105 @@ +--source include/have_maria.inc + +--echo # +--echo # MDEV-30968 mariadb-backup does not copy Aria logs if aria_log_dir_path is used +--echo # + +--let $datadir=`SELECT @@datadir` +--let $targetdir=$MYSQLTEST_VARDIR/tmp/backup + +if ($ARIA_LOGDIR_MARIADB == '') +{ + --let $ARIA_LOGDIR_MARIADB=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path +} + +if ($ARIA_LOGDIR_FS == '') +{ + --let $ARIA_LOGDIR_FS=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path +} + +--let $server_parameters=--aria-log-file-size=8388608 --aria-log-purge-type=external --loose-aria-log-dir-path=$ARIA_LOGDIR_MARIADB + + +--echo # Restart mariadbd with the test specific parameters +--mkdir $ARIA_LOGDIR_FS +--let $restart_parameters=$server_parameters +--source include/restart_mysqld.inc + + +--echo # Create and populate an Aria table (and Aria logs) +CREATE TABLE t1 (id INT, txt LONGTEXT) ENGINE=Aria; +DELIMITER $$; +BEGIN NOT ATOMIC + FOR id IN 0..9 DO + INSERT INTO test.t1 (id, txt) VALUES (id, REPEAT(id,1024*1024)); + END FOR; +END; +$$ +DELIMITER ;$$ + + +--echo # Testing aria log files before --backup +SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; +--file_exists $ARIA_LOGDIR_FS/aria_log_control +--file_exists $ARIA_LOGDIR_FS/aria_log.00000001 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000002 +--error 1 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000003 +--replace_regex /Size +[0-9]+ ; .+aria_log/aria_log/ +SHOW ENGINE aria logs; + + +--echo # mariadb-backup --backup +--disable_result_log +--mkdir $targetdir +--exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir +--enable_result_log + + +--echo # mariadb-backup --prepare +--disable_result_log +--exec $XTRABACKUP --prepare --target-dir=$targetdir +--enable_result_log + + +--echo # shutdown server +--disable_result_log +--source include/shutdown_mysqld.inc +--echo # remove datadir +--rmdir $datadir +--echo # remove aria-log-dir-path +--rmdir $ARIA_LOGDIR_FS + +--echo # mariadb-backup --copy-back +--let $mariadb_backup_parameters=--defaults-file=$MYSQLTEST_VARDIR/my.cnf --copy-back --datadir=$datadir --target-dir=$targetdir --parallel=2 --throttle=1 --aria-log-dir-path=$ARIA_LOGDIR_MARIADB +--replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR $MYSQLTEST_VARDIR MYSQLTEST_VARDIR +--exec echo "# with parameters: $mariadb_backup_parameters" +--exec $XTRABACKUP $mariadb_backup_parameters + +--echo # starting server +--let $restart_parameters=$server_parameters +--source include/start_mysqld.inc +--enable_result_log +--rmdir $targetdir + + +--echo # Check that the table is there after --copy-back +SELECT COUNT(*) from t1; +DROP TABLE t1; + + +--echo # Testing aria log files after --copy-back +SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; +--file_exists $ARIA_LOGDIR_FS/aria_log_control +--file_exists $ARIA_LOGDIR_FS/aria_log.00000001 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000002 +--error 1 +--file_exists $ARIA_LOGDIR_FS/aria_log.00000003 +--replace_regex /Size +[0-9]+ ; .+aria_log/aria_log/ +SHOW ENGINE aria logs; + + +--echo # Restarting mariadbd with default parameters +--let $restart_parameters= +--source include/restart_mysqld.inc +--rmdir $ARIA_LOGDIR_FS diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path_rel.result b/mysql-test/suite/mariabackup/aria_log_dir_path_rel.result new file mode 100644 index 00000000000..7fef26096e0 --- /dev/null +++ b/mysql-test/suite/mariabackup/aria_log_dir_path_rel.result @@ -0,0 +1,41 @@ +# +# MDEV-30968 mariadb-backup does not copy Aria logs if aria_log_dir_path is used +# +# Restart mariadbd with the test specific parameters +# restart: --aria-log-file-size=8388608 --aria-log-purge-type=external --loose-aria-log-dir-path=../../tmp/backup_aria_log_dir_path_rel +# Create and populate an Aria table (and Aria logs) +CREATE TABLE t1 (id INT, txt LONGTEXT) ENGINE=Aria; +BEGIN NOT ATOMIC +FOR id IN 0..9 DO +INSERT INTO test.t1 (id, txt) VALUES (id, REPEAT(id,1024*1024)); +END FOR; +END; +$$ +# Testing aria log files before --backup +SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; +SHOW ENGINE aria logs; +Type Name Status +Aria aria_log.00000001 free +Aria aria_log.00000002 in use +# mariadb-backup --backup +# mariadb-backup --prepare +# shutdown server +# remove datadir +# remove aria-log-dir-path +# mariadb-backup --copy-back +# with parameters: --defaults-file=MYSQLTEST_VARDIR/my.cnf --copy-back --datadir=MYSQLTEST_VARDIR/mysqld.1/data/ --target-dir=MYSQLTEST_VARDIR/tmp/backup --parallel=2 --throttle=1 --aria-log-dir-path=../../tmp/backup_aria_log_dir_path_rel +# starting server +# restart: --aria-log-file-size=8388608 --aria-log-purge-type=external --loose-aria-log-dir-path=../../tmp/backup_aria_log_dir_path_rel +# Check that the table is there after --copy-back +SELECT COUNT(*) from t1; +COUNT(*) +10 +DROP TABLE t1; +# Testing aria log files after --copy-back +SET @@global.aria_checkpoint_interval=DEFAULT /*Force checkpoint*/; +SHOW ENGINE aria logs; +Type Name Status +Aria aria_log.00000001 free +Aria aria_log.00000002 in use +# Restarting mariadbd with default parameters +# restart diff --git a/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test b/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test new file mode 100644 index 00000000000..c8169959929 --- /dev/null +++ b/mysql-test/suite/mariabackup/aria_log_dir_path_rel.test @@ -0,0 +1,4 @@ +--let $ARIA_LOGDIR_MARIADB=../../tmp/backup_aria_log_dir_path_rel +--let $ARIA_LOGDIR_FS=$MYSQLTEST_VARDIR/tmp/backup_aria_log_dir_path_rel + +--source aria_log_dir_path.test diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h index 20fcc0b64b8..76d4f465b95 100644 --- a/storage/innobase/include/os0file.h +++ b/storage/innobase/include/os0file.h @@ -1567,7 +1567,11 @@ is_absolute_path( } #ifdef _WIN32 - if (path[1] == ':' && path[2] == OS_PATH_SEPARATOR) { + // This will conflict during a 10.5->10.6 merge. + // Choose the 10.6 version as is. + if (path[1] == ':' && + (path[2] == OS_PATH_SEPARATOR || + path[2] == OS_PATH_SEPARATOR_ALT)) { return(true); } #endif /* _WIN32 */ -- cgit v1.2.1 From 6dc6c22c14fe204dbac43b6132c5bd130c69aba1 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Fri, 21 Apr 2023 18:49:52 -0700 Subject: MDEV-31085 Crash when processing multi-update using view with optimizer_trace on This bug caused server crash when processing a multi-update statement that used views if optimizer tracing was enabled. The bug was introduced in the patch for MDEV-30539 that could incorrectly detect the most top level selects of queries if views were used in them. Approved by Oleksandr Byelkin --- mysql-test/main/opt_trace.result | 356 +++++++++++++++++++++++++++++++++++++++ mysql-test/main/opt_trace.test | 19 +++ sql/sql_select.cc | 4 +- 3 files changed, 377 insertions(+), 2 deletions(-) diff --git a/mysql-test/main/opt_trace.result b/mysql-test/main/opt_trace.result index a8b391ffbe8..a7a8fb88e6d 100644 --- a/mysql-test/main/opt_trace.result +++ b/mysql-test/main/opt_trace.result @@ -8505,5 +8505,361 @@ SELECT a FROM t1 WHERE (a,b) in (SELECT @c,@d); a DROP TABLE t1; # +# MDEV-31085: multi-update using view with optimizer trace enabled +# +SET SESSION optimizer_trace = 'enabled=on'; +CREATE TABLE t (a int, b int); +CREATE VIEW v AS SELECT 1 AS c UNION SELECT 2 AS c; +INSERT INTO t VALUES (0,4),(5,6); +UPDATE t, v SET t.b = t.a, t.a = v.c WHERE v.c < t.a; +SELECT * FROM information_schema.optimizer_trace; +QUERY TRACE MISSING_BYTES_BEYOND_MAX_MEM_SIZE INSUFFICIENT_PRIVILEGES +UPDATE t, v SET t.b = t.a, t.a = v.c WHERE v.c < t.a { + "steps": [ + { + "view": { + "table": "v", + "select_id": 2, + "algorithm": "materialized" + } + }, + { + "join_preparation": { + "select_id": 2, + "steps": [ + { + "expanded_query": "/* select#2 */ select 1 AS c" + } + ] + } + }, + { + "join_preparation": { + "select_id": 3, + "steps": [ + { + "expanded_query": "/* select#3 */ select 2 AS c" + } + ] + } + }, + { + "join_preparation": { + "select_id": 1, + "steps": [ + { + "expanded_query": "/* select#1 */ update t join v set t.b = t.a,t.a = v.c where v.c < t.a" + } + ] + } + }, + { + "join_optimization": { + "select_id": 1, + "steps": [ + { + "condition_processing": { + "condition": "WHERE", + "original_condition": "v.c < t.a", + "steps": [ + { + "transformation": "equality_propagation", + "resulting_condition": "v.c < t.a" + }, + { + "transformation": "constant_propagation", + "resulting_condition": "v.c < t.a" + }, + { + "transformation": "trivial_condition_removal", + "resulting_condition": "v.c < t.a" + } + ] + } + }, + { + "join_optimization": { + "select_id": 2, + "steps": [] + } + }, + { + "join_optimization": { + "select_id": 3, + "steps": [] + } + }, + { + "table_dependencies": [ + { + "table": "t", + "row_may_be_null": false, + "map_bit": 0, + "depends_on_map_bits": [] + }, + { + "table": "", + "row_may_be_null": false, + "map_bit": 1, + "depends_on_map_bits": [] + } + ] + }, + { + "ref_optimizer_key_uses": [] + }, + { + "rows_estimation": [ + { + "table": "t", + "table_scan": { + "rows": 2, + "cost": 2.0044 + } + }, + { + "table": "", + "table_scan": { + "rows": 2, + "cost": 2 + } + } + ] + }, + { + "considered_execution_plans": [ + { + "plan_prefix": [], + "table": "t", + "best_access_path": { + "considered_access_paths": [ + { + "access_type": "scan", + "resulting_rows": 2, + "cost": 2.0044, + "chosen": true + } + ], + "chosen_access_method": { + "type": "scan", + "records": 2, + "cost": 2.0044, + "uses_join_buffering": false + } + }, + "rows_for_plan": 2, + "cost_for_plan": 2.4044, + "rest_of_plan": [ + { + "plan_prefix": ["t"], + "table": "", + "best_access_path": { + "considered_access_paths": [ + { + "access_type": "scan", + "resulting_rows": 2, + "cost": 2, + "chosen": true + } + ], + "chosen_access_method": { + "type": "scan", + "records": 2, + "cost": 2, + "uses_join_buffering": true + } + }, + "rows_for_plan": 4, + "cost_for_plan": 5.2044, + "estimated_join_cardinality": 4 + } + ] + }, + { + "plan_prefix": [], + "table": "", + "best_access_path": { + "considered_access_paths": [ + { + "access_type": "scan", + "resulting_rows": 2, + "cost": 2, + "chosen": true + } + ], + "chosen_access_method": { + "type": "scan", + "records": 2, + "cost": 2, + "uses_join_buffering": false + } + }, + "rows_for_plan": 2, + "cost_for_plan": 2.4, + "rest_of_plan": [ + { + "plan_prefix": [""], + "table": "t", + "best_access_path": { + "considered_access_paths": [ + { + "access_type": "scan", + "resulting_rows": 2, + "cost": 2.0044, + "chosen": true + } + ], + "chosen_access_method": { + "type": "scan", + "records": 2, + "cost": 2.0044, + "uses_join_buffering": true + } + }, + "rows_for_plan": 4, + "cost_for_plan": 5.2044, + "pruned_by_cost": true + } + ] + } + ] + }, + { + "best_join_order": ["t", ""] + }, + { + "attaching_conditions_to_tables": { + "original_condition": "v.c < t.a", + "attached_conditions_computation": [], + "attached_conditions_summary": [ + { + "table": "t", + "attached": null + }, + { + "table": "", + "attached": "v.c < t.a" + } + ] + } + } + ] + } + }, + { + "join_execution": { + "select_id": 1, + "steps": [ + { + "join_execution": { + "select_id": 2, + "steps": [] + } + }, + { + "join_execution": { + "select_id": 3, + "steps": [] + } + }, + { + "join_preparation": { + "select_id": "fake", + "steps": [ + { + "expanded_query": "select c AS c from dual" + } + ] + } + }, + { + "join_optimization": { + "select_id": "fake", + "steps": [ + { + "table_dependencies": [ + { + "table": "union", + "row_may_be_null": false, + "map_bit": 0, + "depends_on_map_bits": [] + } + ] + }, + { + "rows_estimation": [ + { + "table": "union", + "table_scan": { + "rows": 2, + "cost": 10.1 + } + } + ] + }, + { + "considered_execution_plans": [ + { + "plan_prefix": [], + "table": "union", + "best_access_path": { + "considered_access_paths": [ + { + "access_type": "scan", + "resulting_rows": 2, + "cost": 10.1, + "chosen": true + } + ], + "chosen_access_method": { + "type": "scan", + "records": 2, + "cost": 10.1, + "uses_join_buffering": false + } + }, + "rows_for_plan": 2, + "cost_for_plan": 10.5, + "estimated_join_cardinality": 2 + } + ] + }, + { + "best_join_order": ["union"] + }, + { + "attaching_conditions_to_tables": { + "original_condition": null, + "attached_conditions_computation": [], + "attached_conditions_summary": [ + { + "table": "union", + "attached": null + } + ] + } + } + ] + } + }, + { + "join_execution": { + "select_id": "fake", + "steps": [] + } + } + ] + } + } + ] +} 0 0 +SELECT * FROM t; +a b +0 4 +1 5 +SET optimizer_trace=DEFAULT; +DROP VIEW v; +DROP TABLE t; +# # End of 10.4 tests # diff --git a/mysql-test/main/opt_trace.test b/mysql-test/main/opt_trace.test index 0785d828a07..e0be5360069 100644 --- a/mysql-test/main/opt_trace.test +++ b/mysql-test/main/opt_trace.test @@ -677,6 +677,25 @@ INSERT INTO t1 VALUES (0,0); SELECT a FROM t1 WHERE (a,b) in (SELECT @c,@d); DROP TABLE t1; +--echo # +--echo # MDEV-31085: multi-update using view with optimizer trace enabled +--echo # + +SET SESSION optimizer_trace = 'enabled=on'; + +CREATE TABLE t (a int, b int); +CREATE VIEW v AS SELECT 1 AS c UNION SELECT 2 AS c; +INSERT INTO t VALUES (0,4),(5,6); +UPDATE t, v SET t.b = t.a, t.a = v.c WHERE v.c < t.a; +SELECT * FROM information_schema.optimizer_trace; + +SELECT * FROM t; + +SET optimizer_trace=DEFAULT; + +DROP VIEW v; +DROP TABLE t; + --echo # --echo # End of 10.4 tests --echo # diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 0651c1d58bd..03a2c3d0853 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -28050,7 +28050,7 @@ void st_select_lex::print_item_list(THD *thd, String *str, outer_select() can not be used here because it is for name resolution and will return NULL at any end of name resolution chain (view/derived) */ - bool top_level= (get_master()->get_master() == 0); + bool top_level= (get_master() == &thd->lex->unit); List_iterator_fast it(item_list); Item *item; while ((item= it++)) @@ -28157,7 +28157,7 @@ void st_select_lex::print(THD *thd, String *str, enum_query_type query_type) return; } - bool top_level= (get_master()->get_master() == 0); + bool top_level= (get_master() == &thd->lex->unit); enum explainable_cmd_type sel_type= SELECT_CMD; if (top_level) sel_type= get_explainable_cmd_type(thd); -- cgit v1.2.1 From 2c567b2fa37b60fd6db06ddc113d107fd9863208 Mon Sep 17 00:00:00 2001 From: Thirunarayanan Balathandayuthapani Date: Sat, 22 Apr 2023 16:42:52 +0530 Subject: MDEV-30996 insert.. select in presence of full text index freezes all other commits at commit time - This patch does the following: git revert --no-commit 673243c8938957ef1c01ffd3bfd35b5ae31ef484 git revert --no-commit 6c669b9586f72d6d760cc3956c1a0cb09ace2367 git revert --no-commit bacaf2d4f4c6d77a0b6c1ae4daddd19f81ef6fa3 git checkout HEAD mysql-test git revert --no-commit 1fd7d3a9adac50de37e40e92188077e3515de505 Above command reverts MDEV-29277, MDEV-25581, MDEV-29342. When binlog is enabled, trasaction takes a lot of time to do sync operation on innodb fts table. This leads to block of other transaction commit. To avoid this failure, remove the fulltext sync operation during transaction commit. So reverted MDEV-25581 related patches. We filed MDEV-31105 to avoid the memory consumption problem during fulltext sync operation. --- .../suite/innodb_fts/r/concurrent_insert.result | 2 +- mysql-test/suite/innodb_fts/r/sync.result | 16 +- mysql-test/suite/innodb_fts/r/sync_block.result | 83 ++++++ .../suite/innodb_fts/t/concurrent_insert.test | 2 +- mysql-test/suite/innodb_fts/t/sync.test | 4 +- mysql-test/suite/innodb_fts/t/sync_block.test | 124 ++++++++ storage/innobase/fts/fts0fts.cc | 325 +++++++++++++++------ storage/innobase/fts/fts0opt.cc | 56 +++- storage/innobase/handler/handler0alter.cc | 8 +- storage/innobase/include/fts0fts.h | 9 +- storage/innobase/include/fts0types.h | 32 +- 11 files changed, 552 insertions(+), 109 deletions(-) create mode 100644 mysql-test/suite/innodb_fts/r/sync_block.result create mode 100644 mysql-test/suite/innodb_fts/t/sync_block.test diff --git a/mysql-test/suite/innodb_fts/r/concurrent_insert.result b/mysql-test/suite/innodb_fts/r/concurrent_insert.result index 2335982816b..bc47511b046 100644 --- a/mysql-test/suite/innodb_fts/r/concurrent_insert.result +++ b/mysql-test/suite/innodb_fts/r/concurrent_insert.result @@ -19,7 +19,7 @@ INSERT INTO t2 VALUES('mariadb'); connection default; SET @saved_dbug = @@GLOBAL.debug_dbug; SET GLOBAL debug_dbug ='+d,fts_instrument_sync_request,ib_optimize_wq_hang'; -SET DEBUG_SYNC= 'fts_sync_end +SET DEBUG_SYNC= 'fts_instrument_sync_request SIGNAL drop_index_start WAIT_FOR sync_op'; INSERT INTO t1 VALUES('Keyword'); connect con1,localhost,root,,,; diff --git a/mysql-test/suite/innodb_fts/r/sync.result b/mysql-test/suite/innodb_fts/r/sync.result index 74a5d2f13fb..928efffdb21 100644 --- a/mysql-test/suite/innodb_fts/r/sync.result +++ b/mysql-test/suite/innodb_fts/r/sync.result @@ -11,19 +11,19 @@ INSERT INTO t1(title) VALUES('database'); connection con1; SET @old_dbug = @@SESSION.debug_dbug; SET debug_dbug = '+d,fts_instrument_sync_debug'; -SET DEBUG_SYNC= 'fts_sync_end SIGNAL written WAIT_FOR selected'; +SET DEBUG_SYNC= 'fts_write_node SIGNAL written WAIT_FOR selected'; INSERT INTO t1(title) VALUES('mysql database'); connection default; SET DEBUG_SYNC= 'now WAIT_FOR written'; SET GLOBAL innodb_ft_aux_table="test/t1"; SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION -SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE; -WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION database 2 3 2 2 0 database 2 3 2 3 6 mysql 1 3 2 1 0 mysql 1 3 2 3 0 +SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE; +WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION SET GLOBAL innodb_ft_aux_table=default; SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); FTS_DOC_ID title @@ -59,7 +59,7 @@ INSERT INTO t1(title) VALUES('mysql'); INSERT INTO t1(title) VALUES('database'); connection con1; SET debug_dbug = '+d,fts_instrument_sync_debug'; -SET DEBUG_SYNC= 'fts_sync_end SIGNAL written WAIT_FOR inserted'; +SET DEBUG_SYNC= 'fts_write_node SIGNAL written WAIT_FOR inserted'; INSERT INTO t1(title) VALUES('mysql database'); connection default; SET DEBUG_SYNC= 'now WAIT_FOR written'; @@ -70,14 +70,14 @@ SET debug_dbug = @old_dbug; SET GLOBAL innodb_ft_aux_table="test/t1"; SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_CACHE; WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION -database 4 4 1 4 6 -mysql 4 4 1 4 0 SELECT * FROM INFORMATION_SCHEMA.INNODB_FT_INDEX_TABLE; WORD FIRST_DOC_ID LAST_DOC_ID DOC_COUNT DOC_ID POSITION database 2 3 2 2 0 database 2 3 2 3 6 -mysql 1 3 2 1 0 -mysql 1 3 2 3 0 +database 4 4 1 4 6 +mysql 1 4 3 1 0 +mysql 1 4 3 3 0 +mysql 1 4 3 4 0 SET GLOBAL innodb_ft_aux_table=default; SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); FTS_DOC_ID title diff --git a/mysql-test/suite/innodb_fts/r/sync_block.result b/mysql-test/suite/innodb_fts/r/sync_block.result new file mode 100644 index 00000000000..65bee127e80 --- /dev/null +++ b/mysql-test/suite/innodb_fts/r/sync_block.result @@ -0,0 +1,83 @@ +SET @old_log_output = @@global.log_output; +SET @old_slow_query_log = @@global.slow_query_log; +SET @old_general_log = @@global.general_log; +SET @old_long_query_time = @@global.long_query_time; +SET @old_debug = @@global.debug_dbug; +SET GLOBAL log_output = 'TABLE'; +SET GLOBAL general_log = 1; +SET GLOBAL slow_query_log = 1; +SET GLOBAL long_query_time = 1; +connect con1,localhost,root,,; +connect con2,localhost,root,,; +connection default; +# Case 1: Sync blocks DML(insert) on the same table. +CREATE TABLE t1 ( +FTS_DOC_ID BIGINT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY, +title VARCHAR(200), +FULLTEXT(title) +) ENGINE = InnoDB; +connection con1; +SET GLOBAL debug_dbug='+d,fts_instrument_sync_debug,fts_instrument_sync_sleep'; +SET DEBUG_SYNC= 'fts_sync_begin SIGNAL begin WAIT_FOR continue'; +INSERT INTO t1(title) VALUES('mysql database'); +connection con2; +SET DEBUG_SYNC= 'now WAIT_FOR begin'; +SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); +connection default; +SET DEBUG_SYNC= 'now SIGNAL continue'; +connection con1; +/* connection con1 */ INSERT INTO t1(title) VALUES('mysql database'); +connection con2; +/* conneciton con2 */ SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); +FTS_DOC_ID title +connection default; +# make con1 & con2 show up in mysql.slow_log +SELECT SLEEP(2); +SLEEP(2) +0 +# slow log results should only contain INSERT INTO t1. +SELECT sql_text FROM mysql.slow_log WHERE query_time >= '00:00:02'; +sql_text +INSERT INTO t1(title) VALUES('mysql database') +SET GLOBAL debug_dbug = @old_debug; +TRUNCATE TABLE mysql.slow_log; +DROP TABLE t1; +# Case 2: Sync blocks DML(insert) on other tables. +CREATE TABLE t1 ( +FTS_DOC_ID BIGINT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY, +title VARCHAR(200), +FULLTEXT(title) +) ENGINE = InnoDB; +CREATE TABLE t2(id INT); +connection con1; +SET GLOBAL debug_dbug='+d,fts_instrument_sync_request,fts_instrument_sync_sleep'; +SET DEBUG_SYNC= 'fts_instrument_sync_request SIGNAL begin WAIT_FOR continue'; +INSERT INTO t1(title) VALUES('mysql database'); +connection con2; +SET DEBUG_SYNC= 'now WAIT_FOR begin'; +INSERT INTO t2 VALUES(1); +connection default; +SET DEBUG_SYNC= 'now SIGNAL continue'; +connection con1; +/* connection con1 */ INSERT INTO t1(title) VALUES('mysql database'); +connection con2; +/* conneciton con2 */ INSERT INTO t2 VALUES(1); +connection default; +SET DEBUG_SYNC = 'RESET'; +# make con1 & con2 show up in mysql.slow_log +SELECT SLEEP(2); +SLEEP(2) +0 +# slow log results should be empty here. +SELECT sql_text FROM mysql.slow_log WHERE query_time >= '00:00:02'; +sql_text +SET GLOBAL debug_dbug = @old_debug; +TRUNCATE TABLE mysql.slow_log; +DROP TABLE t1,t2; +disconnect con1; +disconnect con2; +# Restore slow log settings. +SET GLOBAL log_output = @old_log_output; +SET GLOBAL general_log = @old_general_log; +SET GLOBAL slow_query_log = @old_slow_query_log; +SET GLOBAL long_query_time = @old_long_query_time; diff --git a/mysql-test/suite/innodb_fts/t/concurrent_insert.test b/mysql-test/suite/innodb_fts/t/concurrent_insert.test index b6991f6e503..9b4d9517b1a 100644 --- a/mysql-test/suite/innodb_fts/t/concurrent_insert.test +++ b/mysql-test/suite/innodb_fts/t/concurrent_insert.test @@ -31,7 +31,7 @@ INSERT INTO t2 VALUES('mariadb'); connection default; SET @saved_dbug = @@GLOBAL.debug_dbug; SET GLOBAL debug_dbug ='+d,fts_instrument_sync_request,ib_optimize_wq_hang'; -SET DEBUG_SYNC= 'fts_sync_end +SET DEBUG_SYNC= 'fts_instrument_sync_request SIGNAL drop_index_start WAIT_FOR sync_op'; send INSERT INTO t1 VALUES('Keyword'); diff --git a/mysql-test/suite/innodb_fts/t/sync.test b/mysql-test/suite/innodb_fts/t/sync.test index 7c5c835f2ee..168309a5c92 100644 --- a/mysql-test/suite/innodb_fts/t/sync.test +++ b/mysql-test/suite/innodb_fts/t/sync.test @@ -27,7 +27,7 @@ connection con1; SET @old_dbug = @@SESSION.debug_dbug; SET debug_dbug = '+d,fts_instrument_sync_debug'; -SET DEBUG_SYNC= 'fts_sync_end SIGNAL written WAIT_FOR selected'; +SET DEBUG_SYNC= 'fts_write_node SIGNAL written WAIT_FOR selected'; send INSERT INTO t1(title) VALUES('mysql database'); @@ -74,7 +74,7 @@ connection con1; SET debug_dbug = '+d,fts_instrument_sync_debug'; -SET DEBUG_SYNC= 'fts_sync_end SIGNAL written WAIT_FOR inserted'; +SET DEBUG_SYNC= 'fts_write_node SIGNAL written WAIT_FOR inserted'; send INSERT INTO t1(title) VALUES('mysql database'); diff --git a/mysql-test/suite/innodb_fts/t/sync_block.test b/mysql-test/suite/innodb_fts/t/sync_block.test new file mode 100644 index 00000000000..895d2ba8a59 --- /dev/null +++ b/mysql-test/suite/innodb_fts/t/sync_block.test @@ -0,0 +1,124 @@ +# +# BUG#22516559 MYSQL INSTANCE STALLS WHEN SYNCING FTS INDEX +# + +--source include/have_innodb.inc +--source include/have_debug.inc +--source include/have_debug_sync.inc +--source include/have_log_bin.inc +--source include/count_sessions.inc + +SET @old_log_output = @@global.log_output; +SET @old_slow_query_log = @@global.slow_query_log; +SET @old_general_log = @@global.general_log; +SET @old_long_query_time = @@global.long_query_time; +SET @old_debug = @@global.debug_dbug; + +SET GLOBAL log_output = 'TABLE'; +SET GLOBAL general_log = 1; +SET GLOBAL slow_query_log = 1; +SET GLOBAL long_query_time = 1; + +connect (con1,localhost,root,,); +connect (con2,localhost,root,,); +connection default; + +--echo # Case 1: Sync blocks DML(insert) on the same table. +CREATE TABLE t1 ( + FTS_DOC_ID BIGINT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY, + title VARCHAR(200), + FULLTEXT(title) +) ENGINE = InnoDB; + +connection con1; + +SET GLOBAL debug_dbug='+d,fts_instrument_sync_debug,fts_instrument_sync_sleep'; + +SET DEBUG_SYNC= 'fts_sync_begin SIGNAL begin WAIT_FOR continue'; + +send INSERT INTO t1(title) VALUES('mysql database'); + +connection con2; + +SET DEBUG_SYNC= 'now WAIT_FOR begin'; + +send SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); + +connection default; +SET DEBUG_SYNC= 'now SIGNAL continue'; + +connection con1; +--echo /* connection con1 */ INSERT INTO t1(title) VALUES('mysql database'); +--reap + +connection con2; +--echo /* conneciton con2 */ SELECT * FROM t1 WHERE MATCH(title) AGAINST('mysql database'); +--reap + +connection default; +-- echo # make con1 & con2 show up in mysql.slow_log +SELECT SLEEP(2); +-- echo # slow log results should only contain INSERT INTO t1. +SELECT sql_text FROM mysql.slow_log WHERE query_time >= '00:00:02'; + +SET GLOBAL debug_dbug = @old_debug; +TRUNCATE TABLE mysql.slow_log; + +DROP TABLE t1; + +--echo # Case 2: Sync blocks DML(insert) on other tables. +CREATE TABLE t1 ( + FTS_DOC_ID BIGINT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY, + title VARCHAR(200), + FULLTEXT(title) +) ENGINE = InnoDB; + +CREATE TABLE t2(id INT); + +connection con1; + +SET GLOBAL debug_dbug='+d,fts_instrument_sync_request,fts_instrument_sync_sleep'; + +SET DEBUG_SYNC= 'fts_instrument_sync_request SIGNAL begin WAIT_FOR continue'; + +send INSERT INTO t1(title) VALUES('mysql database'); + +connection con2; + +SET DEBUG_SYNC= 'now WAIT_FOR begin'; + +send INSERT INTO t2 VALUES(1); + +connection default; +SET DEBUG_SYNC= 'now SIGNAL continue'; + +connection con1; +--echo /* connection con1 */ INSERT INTO t1(title) VALUES('mysql database'); +--reap + +connection con2; +--echo /* conneciton con2 */ INSERT INTO t2 VALUES(1); +--reap + +connection default; +SET DEBUG_SYNC = 'RESET'; +-- echo # make con1 & con2 show up in mysql.slow_log +SELECT SLEEP(2); +-- echo # slow log results should be empty here. +SELECT sql_text FROM mysql.slow_log WHERE query_time >= '00:00:02'; + +SET GLOBAL debug_dbug = @old_debug; +TRUNCATE TABLE mysql.slow_log; + +DROP TABLE t1,t2; + +disconnect con1; +disconnect con2; + +--source include/wait_until_count_sessions.inc + +-- echo # Restore slow log settings. +SET GLOBAL log_output = @old_log_output; +SET GLOBAL general_log = @old_general_log; +SET GLOBAL slow_query_log = @old_slow_query_log; +SET GLOBAL long_query_time = @old_long_query_time; diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index eed2eb72cd1..f94aed58d21 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -38,22 +38,6 @@ Full Text Search interface #include "dict0stats.h" #include "btr0pcur.h" -/** The SYNC state of the cache. There is one instance of this struct -associated with each ADD thread. */ -struct fts_sync_t { - /** Transaction used for SYNCing the cache to disk */ - trx_t *trx; - /** Table with FTS index(es) */ - dict_table_t *table; - /** Max size in bytes of the cache */ - ulint max_cache_size; - /** The doc id at which the cache was noted as being - full, we use this to set the upper_limit field */ - doc_id_t max_doc_id; - /** SYNC start time; only used if fts_enable_diag_print */ - time_t start_time; -}; - static const ulint FTS_MAX_ID_LEN = 32; /** Column name from the FTS config table */ @@ -201,8 +185,15 @@ struct fts_tokenize_param_t { /** Run SYNC on the table, i.e., write out data from the cache to the FTS auxiliary INDEX table and clear the cache at the end. @param[in,out] sync sync state +@param[in] unlock_cache whether unlock cache lock when write node +@param[in] wait whether wait when a sync is in progress @return DB_SUCCESS if all OK */ -static dberr_t fts_sync(fts_sync_t *sync); +static +dberr_t +fts_sync( + fts_sync_t* sync, + bool unlock_cache, + bool wait); /****************************************************************//** Release all resources help by the words rb tree e.g., the node ilist. */ @@ -275,6 +266,7 @@ fts_cache_destroy(fts_cache_t* cache) mysql_mutex_destroy(&cache->init_lock); mysql_mutex_destroy(&cache->deleted_lock); mysql_mutex_destroy(&cache->doc_id_lock); + pthread_cond_destroy(&cache->sync->cond); if (cache->stopword_info.cached_stopword) { rbt_free(cache->stopword_info.cached_stopword); @@ -574,6 +566,7 @@ fts_index_cache_init( for (i = 0; i < FTS_NUM_AUX_INDEX; ++i) { ut_a(index_cache->ins_graph[i] == NULL); + ut_a(index_cache->sel_graph[i] == NULL); } } @@ -643,6 +636,7 @@ fts_cache_create( mem_heap_zalloc(heap, sizeof(fts_sync_t))); cache->sync->table = table; + pthread_cond_init(&cache->sync->cond, nullptr); /* Create the index cache vector that will hold the inverted indexes. */ cache->indexes = ib_vector_create( @@ -968,6 +962,10 @@ fts_cache_index_cache_create( mem_heap_zalloc(static_cast( cache->self_heap->arg), n_bytes)); + index_cache->sel_graph = static_cast( + mem_heap_zalloc(static_cast( + cache->self_heap->arg), n_bytes)); + fts_index_cache_init(cache->sync_heap, index_cache); if (cache->get_docs) { @@ -1041,6 +1039,13 @@ fts_cache_clear( index_cache->ins_graph[j] = NULL; } + + if (index_cache->sel_graph[j] != NULL) { + + que_graph_free(index_cache->sel_graph[j]); + + index_cache->sel_graph[j] = NULL; + } } index_cache->doc_stats = NULL; @@ -1333,7 +1338,8 @@ fts_cache_add_doc( ib_vector_last(word->nodes)); } - if (!fts_node || fts_node->ilist_size > FTS_ILIST_MAX_SIZE + if (fts_node == NULL || fts_node->synced + || fts_node->ilist_size > FTS_ILIST_MAX_SIZE || doc_id < fts_node->last_doc_id) { fts_node = static_cast( @@ -3320,7 +3326,7 @@ fts_add_doc_from_tuple( if (cache->total_size > fts_max_cache_size / 5 || fts_need_sync) { - fts_sync(cache->sync); + fts_sync(cache->sync, true, false); } mtr_start(&mtr); @@ -3356,7 +3362,7 @@ fts_add_doc_by_id( dict_index_t* fts_id_index; ibool is_id_cluster; fts_cache_t* cache = ftt->table->fts->cache; - bool need_sync= false; + ut_ad(cache->get_docs); /* If Doc ID has been supplied by the user, then the table @@ -3496,32 +3502,44 @@ fts_add_doc_by_id( get_doc->index_cache, doc_id, doc.tokens); - /** FTS cache sync should happen - frequently. Because user thread - shouldn't hold the cache lock for - longer time. So cache should sync - whenever cache size exceeds 512 KB */ - need_sync = - cache->total_size > 512*1024; + bool need_sync = !cache->sync->in_progress + && (fts_need_sync + || (cache->total_size + - cache->total_size_at_sync) + > fts_max_cache_size / 10); + if (need_sync) { + cache->total_size_at_sync = + cache->total_size; + } mysql_mutex_unlock(&table->fts->cache->lock); DBUG_EXECUTE_IF( "fts_instrument_sync", - fts_sync_table(table); + fts_optimize_request_sync_table(table); + mysql_mutex_lock(&cache->lock); + if (cache->sync->in_progress) + my_cond_wait( + &cache->sync->cond, + &cache->lock.m_mutex); + mysql_mutex_unlock(&cache->lock); ); DBUG_EXECUTE_IF( "fts_instrument_sync_debug", - fts_sync(cache->sync); + fts_sync(cache->sync, true, true); ); DEBUG_SYNC_C("fts_instrument_sync_request"); DBUG_EXECUTE_IF( "fts_instrument_sync_request", - need_sync= true; + fts_optimize_request_sync_table(table); ); + if (need_sync) { + fts_optimize_request_sync_table(table); + } + mtr_start(&mtr); if (i < num_idx - 1) { @@ -3547,10 +3565,6 @@ func_exit: ut_free(pcur.old_rec_buf); mem_heap_free(heap); - - if (need_sync) { - fts_sync_table(table); - } } @@ -3910,13 +3924,15 @@ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t fts_sync_write_words( trx_t* trx, - fts_index_cache_t* index_cache) + fts_index_cache_t* index_cache, + bool unlock_cache) { fts_table_t fts_table; ulint n_nodes = 0; ulint n_words = 0; const ib_rbt_node_t* rbt_node; dberr_t error = DB_SUCCESS; + ibool print_error = FALSE; dict_table_t* table = index_cache->index->table; FTS_INIT_INDEX_TABLE( @@ -3947,36 +3963,53 @@ fts_sync_write_words( fts_table.suffix = fts_get_suffix(selected); + /* We iterate over all the nodes even if there was an error */ for (i = 0; i < ib_vector_size(word->nodes); ++i) { fts_node_t* fts_node = static_cast( ib_vector_get(word->nodes, i)); - error = fts_write_node( - trx, &index_cache->ins_graph[selected], - &fts_table, &word->text, fts_node); + if (fts_node->synced) { + continue; + } else { + fts_node->synced = true; + } + + /*FIXME: we need to handle the error properly. */ + if (error == DB_SUCCESS) { + if (unlock_cache) { + mysql_mutex_unlock( + &table->fts->cache->lock); + } + + error = fts_write_node( + trx, + &index_cache->ins_graph[selected], + &fts_table, &word->text, fts_node); - DEBUG_SYNC_C("fts_write_node"); - DBUG_EXECUTE_IF("fts_write_node_crash", + DEBUG_SYNC_C("fts_write_node"); + DBUG_EXECUTE_IF("fts_write_node_crash", DBUG_SUICIDE();); - DBUG_EXECUTE_IF("fts_instrument_sync_sleep", + DBUG_EXECUTE_IF( + "fts_instrument_sync_sleep", std::this_thread::sleep_for( std::chrono::seconds(1));); - if (error != DB_SUCCESS) { - goto err_exit; + if (unlock_cache) { + mysql_mutex_lock( + &table->fts->cache->lock); + } } } n_nodes += ib_vector_size(word->nodes); - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { -err_exit: + if (UNIV_UNLIKELY(error != DB_SUCCESS) && !print_error) { ib::error() << "(" << error << ") writing" " word node to FTS auxiliary index table " << table->name; - break; + print_error = TRUE; } } @@ -4035,44 +4068,58 @@ fts_sync_index( ut_ad(rbt_validate(index_cache->words)); - return(fts_sync_write_words(trx, index_cache)); + return(fts_sync_write_words(trx, index_cache, sync->unlock_cache)); } -/** Rollback a sync operation -@param[in,out] sync sync state */ +/** Check if index cache has been synced completely +@param[in,out] index_cache index cache +@return true if index is synced, otherwise false. */ static -void -fts_sync_rollback( - fts_sync_t* sync) +bool +fts_sync_index_check( + fts_index_cache_t* index_cache) { - trx_t* trx = sync->trx; - fts_cache_t* cache = sync->table->fts->cache; - - for (ulint i = 0; i < ib_vector_size(cache->indexes); ++i) { - ulint j; - fts_index_cache_t* index_cache; - - index_cache = static_cast( - ib_vector_get(cache->indexes, i)); + const ib_rbt_node_t* rbt_node; - for (j = 0; fts_index_selector[j].value; ++j) { + for (rbt_node = rbt_first(index_cache->words); + rbt_node != NULL; + rbt_node = rbt_next(index_cache->words, rbt_node)) { - if (index_cache->ins_graph[j] != NULL) { + fts_tokenizer_word_t* word; + word = rbt_value(fts_tokenizer_word_t, rbt_node); - que_graph_free(index_cache->ins_graph[j]); + fts_node_t* fts_node; + fts_node = static_cast(ib_vector_last(word->nodes)); - index_cache->ins_graph[j] = NULL; - } + if (!fts_node->synced) { + return(false); } } - mysql_mutex_unlock(&cache->lock); + return(true); +} - fts_sql_rollback(trx); +/** Reset synced flag in index cache when rollback +@param[in,out] index_cache index cache */ +static +void +fts_sync_index_reset( + fts_index_cache_t* index_cache) +{ + const ib_rbt_node_t* rbt_node; - /* Avoid assertion in trx_t::free(). */ - trx->dict_operation_lock_mode = false; - trx->free(); + for (rbt_node = rbt_first(index_cache->words); + rbt_node != NULL; + rbt_node = rbt_next(index_cache->words, rbt_node)) { + + fts_tokenizer_word_t* word; + word = rbt_value(fts_tokenizer_word_t, rbt_node); + + fts_node_t* fts_node; + fts_node = static_cast(ib_vector_last(word->nodes)); + + fts_node->synced = false; + } } /** Commit the SYNC, change state of processed doc ids etc. @@ -4105,20 +4152,19 @@ fts_sync_commit( sync, cache->deleted_doc_ids); } + /* We need to do this within the deleted lock since fts_delete() can + attempt to add a deleted doc id to the cache deleted id array. */ + fts_cache_clear(cache); + DEBUG_SYNC_C("fts_deleted_doc_ids_clear"); + fts_cache_init(cache); + mysql_mutex_unlock(&cache->lock); + if (UNIV_LIKELY(error == DB_SUCCESS)) { - /* We need to do this within the deleted lock - since fts_delete() can attempt to add a deleted - doc id to the cache deleted id array. */ - fts_cache_clear(cache); - DEBUG_SYNC_C("fts_deleted_doc_ids_clear"); - fts_cache_init(cache); - mysql_mutex_unlock(&cache->lock); fts_sql_commit(trx); } else { + fts_sql_rollback(trx); ib::error() << "(" << error << ") during SYNC of " "table " << sync->table->name; - fts_sync_rollback(sync); - return error; } if (UNIV_UNLIKELY(fts_enable_diag_print) && elapsed_time) { @@ -4138,13 +4184,66 @@ fts_sync_commit( return(error); } +/** Rollback a sync operation +@param[in,out] sync sync state */ +static +void +fts_sync_rollback( + fts_sync_t* sync) +{ + trx_t* trx = sync->trx; + fts_cache_t* cache = sync->table->fts->cache; + + for (ulint i = 0; i < ib_vector_size(cache->indexes); ++i) { + ulint j; + fts_index_cache_t* index_cache; + + index_cache = static_cast( + ib_vector_get(cache->indexes, i)); + + /* Reset synced flag so nodes will not be skipped + in the next sync, see fts_sync_write_words(). */ + fts_sync_index_reset(index_cache); + + for (j = 0; fts_index_selector[j].value; ++j) { + + if (index_cache->ins_graph[j] != NULL) { + + que_graph_free(index_cache->ins_graph[j]); + + index_cache->ins_graph[j] = NULL; + } + + if (index_cache->sel_graph[j] != NULL) { + + que_graph_free(index_cache->sel_graph[j]); + + index_cache->sel_graph[j] = NULL; + } + } + } + + mysql_mutex_unlock(&cache->lock); + + fts_sql_rollback(trx); + + /* Avoid assertion in trx_t::free(). */ + trx->dict_operation_lock_mode = false; + trx->free(); +} + /** Run SYNC on the table, i.e., write out data from the cache to the FTS auxiliary INDEX table and clear the cache at the end. @param[in,out] sync sync state @param[in] unlock_cache whether unlock cache lock when write node @param[in] wait whether wait when a sync is in progress @return DB_SUCCESS if all OK */ -static dberr_t fts_sync(fts_sync_t *sync) +static +dberr_t +fts_sync( + fts_sync_t* sync, + bool unlock_cache, + bool wait) { if (srv_read_only_mode) { return DB_READ_ONLY; @@ -4155,13 +4254,33 @@ static dberr_t fts_sync(fts_sync_t *sync) fts_cache_t* cache = sync->table->fts->cache; mysql_mutex_lock(&cache->lock); + + /* Check if cache is being synced. + Note: we release cache lock in fts_sync_write_words() to + avoid long wait for the lock by other threads. */ + if (sync->in_progress) { + if (!wait) { + mysql_mutex_unlock(&cache->lock); + return(DB_SUCCESS); + } + do { + my_cond_wait(&sync->cond, &cache->lock.m_mutex); + } while (sync->in_progress); + } + + sync->unlock_cache = unlock_cache; + sync->in_progress = true; + DEBUG_SYNC_C("fts_sync_begin"); fts_sync_begin(sync); +begin_sync: const size_t fts_cache_size= fts_max_cache_size; if (cache->total_size > fts_cache_size) { /* Avoid the case: sync never finish when insert/update keeps comming. */ + ut_ad(sync->unlock_cache); + sync->unlock_cache = false; ib::warn() << "Total InnoDB FTS size " << cache->total_size << " for the table " << cache->sync->table->name @@ -4185,23 +4304,52 @@ static dberr_t fts_sync(fts_sync_t *sync) error = fts_sync_index(sync, index_cache); if (error != DB_SUCCESS) { - goto err_exit; + goto end_sync; + } + + if (!sync->unlock_cache + && cache->total_size < fts_max_cache_size) { + /* Reset the unlock cache if the value + is less than innodb_ft_cache_size */ + sync->unlock_cache = true; } } DBUG_EXECUTE_IF("fts_instrument_sync_interrupted", + sync->interrupted = true; error = DB_INTERRUPTED; - goto err_exit; + goto end_sync; ); - if (error == DB_SUCCESS) { + /* Make sure all the caches are synced. */ + for (i = 0; i < ib_vector_size(cache->indexes); ++i) { + fts_index_cache_t* index_cache; + + index_cache = static_cast( + ib_vector_get(cache->indexes, i)); + + if (index_cache->index->to_be_dropped + || fts_sync_index_check(index_cache)) { + continue; + } + + goto begin_sync; + } + +end_sync: + if (error == DB_SUCCESS && !sync->interrupted) { error = fts_sync_commit(sync); } else { -err_exit: fts_sync_rollback(sync); - return error; } + mysql_mutex_lock(&cache->lock); + ut_ad(sync->in_progress); + sync->interrupted = false; + sync->in_progress = false; + pthread_cond_broadcast(&sync->cond); + mysql_mutex_unlock(&cache->lock); + /* We need to check whether an optimize is required, for that we make copies of the two variables that control the trigger. These variables can change behind our back and we don't want to hold the @@ -4213,7 +4361,6 @@ err_exit: mysql_mutex_unlock(&cache->deleted_lock); - DEBUG_SYNC_C("fts_sync_end"); return(error); } @@ -4222,12 +4369,12 @@ FTS auxiliary INDEX table and clear the cache at the end. @param[in,out] table fts table @param[in] wait whether wait for existing sync to finish @return DB_SUCCESS on success, error code on failure. */ -dberr_t fts_sync_table(dict_table_t* table) +dberr_t fts_sync_table(dict_table_t* table, bool wait) { ut_ad(table->fts); return table->space && !table->corrupted && table->fts->cache - ? fts_sync(table->fts->cache->sync) + ? fts_sync(table->fts->cache->sync, !wait, wait) : DB_SUCCESS; } diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index 7c40a25e6e7..fe31767d901 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -83,8 +83,9 @@ enum fts_msg_type_t { FTS_MSG_ADD_TABLE, /*!< Add table to the optimize thread's work queue */ - FTS_MSG_DEL_TABLE /*!< Remove a table from the optimize + FTS_MSG_DEL_TABLE, /*!< Remove a table from the optimize threads work queue */ + FTS_MSG_SYNC_TABLE /*!< Sync fts cache of a table */ }; /** Compressed list of words that have been read from FTS INDEX @@ -2624,6 +2625,36 @@ fts_optimize_remove_table( mysql_mutex_unlock(&fts_optimize_wq->mutex); } +/** Send sync fts cache for the table. +@param[in] table table to sync */ +void +fts_optimize_request_sync_table( + dict_table_t* table) +{ + /* if the optimize system not yet initialized, return */ + if (!fts_optimize_wq) { + return; + } + + mysql_mutex_lock(&fts_optimize_wq->mutex); + + /* FTS optimizer thread is already exited */ + if (fts_opt_start_shutdown) { + ib::info() << "Try to sync table " << table->name + << " after FTS optimize thread exiting."; + } else if (table->fts->sync_message) { + /* If the table already has SYNC message in + fts_optimize_wq queue then ignore it */ + } else { + add_msg(fts_optimize_create_msg(FTS_MSG_SYNC_TABLE, table)); + table->fts->sync_message = true; + DBUG_EXECUTE_IF("fts_optimize_wq_count_check", + DBUG_ASSERT(fts_optimize_wq->length <= 1000);); + } + + mysql_mutex_unlock(&fts_optimize_wq->mutex); +} + /** Add a table to fts_slots if it doesn't already exist. */ static bool fts_optimize_new_table(dict_table_t* table) { @@ -2765,8 +2796,7 @@ static void fts_optimize_sync_table(dict_table_t *table, if (sync_table->fts && sync_table->fts->cache && sync_table->is_accessible()) { - fts_sync_table(sync_table); - + fts_sync_table(sync_table, false); if (process_message) { mysql_mutex_lock(&fts_optimize_wq->mutex); @@ -2866,6 +2896,24 @@ retry_later: --n_tables; } break; + + case FTS_MSG_SYNC_TABLE: + if (UNIV_UNLIKELY(wsrep_sst_disable_writes)) { + add_msg(msg); + goto retry_later; + } + + DBUG_EXECUTE_IF( + "fts_instrument_msg_sync_sleep", + std::this_thread::sleep_for( + std::chrono::milliseconds( + 300));); + + fts_optimize_sync_table( + static_cast(msg->ptr), + true); + break; + default: ut_error; } @@ -2998,7 +3046,7 @@ void fts_sync_during_ddl(dict_table_t* table) if (!sync_message) return; - fts_sync_table(table); + fts_sync_table(table, false); mysql_mutex_lock(&fts_optimize_wq->mutex); table->fts->sync_message = false; diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 7e4beeede3b..06101e6cca5 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -11584,8 +11584,12 @@ foreign_fail: ut_d(dict_table_check_for_dup_indexes( ctx->new_table, CHECK_ABORTED_OK)); - ut_ad(!ctx->new_table->fts - || fts_check_cached_index(ctx->new_table)); +#ifdef UNIV_DEBUG + if (!(ctx->new_table->fts != NULL + && ctx->new_table->fts->cache->sync->in_progress)) { + ut_a(fts_check_cached_index(ctx->new_table)); + } +#endif } unlock_and_close_files(deleted, trx); diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 720fe7f25b9..c0151b44063 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -648,6 +648,12 @@ fts_optimize_remove_table( void fts_optimize_shutdown(); +/** Send sync fts cache for the table. +@param[in] table table to sync */ +void +fts_optimize_request_sync_table( + dict_table_t* table); + /**********************************************************************//** Take a FTS savepoint. */ void @@ -702,8 +708,9 @@ fts_savepoint_rollback_last_stmt( /** Run SYNC on the table, i.e., write out data from the cache to the FTS auxiliary INDEX table and clear the cache at the end. @param[in,out] table fts table +@param[in] wait whether to wait for existing sync to finish @return DB_SUCCESS on success, error code on failure. */ -dberr_t fts_sync_table(dict_table_t* table); +dberr_t fts_sync_table(dict_table_t* table, bool wait = true); /****************************************************************//** Create an FTS index cache. */ diff --git a/storage/innobase/include/fts0types.h b/storage/innobase/include/fts0types.h index 04e99d595c5..fb278d543c4 100644 --- a/storage/innobase/include/fts0types.h +++ b/storage/innobase/include/fts0types.h @@ -75,6 +75,7 @@ struct fts_index_cache_t { que_t** ins_graph; /*!< Insert query graphs */ + que_t** sel_graph; /*!< Select query graphs */ CHARSET_INFO* charset; /*!< charset */ }; @@ -86,7 +87,35 @@ struct fts_stopword_t { CHARSET_INFO* charset; /*!< charset for stopword */ }; -struct fts_sync_t; +/** The SYNC state of the cache. There is one instance of this struct +associated with each ADD thread. */ +struct fts_sync_t { + trx_t* trx; /*!< The transaction used for SYNCing + the cache to disk */ + dict_table_t* table; /*!< Table with FTS index(es) */ + ulint max_cache_size; /*!< Max size in bytes of the cache */ + ibool cache_full; /*!< flag, when true it indicates that + we need to sync the cache to disk */ + ulint lower_index; /*!< the start index of the doc id + vector from where to start adding + documents to the FTS cache */ + ulint upper_index; /*!< max index of the doc id vector to + add to the FTS cache */ + ibool interrupted; /*!< TRUE if SYNC was interrupted */ + doc_id_t min_doc_id; /*!< The smallest doc id added to the + cache. It should equal to + doc_ids[lower_index] */ + doc_id_t max_doc_id; /*!< The doc id at which the cache was + noted as being full, we use this to + set the upper_limit field */ + time_t start_time; /*!< SYNC start time; only used if + fts_enable_diag_print */ + bool in_progress; /*!< flag whether sync is in progress.*/ + bool unlock_cache; /*!< flag whether unlock cache when + write fts node */ + /** condition variable for in_progress; used with table->fts->cache->lock */ + pthread_cond_t cond; +}; /** The cache for the FTS system. It is a memory-based inverted index that new entries are added to, until it grows over the configured maximum @@ -175,6 +204,7 @@ struct fts_node_t { ulint ilist_size_alloc; /*!< Allocated size of ilist in bytes */ + bool synced; /*!< flag whether the node is synced */ }; /** A tokenizer word. Contains information about one word. */ -- cgit v1.2.1 From d3e394b3b1ff1e2c4e160972aad1f78a13fbb62e Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Mon, 24 Apr 2023 10:27:55 +0400 Subject: A cleanup for MDEV-30968 mariadb-backup does not copy Aria logs if aria_log_dir_path is used Fixing buildbot failures on mariabackup.aria_log_dir_path_rel. The problem was that directory_exists() was called with the relative aria_log_dir_path value, while the current directory in mariadb-backup is not necessarily equal to datadir when MTR is running. Fix: - Moving building the absolute path un level upper: from the function copy_back_aria_logs() to the function copy_back(). - Passing the built absolute path to both directory_exists() and copy_back_aria_logs() as a parameter. --- extra/mariabackup/backup_copy.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index 8ab52fa983b..05acc1e6765 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -1849,10 +1849,8 @@ is_aria_log_dir_file(const datadir_node_t &node) bool -copy_back_aria_logs() +copy_back_aria_logs(const char *dstdir) { - Copy_back_dst_dir dst_dir_buf; - const char *dstdir= dst_dir_buf.make(aria_log_dir_path); std::unique_ptr ds_ctxt_aria_log_dir_path(ds_create(dstdir, DS_TYPE_LOCAL), ds_destroy); @@ -1907,8 +1905,11 @@ copy_back() && !directory_exists(srv_log_group_home_dir, true)) { return(false); } + + Copy_back_dst_dir aria_log_dir_path_dst; + const char *aria_log_dir_path_abs= aria_log_dir_path_dst.make(aria_log_dir_path); if (aria_log_dir_path && *aria_log_dir_path - && !directory_exists(aria_log_dir_path, true)) { + && !directory_exists(aria_log_dir_path_abs, true)) { return false; } @@ -1919,7 +1920,7 @@ copy_back() return(false); } - if (!copy_back_aria_logs()) + if (!copy_back_aria_logs(aria_log_dir_path_abs)) return false; /* parse data file path */ -- cgit v1.2.1 From 0976afec889d8914326f9e71b15ea215470dadba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 24 Apr 2023 09:57:58 +0300 Subject: MDEV-31114 Assertion !...is_waiting() failed in os_aio_wait_until_no_pending_writes() os_aio_wait_until_no_pending_reads(), os_aio_wait_until_pending_writes(): Add a Boolean parameter to indicate whether the wait should be declared in the thread pool. buf_flush_wait(): The callers have already declared a wait, so let us avoid doing that again, just call os_aio_wait_until_pending_writes(false). buf_flush_wait_flushed(): Do not declare a wait in the rare case that the buf_flush_page_cleaner thread has been shut down already. buf_flush_page_cleaner(), buf_flush_buffer_pool(): In the code that runs during shutdown, do not declare waits. buf_flush_buffer_pool(): Remove a debug assertion that might fail. What really matters here is buf_pool.flush_list.count==0. buf_read_recv_pages(), srv_prepare_to_delete_redo_log_file(): Do not declare waits during InnoDB startup. --- storage/innobase/buf/buf0buf.cc | 4 ++-- storage/innobase/buf/buf0dump.cc | 2 +- storage/innobase/buf/buf0flu.cc | 13 ++++++------- storage/innobase/buf/buf0rea.cc | 2 +- storage/innobase/include/os0file.h | 10 ++++++---- storage/innobase/log/log0recv.cc | 2 +- storage/innobase/os/os0file.cc | 18 ++++++++++-------- storage/innobase/row/row0quiesce.cc | 2 +- storage/innobase/srv/srv0start.cc | 2 +- 9 files changed, 29 insertions(+), 26 deletions(-) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 462b1eb634a..693826917c9 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -1317,11 +1317,11 @@ buf_tmp_buffer_t *buf_pool_t::io_buf_t::reserve() for (buf_tmp_buffer_t *s= slots, *e= slots + n_slots; s != e; s++) if (s->acquire()) return s; - os_aio_wait_until_no_pending_writes(); + os_aio_wait_until_no_pending_writes(true); for (buf_tmp_buffer_t *s= slots, *e= slots + n_slots; s != e; s++) if (s->acquire()) return s; - os_aio_wait_until_no_pending_reads(); + os_aio_wait_until_no_pending_reads(true); } } diff --git a/storage/innobase/buf/buf0dump.cc b/storage/innobase/buf/buf0dump.cc index 05b18de1d5b..03876666f1d 100644 --- a/storage/innobase/buf/buf0dump.cc +++ b/storage/innobase/buf/buf0dump.cc @@ -647,7 +647,7 @@ buf_load() ut_free(dump); if (i == dump_n) { - os_aio_wait_until_no_pending_reads(); + os_aio_wait_until_no_pending_reads(true); } ut_sprintf_timestamp(now); diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 75286d9d33b..fff70eefd13 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -246,7 +246,7 @@ void buf_flush_remove_pages(ulint id) if (!deferred) break; - os_aio_wait_until_no_pending_writes(); + os_aio_wait_until_no_pending_writes(true); } } @@ -1692,7 +1692,7 @@ done: space->release(); if (space->purpose == FIL_TYPE_IMPORT) - os_aio_wait_until_no_pending_writes(); + os_aio_wait_until_no_pending_writes(true); else buf_dblwr.flush_buffered_writes(); @@ -1862,7 +1862,7 @@ static void buf_flush_wait(lsn_t lsn) break; } mysql_mutex_unlock(&buf_pool.flush_list_mutex); - os_aio_wait_until_no_pending_writes(); + os_aio_wait_until_no_pending_writes(false); mysql_mutex_lock(&buf_pool.flush_list_mutex); } } @@ -1898,7 +1898,7 @@ ATTRIBUTE_COLD void buf_flush_wait_flushed(lsn_t sync_lsn) MONITOR_FLUSH_SYNC_COUNT, MONITOR_FLUSH_SYNC_PAGES, n_pages); } - os_aio_wait_until_no_pending_writes(); + os_aio_wait_until_no_pending_writes(false); mysql_mutex_lock(&buf_pool.flush_list_mutex); } while (buf_pool.get_oldest_modification(sync_lsn) < sync_lsn); @@ -2421,7 +2421,7 @@ static void buf_flush_page_cleaner() mysql_mutex_lock(&buf_pool.flush_list_mutex); buf_flush_wait_LRU_batch_end(); mysql_mutex_unlock(&buf_pool.flush_list_mutex); - os_aio_wait_until_no_pending_writes(); + os_aio_wait_until_no_pending_writes(false); } mysql_mutex_lock(&buf_pool.flush_list_mutex); @@ -2471,7 +2471,7 @@ ATTRIBUTE_COLD void buf_flush_buffer_pool() { mysql_mutex_unlock(&buf_pool.flush_list_mutex); buf_flush_list(srv_max_io_capacity); - os_aio_wait_until_no_pending_writes(); + os_aio_wait_until_no_pending_writes(false); mysql_mutex_lock(&buf_pool.flush_list_mutex); service_manager_extend_timeout(INNODB_EXTEND_TIMEOUT_INTERVAL, "Waiting to flush " ULINTPF " pages", @@ -2479,7 +2479,6 @@ ATTRIBUTE_COLD void buf_flush_buffer_pool() } mysql_mutex_unlock(&buf_pool.flush_list_mutex); - ut_ad(!os_aio_pending_writes()); ut_ad(!os_aio_pending_reads()); } diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index b8fa3055adf..1fe629ca8a7 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -687,7 +687,7 @@ void buf_read_recv_pages(ulint space_id, const uint32_t* page_nos, ulint n) } if (os_aio_pending_reads() >= limit) { - os_aio_wait_until_no_pending_reads(); + os_aio_wait_until_no_pending_reads(false); } space->reacquire(); diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h index f8ae0f51557..7ac0579cc07 100644 --- a/storage/innobase/include/os0file.h +++ b/storage/innobase/include/os0file.h @@ -1066,11 +1066,13 @@ size_t os_aio_pending_reads_approx(); /** @return number of pending writes */ size_t os_aio_pending_writes(); -/** Wait until there are no pending asynchronous writes. */ -void os_aio_wait_until_no_pending_writes(); +/** Wait until there are no pending asynchronous writes. +@param declare whether the wait will be declared in tpool */ +void os_aio_wait_until_no_pending_writes(bool declare); -/** Wait until all pending asynchronous reads have completed. */ -void os_aio_wait_until_no_pending_reads(); +/** Wait until all pending asynchronous reads have completed. +@param declare whether the wait will be declared in tpool */ +void os_aio_wait_until_no_pending_reads(bool declare); /** Prints info of the aio arrays. @param[in/out] file file where to print */ diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index f87ff15c393..78ba8b70a49 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -3407,7 +3407,7 @@ next_free_block: else { mysql_mutex_unlock(&mutex); - os_aio_wait_until_no_pending_reads(); + os_aio_wait_until_no_pending_reads(false); mysql_mutex_lock(&mutex); ut_ad(pages.empty()); } diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index e816f6ef7b1..3b81dc7ee07 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -3644,9 +3644,9 @@ void os_aio_free() } /** Wait until there are no pending asynchronous writes. */ -static void os_aio_wait_until_no_pending_writes_low() +static void os_aio_wait_until_no_pending_writes_low(bool declare) { - bool notify_wait = write_slots->pending_io_count() > 0; + const bool notify_wait= declare && write_slots->pending_io_count(); if (notify_wait) tpool::tpool_wait_begin(); @@ -3657,10 +3657,11 @@ static void os_aio_wait_until_no_pending_writes_low() tpool::tpool_wait_end(); } -/** Wait until there are no pending asynchronous writes. */ -void os_aio_wait_until_no_pending_writes() +/** Wait until there are no pending asynchronous writes. +@param declare whether the wait will be declared in tpool */ +void os_aio_wait_until_no_pending_writes(bool declare) { - os_aio_wait_until_no_pending_writes_low(); + os_aio_wait_until_no_pending_writes_low(declare); buf_dblwr.wait_flush_buffered_writes(); } @@ -3688,10 +3689,11 @@ size_t os_aio_pending_writes() return pending; } -/** Wait until all pending asynchronous reads have completed. */ -void os_aio_wait_until_no_pending_reads() +/** Wait until all pending asynchronous reads have completed. +@param declare whether the wait will be declared in tpool */ +void os_aio_wait_until_no_pending_reads(bool declare) { - const auto notify_wait= read_slots->pending_io_count(); + const bool notify_wait= declare && read_slots->pending_io_count(); if (notify_wait) tpool::tpool_wait_begin(); diff --git a/storage/innobase/row/row0quiesce.cc b/storage/innobase/row/row0quiesce.cc index a4d634f2d14..eadb30bfcfa 100644 --- a/storage/innobase/row/row0quiesce.cc +++ b/storage/innobase/row/row0quiesce.cc @@ -553,7 +553,7 @@ row_quiesce_table_start( if (!trx_is_interrupted(trx)) { /* Ensure that all asynchronous IO is completed. */ - os_aio_wait_until_no_pending_writes(); + os_aio_wait_until_no_pending_writes(true); table->space->flush(); if (row_quiesce_write_cfg(table, trx->mysql_thd) diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 5f6b4b02e16..c111120ea0e 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -973,7 +973,7 @@ same_size: ut_d(mysql_mutex_lock(&buf_pool.flush_list_mutex)); ut_ad(!buf_pool.get_oldest_modification(0)); ut_d(mysql_mutex_unlock(&buf_pool.flush_list_mutex)); - ut_d(os_aio_wait_until_no_pending_writes()); + ut_d(os_aio_wait_until_no_pending_writes(false)); DBUG_RETURN(flushed_lsn); } -- cgit v1.2.1 From 31f09e36c183d026a28f42ddbb9be2229613a3ed Mon Sep 17 00:00:00 2001 From: Brandon Nesterenko Date: Tue, 18 Apr 2023 13:22:43 -0600 Subject: MDEV-31038: Parallel Replication Breaks if XA PREPARE Fails Updating Slave GTID State If a replica failed to update the GTID slave state when committing an XA PREPARE, the replica would retry the transaction and get an out-of-order GTID error. This is because the commit phase of an XA PREPARE is bifurcated. That is, first, the prepare is handled by the relevant storage engines. Then second, the GTID slave state is updated as a separate autocommit transaction. If the second phase fails, and the transaction is retried, then the same transaction is attempted to be committed again, resulting in a GTID out-of-order error. This patch fixes this error by immediately stopping the slave and reporting the appropriate error. That is, there was logic to bypass the error when updating the GTID slave state table if the underlying error is allowed for retry on a parallel slave. This patch adds a parameter to disallow the error bypass, thereby forcing the error state to still happen. Reviewed By ============ Andrei Elkin --- .../suite/rpl/r/rpl_xa_prepare_gtid_fail.result | 51 ++++++++++ .../suite/rpl/t/rpl_xa_prepare_gtid_fail.test | 106 +++++++++++++++++++++ sql/handler.cc | 7 +- sql/log_event.h | 2 +- sql/log_event_server.cc | 64 +++++++++---- 5 files changed, 208 insertions(+), 22 deletions(-) create mode 100644 mysql-test/suite/rpl/r/rpl_xa_prepare_gtid_fail.result create mode 100644 mysql-test/suite/rpl/t/rpl_xa_prepare_gtid_fail.test diff --git a/mysql-test/suite/rpl/r/rpl_xa_prepare_gtid_fail.result b/mysql-test/suite/rpl/r/rpl_xa_prepare_gtid_fail.result new file mode 100644 index 00000000000..f3fecbda349 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_xa_prepare_gtid_fail.result @@ -0,0 +1,51 @@ +include/master-slave.inc +[connection master] +connection slave; +include/stop_slave.inc +change master to master_use_gtid=slave_pos; +set @@global.slave_parallel_threads= 4; +set @@global.slave_parallel_mode= optimistic; +set @@global.gtid_strict_mode=ON; +set sql_log_bin= 0; +alter table mysql.gtid_slave_pos engine=innodb; +call mtr.add_suppression("Deadlock found.*"); +set sql_log_bin= 1; +include/start_slave.inc +connection master; +create table t1 (a int primary key, b int) engine=innodb; +insert t1 values (1,1); +include/save_master_gtid.inc +connection slave; +include/sync_with_master_gtid.inc +include/stop_slave.inc +set @@global.innodb_lock_wait_timeout= 1; +connection master; +set @@session.gtid_seq_no=100; +xa start '1'; +update t1 set b=b+10 where a=1; +xa end '1'; +xa prepare '1'; +xa commit '1'; +include/save_master_gtid.inc +connection slave; +connection slave1; +BEGIN; +SELECT * FROM mysql.gtid_slave_pos WHERE seq_no=100 FOR UPDATE; +domain_id sub_id server_id seq_no +connection slave; +include/start_slave.inc +include/wait_for_slave_sql_error.inc [errno=1942,1213] +connection slave1; +ROLLBACK; +# Cleanup +connection master; +drop table t1; +connection slave; +include/stop_slave.inc +set @@global.gtid_slave_pos= "0-1-100"; +set @@global.slave_parallel_threads= 0; +set @@global.gtid_strict_mode= 0; +set @@global.innodb_lock_wait_timeout= 50; +include/start_slave.inc +include/rpl_end.inc +# End of rpl_xa_prepare_gtid_fail.test diff --git a/mysql-test/suite/rpl/t/rpl_xa_prepare_gtid_fail.test b/mysql-test/suite/rpl/t/rpl_xa_prepare_gtid_fail.test new file mode 100644 index 00000000000..8042b355754 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_xa_prepare_gtid_fail.test @@ -0,0 +1,106 @@ +# +# When handling the replication of an XA PREPARE, the commit phase is +# bifurcated. First, the prepare is handled by the relevant storage engines. +# Then second,the GTID slave state is updated as a separate autocommit +# transaction. If the second stage fails, i.e. we are unable to update the +# GTID slave state, then the slave should immediately quit in error, without +# retry. +# +# This tests validates the above behavior by simulating a deadlock on the +# GTID slave state table during the second part of XA PREPARE's commit, to +# ensure that the appropriate error is reported and the transaction was never +# retried. +# +# +# References +# MDEV-31038: Parallel Replication Breaks if XA PREPARE Fails Updating Slave +# GTID State +# +source include/master-slave.inc; +source include/have_binlog_format_row.inc; +source include/have_innodb.inc; + +--connection slave +--source include/stop_slave.inc + +--let $save_par_thds= `SELECT @@global.slave_parallel_threads` +--let $save_strict_mode= `SELECT @@global.gtid_strict_mode` +--let $save_innodb_lock_wait_timeout= `SELECT @@global.innodb_lock_wait_timeout` + +change master to master_use_gtid=slave_pos; +set @@global.slave_parallel_threads= 4; +set @@global.slave_parallel_mode= optimistic; +set @@global.gtid_strict_mode=ON; + +set sql_log_bin= 0; +alter table mysql.gtid_slave_pos engine=innodb; +call mtr.add_suppression("Deadlock found.*"); +set sql_log_bin= 1; +--source include/start_slave.inc + +--connection master +let $datadir= `select @@datadir`; +create table t1 (a int primary key, b int) engine=innodb; +insert t1 values (1,1); +--source include/save_master_gtid.inc + +--connection slave +--source include/sync_with_master_gtid.inc +--source include/stop_slave.inc +set @@global.innodb_lock_wait_timeout= 1; + +--let $retried_tx_initial= query_get_value(SHOW ALL SLAVES STATUS, Retried_transactions, 1) + +--connection master +--let $gtid_domain_id=`SELECT @@GLOBAL.gtid_domain_id` +--let $gtid_server_id=`SELECT @@GLOBAL.server_id` +--let $xap_seq_no=100 +--eval set @@session.gtid_seq_no=$xap_seq_no +xa start '1'; +update t1 set b=b+10 where a=1; +xa end '1'; +xa prepare '1'; +--let $new_gtid= `SELECT @@global.gtid_binlog_pos` +xa commit '1'; +--source include/save_master_gtid.inc + + +--connection slave + +#--eval set statement sql_log_bin=0 for insert into mysql.gtid_slave_pos values ($gtid_domain_id, 5, $gtid_server_id, $xap_seq_no) + +--connection slave1 +BEGIN; +--eval SELECT * FROM mysql.gtid_slave_pos WHERE seq_no=$xap_seq_no FOR UPDATE + +--connection slave +--source include/start_slave.inc + +--let $slave_sql_errno= 1942,1213 +--source include/wait_for_slave_sql_error.inc + +--let $retried_tx_test= query_get_value(SHOW ALL SLAVES STATUS, Retried_transactions, 1) +if ($retried_tx_initial != $retried_tx_test) +{ + --echo Transaction was retried when a failed XA PREPARE slave GTID update should lead to immediate slave stop without retry + --die Transaction was retried when a failed XA PREPARE slave GTID update should lead to immediate slave stop without retry +} + +--connection slave1 +ROLLBACK; + +--echo # Cleanup + +--connection master +drop table t1; + +--connection slave +--source include/stop_slave.inc +--eval set @@global.gtid_slave_pos= "$new_gtid" +--eval set @@global.slave_parallel_threads= $save_par_thds +--eval set @@global.gtid_strict_mode= $save_strict_mode +--eval set @@global.innodb_lock_wait_timeout= $save_innodb_lock_wait_timeout +--source include/start_slave.inc + +--source include/rpl_end.inc +--echo # End of rpl_xa_prepare_gtid_fail.test diff --git a/sql/handler.cc b/sql/handler.cc index 926ef2f4f54..eaaf4664c07 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -2057,8 +2057,11 @@ int ha_rollback_trans(THD *thd, bool all) rollback without signalling following transactions. And in release builds, we explicitly do the signalling before rolling back. */ - DBUG_ASSERT(!(thd->rgi_slave && thd->rgi_slave->did_mark_start_commit) || - thd->transaction->xid_state.is_explicit_XA()); + DBUG_ASSERT( + !(thd->rgi_slave && thd->rgi_slave->did_mark_start_commit) || + (thd->transaction->xid_state.is_explicit_XA() || + (thd->rgi_slave->gtid_ev_flags2 & Gtid_log_event::FL_PREPARED_XA))); + if (thd->rgi_slave && thd->rgi_slave->did_mark_start_commit) thd->rgi_slave->unmark_start_commit(); } diff --git a/sql/log_event.h b/sql/log_event.h index 5dcf0315f68..2fcc4549bc9 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -3042,7 +3042,7 @@ private: virtual int do_commit()= 0; virtual int do_apply_event(rpl_group_info *rgi); int do_record_gtid(THD *thd, rpl_group_info *rgi, bool in_trans, - void **out_hton); + void **out_hton, bool force_err= false); enum_skip_reason do_shall_skip(rpl_group_info *rgi); virtual const char* get_query()= 0; #endif diff --git a/sql/log_event_server.cc b/sql/log_event_server.cc index c5fb637a000..a2b78bc241d 100644 --- a/sql/log_event_server.cc +++ b/sql/log_event_server.cc @@ -152,6 +152,30 @@ is_parallel_retry_error(rpl_group_info *rgi, int err) return has_temporary_error(rgi->thd); } +/** + Accumulate a Diagnostics_area's errors and warnings into an output buffer + + @param errbuf The output buffer to write error messages + @param errbuf_size The size of the output buffer + @param da The Diagnostics_area to check for errors +*/ +static void inline aggregate_da_errors(char *errbuf, size_t errbuf_size, + Diagnostics_area *da) +{ + const char *errbuf_end= errbuf + errbuf_size; + char *slider; + Diagnostics_area::Sql_condition_iterator it= da->sql_conditions(); + const Sql_condition *err; + size_t len; + for (err= it++, slider= errbuf; err && slider < errbuf_end - 1; + slider += len, err= it++) + { + len= my_snprintf(slider, errbuf_end - slider, + " %s, Error_code: %d;", err->get_message_text(), + err->get_sql_errno()); + } +} + /** Error reporting facility for Rows_log_event::do_apply_event @@ -172,13 +196,8 @@ static void inline slave_rows_error_report(enum loglevel level, int ha_error, const char *log_name, my_off_t pos) { const char *handler_error= (ha_error ? HA_ERR(ha_error) : NULL); - char buff[MAX_SLAVE_ERRMSG], *slider; - const char *buff_end= buff + sizeof(buff); - size_t len; - Diagnostics_area::Sql_condition_iterator it= - thd->get_stmt_da()->sql_conditions(); + char buff[MAX_SLAVE_ERRMSG]; Relay_log_info const *rli= rgi->rli; - const Sql_condition *err; buff[0]= 0; int errcode= thd->is_error() ? thd->get_stmt_da()->sql_errno() : 0; @@ -191,13 +210,7 @@ static void inline slave_rows_error_report(enum loglevel level, int ha_error, if (is_parallel_retry_error(rgi, errcode)) return; - for (err= it++, slider= buff; err && slider < buff_end - 1; - slider += len, err= it++) - { - len= my_snprintf(slider, buff_end - slider, - " %s, Error_code: %d;", err->get_message_text(), - err->get_sql_errno()); - } + aggregate_da_errors(buff, sizeof(buff), thd->get_stmt_da()); if (ha_error != 0) rli->report(level, errcode, rgi->gtid_info(), @@ -3893,7 +3906,8 @@ bool slave_execute_deferred_events(THD *thd) #if defined(HAVE_REPLICATION) int Xid_apply_log_event::do_record_gtid(THD *thd, rpl_group_info *rgi, - bool in_trans, void **out_hton) + bool in_trans, void **out_hton, + bool force_err) { int err= 0; Relay_log_info const *rli= rgi->rli; @@ -3908,14 +3922,26 @@ int Xid_apply_log_event::do_record_gtid(THD *thd, rpl_group_info *rgi, int ec= thd->get_stmt_da()->sql_errno(); /* Do not report an error if this is really a kill due to a deadlock. - In this case, the transaction will be re-tried instead. + In this case, the transaction will be re-tried instead. Unless force_err + is set, as in the case of XA PREPARE, as the GTID state is updated as a + separate transaction, and if that fails, we should not retry but exit in + error immediately. */ - if (!is_parallel_retry_error(rgi, ec)) + if (!is_parallel_retry_error(rgi, ec) || force_err) + { + char buff[MAX_SLAVE_ERRMSG]; + buff[0]= 0; + aggregate_da_errors(buff, sizeof(buff), thd->get_stmt_da()); + + if (force_err) + thd->clear_error(); + rli->report(ERROR_LEVEL, ER_CANNOT_UPDATE_GTID_STATE, rgi->gtid_info(), "Error during XID COMMIT: failed to update GTID state in " - "%s.%s: %d: %s", + "%s.%s: %d: %s the event's master log %s, end_log_pos %llu", "mysql", rpl_gtid_slave_state_table_name.str, ec, - thd->get_stmt_da()->message()); + buff, RPL_LOG_NAME, log_pos); + } thd->is_slave_error= 1; } @@ -3989,7 +4015,7 @@ int Xid_apply_log_event::do_apply_event(rpl_group_info *rgi) { DBUG_ASSERT(!thd->transaction->xid_state.is_explicit_XA()); - if ((err= do_record_gtid(thd, rgi, false, &hton))) + if ((err= do_record_gtid(thd, rgi, false, &hton, true))) return err; } -- cgit v1.2.1 From 50f3b7d1649002df3c73ec88827707096ce3135c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 25 Apr 2023 12:17:06 +0300 Subject: MDEV-31124 Innodb_data_written miscounts doublewrites When commit a5a2ef079cec378340d8b575aef05974b0b3442e implemented asynchronous doublewrite, the writes via the doublewrite buffer started to be counted incorrectly, without multiplying them by innodb_page_size. srv_export_innodb_status(): Correctly count the Innodb_data_written. buf_dblwr_t: Remove submitted(), because it is close to written() and only Innodb_data_written was interested in it. According to its name, it should count completed and not submitted writes. Tested by: Axel Schwenke --- storage/innobase/buf/buf0dblwr.cc | 1 - storage/innobase/include/buf0dblwr.h | 5 ----- storage/innobase/srv/srv0srv.cc | 7 ++++--- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 1d582b6cfbf..b6b6e87b6df 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -582,7 +582,6 @@ bool buf_dblwr_t::flush_buffered_writes(const ulint size) const bool multi_batch= block1 + static_cast(size) != block2 && old_first_free > size; flushing_buffered_writes= 1 + multi_batch; - pages_submitted+= old_first_free; /* Now safe to release the mutex. */ mysql_mutex_unlock(&mutex); #ifdef UNIV_DEBUG diff --git a/storage/innobase/include/buf0dblwr.h b/storage/innobase/include/buf0dblwr.h index fb9df55504c..f82baeec89a 100644 --- a/storage/innobase/include/buf0dblwr.h +++ b/storage/innobase/include/buf0dblwr.h @@ -66,8 +66,6 @@ class buf_dblwr_t bool batch_running; /** number of expected flush_buffered_writes_completed() calls */ unsigned flushing_buffered_writes; - /** pages submitted to flush_buffered_writes() */ - ulint pages_submitted; /** number of flush_buffered_writes_completed() calls */ ulint writes_completed; /** number of pages written by flush_buffered_writes_completed() */ @@ -92,9 +90,6 @@ public: /** Acquire the mutex */ void lock() { mysql_mutex_lock(&mutex); } - /** @return the number of submitted page writes */ - ulint submitted() const - { mysql_mutex_assert_owner(&mutex); return pages_submitted; } /** @return the number of completed batches */ ulint batches() const { mysql_mutex_assert_owner(&mutex); return writes_completed; } diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 337460fc4d2..57aa4bef9fe 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -1013,13 +1013,14 @@ srv_export_innodb_status(void) if (buf_dblwr.is_initialised()) { buf_dblwr.lock(); - dblwr = buf_dblwr.submitted(); - export_vars.innodb_dblwr_pages_written = buf_dblwr.written(); + dblwr = buf_dblwr.written(); + export_vars.innodb_dblwr_pages_written = dblwr; export_vars.innodb_dblwr_writes = buf_dblwr.batches(); buf_dblwr.unlock(); } - export_vars.innodb_data_written = srv_stats.data_written + dblwr; + export_vars.innodb_data_written = srv_stats.data_written + + (dblwr << srv_page_size_shift); export_vars.innodb_buffer_pool_read_requests = buf_pool.stat.n_page_gets; -- cgit v1.2.1 From c22ab93f8af5baf2cefe802dfd4a3819225d6df5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 25 Apr 2023 15:03:38 +0300 Subject: MDEV-26827 fixup: Prevent a hang in LRU eviction buf_pool_t::page_cleaner_wakeup(): If for_LRU=true, wake up the page cleaner immediately, also when it is in a timed wait. This avoids an unnecessary delay of up to 1 second. --- storage/innobase/buf/buf0flu.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index fff70eefd13..b201250bc3b 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -115,7 +115,12 @@ static void buf_flush_validate_skip() void buf_pool_t::page_cleaner_wakeup(bool for_LRU) { if (!page_cleaner_idle()) + { + if (for_LRU) + /* Ensure that the page cleaner is not in a timed wait. */ + pthread_cond_signal(&do_flush_list); return; + } double dirty_pct= double(UT_LIST_GET_LEN(buf_pool.flush_list)) * 100.0 / double(UT_LIST_GET_LEN(buf_pool.LRU) + UT_LIST_GET_LEN(buf_pool.free)); double pct_lwm= srv_max_dirty_pages_pct_lwm; -- cgit v1.2.1 From 898320b5f8afd20ba07efa8ea193e18661f2a0a4 Mon Sep 17 00:00:00 2001 From: Julius Goryavsky Date: Tue, 25 Apr 2023 18:55:53 +0200 Subject: MDEV-30804 addendum for 10.6+ branches This addition to MDEV-30804 is relevant for 10.6+, it excludes the mixed transaction section using both innodb and aria storage engines from the galera_var_replicate_aria_off test, since such transactions cannot be executed unless aria supports two-phase transaction commit. No additional tests are required as this commit fixes the mtr test itself. --- mysql-test/suite/galera/r/galera_var_replicate_aria_on.result | 5 ++--- mysql-test/suite/galera/t/galera_var_replicate_aria_on.test | 8 ++++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_var_replicate_aria_on.result b/mysql-test/suite/galera/r/galera_var_replicate_aria_on.result index 39fd748314c..91c07ba6681 100644 --- a/mysql-test/suite/galera/r/galera_var_replicate_aria_on.result +++ b/mysql-test/suite/galera/r/galera_var_replicate_aria_on.result @@ -89,11 +89,8 @@ connection node_1; SET GLOBAL wsrep_sync_wait=15; CREATE TABLE t1 (f1 INTEGER NOT NULL PRIMARY KEY) ENGINE=Aria; CREATE TABLE t2 (f1 INTEGER NOT NULL PRIMARY KEY) ENGINE=InnoDB; -SET AUTOCOMMIT=OFF; -START TRANSACTION; INSERT INTO t1 VALUES (1); INSERT INTO t2 VALUES (1); -COMMIT; connection node_2; SET GLOBAL wsrep_sync_wait=15; SELECT COUNT(*) AS EXPECT_1 FROM t1; @@ -103,6 +100,7 @@ SELECT COUNT(*) AS EXPECT_1 FROM t2; EXPECT_1 1 connection node_1; +SET AUTOCOMMIT=OFF; START TRANSACTION; INSERT INTO t1 VALUES (2); INSERT INTO t2 VALUES (2); @@ -129,6 +127,7 @@ INSERT INTO t1 VALUES (1); ERROR 23000: Duplicate entry '1' for key 'PRIMARY' connection node_1; COMMIT; +ERROR HY000: Transactional commit not supported by involved engine(s) DROP TABLE t1,t2; connection node_1; CREATE TABLE t1 (i INT NOT NULL PRIMARY KEY) ENGINE=INNODB; diff --git a/mysql-test/suite/galera/t/galera_var_replicate_aria_on.test b/mysql-test/suite/galera/t/galera_var_replicate_aria_on.test index c3bc53ee17f..ac9a79e6196 100644 --- a/mysql-test/suite/galera/t/galera_var_replicate_aria_on.test +++ b/mysql-test/suite/galera/t/galera_var_replicate_aria_on.test @@ -85,18 +85,15 @@ SELECT * FROM t1; DROP TABLE t1; # -# Transaction +# Preparation for next tests # --connection node_1 SET GLOBAL wsrep_sync_wait=15; CREATE TABLE t1 (f1 INTEGER NOT NULL PRIMARY KEY) ENGINE=Aria; CREATE TABLE t2 (f1 INTEGER NOT NULL PRIMARY KEY) ENGINE=InnoDB; -SET AUTOCOMMIT=OFF; -START TRANSACTION; INSERT INTO t1 VALUES (1); INSERT INTO t2 VALUES (1); -COMMIT; --connection node_2 SET GLOBAL wsrep_sync_wait=15; @@ -108,6 +105,7 @@ SELECT COUNT(*) AS EXPECT_1 FROM t2; # --connection node_1 +SET AUTOCOMMIT=OFF; START TRANSACTION; INSERT INTO t1 VALUES (2); INSERT INTO t2 VALUES (2); @@ -138,6 +136,8 @@ INSERT INTO t2 VALUES (1); INSERT INTO t1 VALUES (1); --connection node_1 + +--error ER_ERROR_DURING_COMMIT COMMIT; DROP TABLE t1,t2; -- cgit v1.2.1 From d4265fbde587bd79b6fe3793225d3f4798ee955e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 26 Apr 2023 11:53:42 +0300 Subject: MDEV-26055: Correct the formula for adaptive flushing page_cleaner_flush_pages_recommendation(): If dirty_pct is between innodb_max_dirty_pages_pct_lwm and innodb_max_dirty_pages_pct, scale the effort relative to how close we are to innodb_max_dirty_pages_pct. The previous formula was missing a multiplication by 100. Tested by: Axel Schwenke --- storage/innobase/buf/buf0flu.cc | 37 ++++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index b201250bc3b..73fc2957ebb 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -2081,10 +2081,12 @@ Based on various factors it decides if there is a need to do flushing. @return number of pages recommended to be flushed @param last_pages_in number of pages flushed in previous batch @param oldest_lsn buf_pool.get_oldest_modification(0) +@param pct_lwm innodb_max_dirty_pages_pct_lwm, or 0 to ignore it @param dirty_blocks UT_LIST_GET_LEN(buf_pool.flush_list) @param dirty_pct 100*flush_list.count / (LRU.count + free.count) */ static ulint page_cleaner_flush_pages_recommendation(ulint last_pages_in, lsn_t oldest_lsn, + double pct_lwm, ulint dirty_blocks, double dirty_pct) { @@ -2154,11 +2156,17 @@ func_exit: sum_pages = 0; } - const ulint pct_for_dirty = srv_max_dirty_pages_pct_lwm == 0 - ? (dirty_pct >= max_pct ? 100 : 0) - : static_cast - (max_pct > 0.0 ? dirty_pct / max_pct : dirty_pct); - ulint pct_total = std::max(pct_for_dirty, pct_for_lsn); + MONITOR_SET(MONITOR_FLUSH_PCT_FOR_LSN, pct_for_lsn); + + double total_ratio; + if (pct_lwm == 0.0 || max_pct == 0.0) { + total_ratio = 1; + } else { + total_ratio = std::max(double(pct_for_lsn) / 100, + (dirty_pct / max_pct)); + } + + MONITOR_SET(MONITOR_FLUSH_PCT_FOR_DIRTY, ulint(total_ratio * 100)); /* Estimate pages to be flushed for the lsn progress */ lsn_t target_lsn = oldest_lsn @@ -2184,7 +2192,7 @@ func_exit: pages_for_lsn = 1; } - n_pages = (ulint(double(srv_io_capacity) * double(pct_total) / 100.0) + n_pages = (ulint(double(srv_io_capacity) * total_ratio) + avg_page_rate + pages_for_lsn) / 3; if (n_pages > srv_max_io_capacity) { @@ -2197,8 +2205,6 @@ func_exit: MONITOR_SET(MONITOR_FLUSH_AVG_PAGE_RATE, avg_page_rate); MONITOR_SET(MONITOR_FLUSH_LSN_AVG_RATE, lsn_avg_rate); - MONITOR_SET(MONITOR_FLUSH_PCT_FOR_DIRTY, pct_for_dirty); - MONITOR_SET(MONITOR_FLUSH_PCT_FOR_LSN, pct_for_lsn); goto func_exit; } @@ -2305,7 +2311,7 @@ static void buf_flush_page_cleaner() soft_lsn_limit= lsn_limit; } - bool idle_flush= false; + double pct_lwm= 0.0; ulint n_flushed= 0, n; if (UNIV_UNLIKELY(soft_lsn_limit != 0)) @@ -2324,7 +2330,7 @@ static void buf_flush_page_cleaner() mysql_mutex_unlock(&buf_pool.mutex); last_pages+= n; - if (!idle_flush) + if (pct_lwm == 0.0) goto end_of_batch; /* when idle flushing kicks in page_cleaner is marked active. @@ -2342,7 +2348,8 @@ static void buf_flush_page_cleaner() guaranteed to be nonempty, and it is a subset of buf_pool.LRU. */ const double dirty_pct= double(dirty_blocks) * 100.0 / double(UT_LIST_GET_LEN(buf_pool.LRU) + UT_LIST_GET_LEN(buf_pool.free)); - if (srv_max_dirty_pages_pct_lwm != 0.0) + pct_lwm= srv_max_dirty_pages_pct_lwm; + if (pct_lwm != 0.0) { const ulint activity_count= srv_get_activity_count(); if (activity_count != last_activity_count) @@ -2359,13 +2366,16 @@ static void buf_flush_page_cleaner() - there are no pending reads but there are dirty pages to flush */ buf_pool.update_last_activity_count(activity_count); mysql_mutex_unlock(&buf_pool.flush_list_mutex); - idle_flush= true; goto idle_flush; } else + { maybe_unemployed: - if (dirty_pct < srv_max_dirty_pages_pct_lwm) + const bool below{dirty_pct < pct_lwm}; + pct_lwm= 0.0; + if (below) goto possibly_unemployed; + } } else if (dirty_pct < srv_max_buf_pool_modified_pct) possibly_unemployed: @@ -2396,6 +2406,7 @@ static void buf_flush_page_cleaner() } else if ((n= page_cleaner_flush_pages_recommendation(last_pages, oldest_lsn, + pct_lwm, dirty_blocks, dirty_pct)) != 0) { -- cgit v1.2.1 From 5740638c4c3337a0021a82f5b744afca1ab1346c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 26 Apr 2023 12:08:59 +0300 Subject: MDEV-31132 Deadlock between DDL and purge of InnoDB history log_free_check(): Assert that the caller must not hold exclusive lock_sys.latch. This was the case for calls from ibuf_delete_for_discarded_space(). This caused a deadlock with another thread that would be holding a latch on a dirty page that would need to be written so that the checkpoint would advance and log_free_check() could return. That other thread was waiting for a shared lock_sys.latch. fil_delete_tablespace(): Do not invoke ibuf_delete_for_discarded_space() because in DDL operations, we will be holding exclusive lock_sys.latch. trx_t::commit(std::vector&), innodb_drop_database(), row_purge_remove_clust_if_poss_low(), row_undo_ins_remove_clust_rec(), row_discard_tablespace_for_mysql(): Invoke ibuf_delete_for_discarded_space() on the deleted tablespaces after releasing all latches. --- storage/innobase/dict/drop.cc | 7 +++++++ storage/innobase/fil/fil0fil.cc | 2 -- storage/innobase/handler/ha_innodb.cc | 5 +++++ storage/innobase/include/log0log.h | 13 ++++--------- storage/innobase/include/log0log.inl | 20 -------------------- storage/innobase/log/log0log.cc | 10 ++++++++++ storage/innobase/row/row0mysql.cc | 1 + storage/innobase/row/row0purge.cc | 9 +++++++-- storage/innobase/row/row0uins.cc | 9 +++++++-- 9 files changed, 41 insertions(+), 35 deletions(-) diff --git a/storage/innobase/dict/drop.cc b/storage/innobase/dict/drop.cc index 9013841ba5e..45747fb3596 100644 --- a/storage/innobase/dict/drop.cc +++ b/storage/innobase/dict/drop.cc @@ -68,6 +68,7 @@ before transaction commit and must be rolled back explicitly are as follows: #include "dict0defrag_bg.h" #include "btr0defragment.h" +#include "ibuf0ibuf.h" #include "lock0lock.h" #include "que0que.h" @@ -237,6 +238,8 @@ void trx_t::commit(std::vector &deleted) commit_persist(); if (dict_operation) { + std::vector space_ids; + space_ids.reserve(mod_tables.size()); ut_ad(dict_sys.locked()); lock_sys.wr_lock(SRW_LOCK_CALL); mutex_lock(); @@ -271,6 +274,7 @@ void trx_t::commit(std::vector &deleted) dict_sys.remove(table); if (const auto id= space ? space->id : 0) { + space_ids.emplace_back(uint32_t(id)); pfs_os_file_t d= fil_delete_tablespace(id); if (d != OS_FILE_CLOSED) deleted.emplace_back(d); @@ -283,6 +287,9 @@ void trx_t::commit(std::vector &deleted) mysql_mutex_lock(&lock_sys.wait_mutex); lock_sys.deadlock_check(); mysql_mutex_unlock(&lock_sys.wait_mutex); + + for (const auto id : space_ids) + ibuf_delete_for_discarded_space(id); } commit_cleanup(); } diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 48d205f428a..23f0cf75f39 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -45,7 +45,6 @@ Created 10/25/1995 Heikki Tuuri #include "srv0start.h" #include "trx0purge.h" #include "buf0lru.h" -#include "ibuf0ibuf.h" #include "buf0flu.h" #include "log.h" #ifdef __linux__ @@ -1689,7 +1688,6 @@ pfs_os_file_t fil_delete_tablespace(ulint id) fil_space_free_low(space); } - ibuf_delete_for_discarded_space(id); return handle; } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index f102789d7ab..0b117c02e29 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1542,6 +1542,7 @@ static void innodb_drop_database(handlerton*, char *path) dfield_set_data(&dfield, namebuf, len); dict_index_copy_types(&tuple, sys_index, 1); std::vector to_close; + std::vector space_ids; mtr_t mtr; mtr.start(); pcur.btr_cur.page_cur.index = sys_index; @@ -1585,6 +1586,7 @@ static void innodb_drop_database(handlerton*, char *path) ut_ad("corrupted SYS_TABLES.SPACE" == 0); else if (uint32_t space_id= mach_read_from_4(s)) { + space_ids.emplace_back(space_id); pfs_os_file_t detached= fil_delete_tablespace(space_id); if (detached != OS_FILE_CLOSED) to_close.emplace_back(detached); @@ -1594,6 +1596,9 @@ static void innodb_drop_database(handlerton*, char *path) mtr.commit(); for (pfs_os_file_t detached : to_close) os_file_close(detached); + for (const auto id : space_ids) + ibuf_delete_for_discarded_space(id); + /* Any changes must be persisted before we return. */ log_write_up_to(mtr.commit_lsn(), true); } diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 0f9a4da049b..0996b66ef52 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -73,15 +73,10 @@ log_reserve_and_write_fast( const void* str, ulint len, lsn_t* start_lsn); -/***********************************************************************//** -Checks if there is need for a log buffer flush or a new checkpoint, and does -this if yes. Any database operation should call this when it has modified -more than about 4 pages. NOTE that this function may only be called when the -OS thread owns no synchronization objects except dict_sys.latch. */ -UNIV_INLINE -void -log_free_check(void); -/*================*/ +/** Wait for a log checkpoint if needed. +NOTE that this function may only be called while not holding +any synchronization objects except dict_sys.latch. */ +void log_free_check(); /** Extends the log buffer. @param[in] len requested minimum size in bytes */ diff --git a/storage/innobase/include/log0log.inl b/storage/innobase/include/log0log.inl index c29c0bfa55f..27856fca8f9 100644 --- a/storage/innobase/include/log0log.inl +++ b/storage/innobase/include/log0log.inl @@ -289,23 +289,3 @@ log_reserve_and_write_fast( return lsn; } - -/***********************************************************************//** -Checks if there is need for a log buffer flush or a new checkpoint, and does -this if yes. Any database operation should call this when it has modified -more than about 4 pages. NOTE that this function may only be called when the -OS thread owns no synchronization objects except dict_sys.latch. */ -UNIV_INLINE -void -log_free_check(void) -/*================*/ -{ - /* During row_log_table_apply(), this function will be called while we - are holding some latches. This is OK, as long as we are not holding - any latches on buffer blocks. */ - - if (log_sys.check_flush_or_checkpoint()) { - - log_check_margins(); - } -} diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index c53e2fd5074..f65e812f40f 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -1065,6 +1065,16 @@ ATTRIBUTE_COLD void log_check_margins() while (log_sys.check_flush_or_checkpoint()); } +/** Wait for a log checkpoint if needed. +NOTE that this function may only be called while not holding +any synchronization objects except dict_sys.latch. */ +void log_free_check() +{ + ut_ad(!lock_sys.is_writer()); + if (log_sys.check_flush_or_checkpoint()) + log_check_margins(); +} + extern void buf_resize_shutdown(); /** Make a checkpoint at the latest lsn on shutdown. */ diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 67167f19c70..d27fc964219 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -2492,6 +2492,7 @@ rollback: if (fts_exist) purge_sys.resume_FTS(); + ibuf_delete_for_discarded_space(space_id); buf_flush_remove_pages(space_id); trx->op_info= ""; return err; diff --git a/storage/innobase/row/row0purge.cc b/storage/innobase/row/row0purge.cc index 753b42332fc..f4db252b069 100644 --- a/storage/innobase/row/row0purge.cc +++ b/storage/innobase/row/row0purge.cc @@ -162,8 +162,9 @@ close_and_exit: ut_ad("corrupted SYS_INDEXES record" == 0); } - if (const uint32_t space_id = dict_drop_index_tree( - &node->pcur, nullptr, &mtr)) { + const uint32_t space_id = dict_drop_index_tree( + &node->pcur, nullptr, &mtr); + if (space_id) { if (table) { if (table->get_ref_count() == 0) { dict_sys.remove(table); @@ -184,6 +185,10 @@ close_and_exit: table = nullptr; } + if (space_id) { + ibuf_delete_for_discarded_space(space_id); + } + purge_sys.check_stop_SYS(); mtr.start(); index->set_modified(mtr); diff --git a/storage/innobase/row/row0uins.cc b/storage/innobase/row/row0uins.cc index 50196e78092..ff1a34b46c3 100644 --- a/storage/innobase/row/row0uins.cc +++ b/storage/innobase/row/row0uins.cc @@ -147,8 +147,9 @@ restart: pfs_os_file_t d = OS_FILE_CLOSED; - if (const uint32_t space_id = dict_drop_index_tree( - &node->pcur, node->trx, &mtr)) { + const uint32_t space_id = dict_drop_index_tree( + &node->pcur, node->trx, &mtr); + if (space_id) { if (table) { lock_release_on_rollback(node->trx, table); @@ -187,6 +188,10 @@ restart: os_file_close(d); } + if (space_id) { + ibuf_delete_for_discarded_space(space_id); + } + mtr.start(); ut_a(node->pcur.restore_position( BTR_MODIFY_LEAF, &mtr) == btr_pcur_t::SAME_ALL); -- cgit v1.2.1 From 09fdd3aca6f0d6cfaf8218def9d1a99f5a325160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 26 Apr 2023 18:19:44 +0300 Subject: After-merge fix This fixes up commit e3f6e1c92e1604c63e4de113f8863953c05b29a9 --- storage/spider/spd_db_mysql.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/storage/spider/spd_db_mysql.cc b/storage/spider/spd_db_mysql.cc index 753e777017c..f11bf4fc439 100644 --- a/storage/spider/spd_db_mysql.cc +++ b/storage/spider/spd_db_mysql.cc @@ -2261,8 +2261,7 @@ void spider_db_mbase::fetch_and_print_warnings(struct tm *l_time) DBUG_VOID_RETURN; if (mysql_num_fields(res) == 3) - for (MYSQL_ROW row= mysql_fetch_row(res); row; - mysql_fetch_row(res)) + while (MYSQL_ROW row= mysql_fetch_row(res)) fprintf(stderr, "%04d%02d%02d %02d:%02d:%02d [WARN SPIDER RESULT] from [%s] %ld " "to %ld: %s %s %s\n", -- cgit v1.2.1