From 6bdc03ebccad17f7e9c202e64bf7c2c450a7396c Mon Sep 17 00:00:00 2001 From: Monty Date: Tue, 31 Aug 2021 15:34:28 +0300 Subject: Added options s3_port and s3_use_http to aria_s3_copy These options was needed in some cases, like when using minio that require the port option, to be able to connect to the S3 storage. The sympthom was that one could get the error "Table t1.MAI doesn't exist in s3" even if the table did exits. Other things: - Improved error message for non existing S3 files --- storage/maria/aria_s3_copy.cc | 28 ++++++++++++++++++++++------ storage/maria/s3_func.c | 4 ++-- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/storage/maria/aria_s3_copy.cc b/storage/maria/aria_s3_copy.cc index f324c0896e5..77c41ba4572 100644 --- a/storage/maria/aria_s3_copy.cc +++ b/storage/maria/aria_s3_copy.cc @@ -40,8 +40,10 @@ static const char *opt_s3_host_name= DEFAULT_AWS_HOST_NAME; static const char *opt_database; static const char *opt_s3_bucket="MariaDB"; static my_bool opt_compression, opt_verbose, opt_force, opt_s3_debug; +static my_bool opt_s3_use_http; static ulong opt_operation= OP_IMPOSSIBLE, opt_protocol_version= 1; static ulong opt_block_size; +static ulong opt_s3_port; static char **default_argv=0; static ms3_st *global_s3_client= 0; @@ -65,6 +67,12 @@ static struct my_option my_long_options[] = {"s3_host_name", 'h', "Host name to S3 provider", (char**) &opt_s3_host_name, (char**) &opt_s3_host_name, 0, GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, + {"s3_port", 'p', "Port number to connect to (0 means use default)", + (char**) &opt_s3_port, (char**) &opt_s3_port, 0, GET_ULONG, REQUIRED_ARG, + 0, 0, 65536, 0, 1, 0 }, + {"s3_use_http", 'P', "If true, force use of HTTP protocol", + (char**) &opt_s3_use_http, (char**) &opt_s3_use_http, + 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"compress", 'c', "Use compression", &opt_compression, &opt_compression, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}, {"op", 'o', "Operation to execute. One of 'from_s3', 'to_s3' or " @@ -196,6 +204,7 @@ int main(int argc, char** argv) { MY_INIT(argv[0]); get_options(&argc,(char***) &argv); + size_t block_size= opt_block_size; s3_init_library(); if (!(global_s3_client= ms3_init(opt_s3_access_key, @@ -207,15 +216,22 @@ int main(int argc, char** argv) my_exit(1); } + ms3_set_option(global_s3_client, MS3_OPT_BUFFER_CHUNK_SIZE, &block_size); + + if (opt_protocol_version) { - size_t block_size= opt_block_size; uint8_t protocol_version= (uint8_t) opt_protocol_version; - ms3_set_option(global_s3_client, MS3_OPT_BUFFER_CHUNK_SIZE, &block_size); - - if (protocol_version) - ms3_set_option(global_s3_client, MS3_OPT_FORCE_PROTOCOL_VERSION, - &protocol_version); + ms3_set_option(global_s3_client, MS3_OPT_FORCE_PROTOCOL_VERSION, + &protocol_version); } + if (opt_s3_port) + { + int port= (int) opt_s3_port; + ms3_set_option(global_s3_client, MS3_OPT_PORT_NUMBER, &port); + } + if (opt_s3_use_http) + ms3_set_option(global_s3_client, MS3_OPT_USE_HTTP, NULL); + for (; *argv ; argv++) { diff --git a/storage/maria/s3_func.c b/storage/maria/s3_func.c index 9f40790a371..06178c1c0c2 100644 --- a/storage/maria/s3_func.c +++ b/storage/maria/s3_func.c @@ -583,8 +583,8 @@ int aria_copy_from_s3(ms3_st *s3_client, const char *aws_bucket, if (s3_get_object(s3_client, aws_bucket, aws_path, &block, 0, 0)) { - my_printf_error(EE_FILENOTFOUND, "Table %s doesn't exist in s3", MYF(0), - filename); + my_printf_error(EE_FILENOTFOUND, "File %s/%s doesn't exist in s3", MYF(0), + database,filename); goto err; } if (block.length < MARIA_STATE_INFO_SIZE) -- cgit v1.2.1 From 49ae199604fa3f8129ded9d4490df022f61d7a1d Mon Sep 17 00:00:00 2001 From: Monty Date: Tue, 31 Aug 2021 18:02:38 +0300 Subject: Added support for ANALYZE TABLE to S3 tables Other things - Cleaned up error messages for CHECK, REPAIR and OPTIMIZE --- mysql-test/suite/s3/basic.result | 30 ++++++++++++++ mysql-test/suite/s3/basic.test | 25 +++++++++++ mysql-test/suite/s3/partition.result | 7 ++-- sql/ha_partition.cc | 3 +- sql/protocol.h | 4 ++ sql/sql_admin.cc | 80 +++++++++++++++++++++++++----------- storage/maria/ha_s3.h | 2 +- 7 files changed, 121 insertions(+), 30 deletions(-) diff --git a/mysql-test/suite/s3/basic.result b/mysql-test/suite/s3/basic.result index cf7d7377d92..34fe918353d 100644 --- a/mysql-test/suite/s3/basic.result +++ b/mysql-test/suite/s3/basic.result @@ -55,6 +55,36 @@ update t1 set b=100 where a=1; ERROR HY000: Table 't1' is read only delete from t1 where a>10; ERROR HY000: Table 't1' is read only +# +# Analyze, repair, optimize and check table +# +set @@use_stat_tables='never'; +truncate mysql.table_stats; +check table t1; +Table Op Msg_type Msg_text +database.t1 check status Table 'database.t1' is read only +analyze table t1; +Table Op Msg_type Msg_text +database.t1 analyze status Table 'database.t1' is read only +analyze table t1 persistent for all; +Table Op Msg_type Msg_text +database.t1 analyze status Table 'database.t1' is read only +database.t1 analyze status Engine-independent statistics collected +database.t1 analyze status OK +repair table t1; +Table Op Msg_type Msg_text +database.t1 repair Error Table 't1' is read only +database.t1 repair status Operation failed +optimize table t1; +Table Op Msg_type Msg_text +database.t1 optimize Error Table 't1' is read only +database.t1 optimize status Operation failed +select * from mysql.table_stats; +db_name table_name cardinality +database t1 10000 +# +# Converting table back to Aria +# alter table t1 engine=aria; show create table t1; Table Create Table diff --git a/mysql-test/suite/s3/basic.test b/mysql-test/suite/s3/basic.test index 91183841a62..2e104444e0f 100644 --- a/mysql-test/suite/s3/basic.test +++ b/mysql-test/suite/s3/basic.test @@ -34,6 +34,31 @@ insert into t1 values (1,1); update t1 set b=100 where a=1; --error ER_OPEN_AS_READONLY delete from t1 where a>10; + + +--echo # +--echo # Analyze, repair, optimize and check table +--echo # + +set @@use_stat_tables='never'; +truncate mysql.table_stats; +--replace_result $database database +check table t1; +--replace_result $database database +analyze table t1; +--replace_result $database database +analyze table t1 persistent for all; +--replace_result $database database +repair table t1; +--replace_result $database database +optimize table t1; +--replace_result $database database +select * from mysql.table_stats; + +--echo # +--echo # Converting table back to Aria +--echo # + alter table t1 engine=aria; show create table t1; select * from t1 limit 10; diff --git a/mysql-test/suite/s3/partition.result b/mysql-test/suite/s3/partition.result index ee837611d50..08f61da012c 100644 --- a/mysql-test/suite/s3/partition.result +++ b/mysql-test/suite/s3/partition.result @@ -56,14 +56,15 @@ ALTER TABLE t2 TRUNCATE PARTITION p3; ERROR HY000: Table 't2' is read only ALTER TABLE t2 ANALYZE PARTITION p3; Table Op Msg_type Msg_text -s3.t2 analyze error Table 's3.t2' is read only +s3.t2 analyze status Table 's3.t2' is read only +s3.t2 analyze status Engine-independent statistics collected +s3.t2 analyze status OK SELECT count(*) FROM t2; count(*) 6 ALTER TABLE t2 CHECK PARTITION p3; Table Op Msg_type Msg_text -s3.t2 check error Subpartition p3sp0 returned error -s3.t2 check error Unknown - internal error 165 during operation +s3.t2 check status Table 's3.t2' is read only SELECT count(*) FROM t2; count(*) 6 diff --git a/sql/ha_partition.cc b/sql/ha_partition.cc index 73e9f8ee84f..6b82bc811aa 100644 --- a/sql/ha_partition.cc +++ b/sql/ha_partition.cc @@ -1509,7 +1509,8 @@ int ha_partition::handle_opt_partitions(THD *thd, HA_CHECK_OPT *check_opt, /* print a line which partition the error belongs to */ if (error != HA_ADMIN_NOT_IMPLEMENTED && error != HA_ADMIN_ALREADY_DONE && - error != HA_ADMIN_TRY_ALTER) + error != HA_ADMIN_TRY_ALTER && + error != HA_ERR_TABLE_READONLY) { print_admin_msg(thd, MYSQL_ERRMSG_SIZE, "error", table_share->db.str, table->alias, diff --git a/sql/protocol.h b/sql/protocol.h index eb11304a4d5..f98b4cabfed 100644 --- a/sql/protocol.h +++ b/sql/protocol.h @@ -114,6 +114,10 @@ public: { return store_longlong((longlong) from, 1); } inline bool store(String *str) { return store((char*) str->ptr(), str->length(), str->charset()); } + inline bool store(const LEX_CSTRING *from, CHARSET_INFO *cs) + { + return store(from->str, from->length, cs); + } virtual bool prepare_for_send(uint num_columns) { diff --git a/sql/sql_admin.cc b/sql/sql_admin.cc index c62e4c1669c..f713064f197 100644 --- a/sql/sql_admin.cc +++ b/sql/sql_admin.cc @@ -32,6 +32,9 @@ #include "sql_admin.h" #include "sql_statistics.h" #include "wsrep_mysqld.h" + +const LEX_CSTRING msg_status= {STRING_WITH_LEN("status")}; + /* Prepare, run and cleanup for mysql_recreate_table() */ static bool admin_recreate_table(THD *thd, TABLE_LIST *table_list) @@ -472,6 +475,21 @@ static bool wsrep_toi_replication(THD *thd, TABLE_LIST *tables) } #endif /* WITH_WSREP */ + +static void send_read_only_warning(THD *thd, const LEX_CSTRING *msg_status, + const LEX_CSTRING *table_name) +{ + Protocol *protocol= thd->protocol; + char buf[MYSQL_ERRMSG_SIZE]; + size_t length; + length= my_snprintf(buf, sizeof(buf), + ER_THD(thd, ER_OPEN_AS_READONLY), + table_name->str); + protocol->store(msg_status, system_charset_info); + protocol->store(buf, length, system_charset_info); +} + + /* RETURN VALUES FALSE Message sent to net (admin operation went ok) @@ -555,7 +573,8 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, for (table= tables; table; table= table->next_local) { - char table_name[SAFE_NAME_LEN*2+2]; + char table_name_buff[SAFE_NAME_LEN*2+2]; + LEX_CSTRING table_name= { table_name_buff, 0}; const char *db= table->db.str; bool fatal_error=0; bool open_error; @@ -565,7 +584,8 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, DBUG_PRINT("admin", ("table: '%s'.'%s'", db, table->table_name.str)); DEBUG_SYNC(thd, "admin_command_kill_before_modify"); - strxmov(table_name, db, ".", table->table_name.str, NullS); + table_name.length= strxmov(table_name_buff, db, ".", table->table_name.str, + NullS) - table_name_buff; thd->open_options|= extra_open_options; table->lock_type= lock_type; /* @@ -651,12 +671,12 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, size_t length; DBUG_PRINT("admin", ("sending non existent partition error")); protocol->prepare_for_resend(); - protocol->store(table_name, system_charset_info); + protocol->store(&table_name, system_charset_info); protocol->store(operator_name, system_charset_info); protocol->store(STRING_WITH_LEN("error"), system_charset_info); length= my_snprintf(buff, sizeof(buff), ER_THD(thd, ER_DROP_PARTITION_NON_EXISTENT), - table_name); + table_name.str); protocol->store(buff, length, system_charset_info); if(protocol->write()) goto err; @@ -732,20 +752,17 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, goto send_result; } - if ((table->table->db_stat & HA_READ_ONLY) && open_for_modify) + if ((table->table->db_stat & HA_READ_ONLY) && open_for_modify && + operator_func != &handler::ha_analyze) { /* purecov: begin inspected */ - char buff[FN_REFLEN + MYSQL_ERRMSG_SIZE]; - size_t length; enum_sql_command save_sql_command= lex->sql_command; + LEX_CSTRING error_clex_str= { STRING_WITH_LEN("error") }; DBUG_PRINT("admin", ("sending error message")); protocol->prepare_for_resend(); - protocol->store(table_name, system_charset_info); + protocol->store(&table_name, system_charset_info); protocol->store(operator_name, system_charset_info); - protocol->store(STRING_WITH_LEN("error"), system_charset_info); - length= my_snprintf(buff, sizeof(buff), ER_THD(thd, ER_OPEN_AS_READONLY), - table_name); - protocol->store(buff, length, system_charset_info); + send_read_only_warning(thd, &error_clex_str, &table_name); trans_commit_stmt(thd); trans_commit(thd); close_thread_tables(thd); @@ -800,7 +817,7 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, /* purecov: begin inspected */ DBUG_PRINT("admin", ("sending crashed warning")); protocol->prepare_for_resend(); - protocol->store(table_name, system_charset_info); + protocol->store(&table_name, system_charset_info); protocol->store(operator_name, system_charset_info); protocol->store(STRING_WITH_LEN("warning"), system_charset_info); protocol->store(STRING_WITH_LEN("Table is marked as crashed"), @@ -871,6 +888,15 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, if (compl_result_code == HA_ADMIN_OK && collect_eis) { + if (result_code == HA_ERR_TABLE_READONLY) + { + protocol->prepare_for_resend(); + protocol->store(&table_name, system_charset_info); + protocol->store(operator_name, system_charset_info); + send_read_only_warning(thd, &msg_status, &table_name); + (void) protocol->write(); + result_code= HA_ADMIN_OK; + } /* Here we close and reopen table in read mode because operation of collecting statistics is long and it will be better do not block @@ -978,9 +1004,9 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables, else { protocol->prepare_for_resend(); - protocol->store(table_name, system_charset_info); + protocol->store(&table_name, system_charset_info); protocol->store(operator_name, system_charset_info); - protocol->store(STRING_WITH_LEN("status"), system_charset_info); + protocol->store(&msg_status, system_charset_info); protocol->store(STRING_WITH_LEN("Engine-independent statistics collected"), system_charset_info); if (protocol->write()) @@ -1007,7 +1033,7 @@ send_result: while ((err= it++)) { protocol->prepare_for_resend(); - protocol->store(table_name, system_charset_info); + protocol->store(&table_name, system_charset_info); protocol->store((char*) operator_name, system_charset_info); protocol->store(warning_level_names[err->get_level()].str, warning_level_names[err->get_level()].length, @@ -1019,7 +1045,7 @@ send_result: thd->get_stmt_da()->clear_warning_info(thd->query_id); } protocol->prepare_for_resend(); - protocol->store(table_name, system_charset_info); + protocol->store(&table_name, system_charset_info); protocol->store(operator_name, system_charset_info); send_result_message: @@ -1042,32 +1068,32 @@ send_result_message: char buf[MYSQL_ERRMSG_SIZE]; size_t length= my_snprintf(buf, sizeof(buf), ER_THD(thd, ER_BAD_TABLE_ERROR), - table_name); + table_name.str); protocol->store(STRING_WITH_LEN("note"), system_charset_info); protocol->store(buf, length, system_charset_info); } break; case HA_ADMIN_OK: - protocol->store(STRING_WITH_LEN("status"), system_charset_info); + protocol->store(&msg_status, system_charset_info); protocol->store(STRING_WITH_LEN("OK"), system_charset_info); break; case HA_ADMIN_FAILED: - protocol->store(STRING_WITH_LEN("status"), system_charset_info); + protocol->store(&msg_status, system_charset_info); protocol->store(STRING_WITH_LEN("Operation failed"), system_charset_info); break; case HA_ADMIN_REJECT: - protocol->store(STRING_WITH_LEN("status"), system_charset_info); + protocol->store(&msg_status, system_charset_info); protocol->store(STRING_WITH_LEN("Operation need committed state"), system_charset_info); open_for_modify= FALSE; break; case HA_ADMIN_ALREADY_DONE: - protocol->store(STRING_WITH_LEN("status"), system_charset_info); + protocol->store(&msg_status, system_charset_info); protocol->store(STRING_WITH_LEN("Table is already up to date"), system_charset_info); break; @@ -1147,7 +1173,7 @@ send_result_message: } /* Start a new row for the final status row */ protocol->prepare_for_resend(); - protocol->store(table_name, system_charset_info); + protocol->store(&table_name, system_charset_info); protocol->store(operator_name, system_charset_info); if (result_code) // either mysql_recreate_table or analyze failed { @@ -1168,7 +1194,7 @@ send_result_message: goto err; /* Start off another row for HA_ADMIN_FAILED */ protocol->prepare_for_resend(); - protocol->store(table_name, system_charset_info); + protocol->store(&table_name, system_charset_info); protocol->store(operator_name, system_charset_info); } thd->clear_error(); @@ -1212,7 +1238,11 @@ send_result_message: fatal_error=1; break; } - + case HA_ERR_TABLE_READONLY: + { + send_read_only_warning(thd, &msg_status, &table_name); + break; + } default: // Probably HA_ADMIN_INTERNAL_ERROR { char buf[MYSQL_ERRMSG_SIZE]; diff --git a/storage/maria/ha_s3.h b/storage/maria/ha_s3.h index e16353b2a32..ca021022c7b 100644 --- a/storage/maria/ha_s3.h +++ b/storage/maria/ha_s3.h @@ -46,7 +46,7 @@ public: } int check(THD *, HA_CHECK_OPT *) override { - DBUG_ENTER("delete_row"); + DBUG_ENTER("check"); DBUG_RETURN(HA_ERR_TABLE_READONLY); } int analyze(THD *, HA_CHECK_OPT *) override -- cgit v1.2.1 From 10f08aff159f916358cf9902fc5f6a976b8faeda Mon Sep 17 00:00:00 2001 From: Monty Date: Wed, 1 Sep 2021 13:32:57 +0300 Subject: Added support for CHECK TABLE for S3 tables Other things: - Don't allocate an IO_CACHE for scanning tables of type BLOCK (It was never used in this case) - Fixed bug in page cache that cased a hang when trying to read a not existing S3 block. --- mysql-test/suite/s3/basic.result | 32 +++++++++++++++++++++----------- mysql-test/suite/s3/basic.test | 16 ++++++++++------ mysql-test/suite/s3/partition.result | 2 +- storage/maria/ha_maria.cc | 15 ++++++++++----- storage/maria/ha_s3.h | 5 ----- storage/maria/ma_check.c | 6 ++++++ storage/maria/ma_pagecache.c | 8 ++++++-- 7 files changed, 54 insertions(+), 30 deletions(-) diff --git a/mysql-test/suite/s3/basic.result b/mysql-test/suite/s3/basic.result index 34fe918353d..ac391ba3574 100644 --- a/mysql-test/suite/s3/basic.result +++ b/mysql-test/suite/s3/basic.result @@ -2,23 +2,25 @@ drop table if exists t1; # # Test simple create of s3 table # -create or replace table t1 (a int, b int, key (a)) engine=aria; -insert into t1 select seq,seq+10 from seq_1_to_10000; +create or replace table t1 (a int, b int, c varchar(1000), key (a), key(c)) engine=aria; +insert into t1 select seq, seq+10, repeat(char(65+ mod(seq, 20)),mod(seq,1000)) from seq_1_to_10000; alter table t1 engine=s3; show create table t1; Table Create Table t1 CREATE TABLE `t1` ( `a` int(11) DEFAULT NULL, `b` int(11) DEFAULT NULL, - KEY `a` (`a`) + `c` varchar(1000) DEFAULT NULL, + KEY `a` (`a`), + KEY `c` (`c`) ) ENGINE=S3 DEFAULT CHARSET=latin1 PAGE_CHECKSUM=1 select * from information_schema.tables where table_schema="database" and table_name="t1";; TABLE_CATALOG TABLE_SCHEMA TABLE_NAME TABLE_TYPE ENGINE VERSION ROW_FORMAT TABLE_ROWS AVG_ROW_LENGTH DATA_LENGTH MAX_DATA_LENGTH INDEX_LENGTH DATA_FREE AUTO_INCREMENT CREATE_TIME UPDATE_TIME CHECK_TIME TABLE_COLLATION CHECKSUM CREATE_OPTIONS TABLE_COMMENT MAX_INDEX_LENGTH TEMPORARY -def # t1 BASE TABLE S3 10 Page 10000 33 335872 # 122880 0 NULL # # # latin1_swedish_ci NULL page_checksum=1 9007199254732800 # +def # t1 BASE TABLE S3 10 Page 10000 567 5677056 # 761856 0 NULL # # # latin1_swedish_ci NULL page_checksum=1 2305843009213685760 # show table status like "t1"; Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment Max_index_length Temporary -t1 S3 10 Page 10000 33 335872 # 122880 0 NULL # # # latin1_swedish_ci NULL page_checksum=1 # N -select * from t1 limit 10; +t1 S3 10 Page 10000 567 5677056 # 761856 0 NULL # # # latin1_swedish_ci NULL page_checksum=1 # N +select a,b from t1 limit 10; a b 1 11 2 12 @@ -33,7 +35,7 @@ a b select count(*) from t1; count(*) 10000 -select * from t1 where a between 10 and 20; +select a,b from t1 where a between 10 and 20; a b 10 20 11 21 @@ -60,9 +62,15 @@ ERROR HY000: Table 't1' is read only # set @@use_stat_tables='never'; truncate mysql.table_stats; -check table t1; +check table t1 fast; Table Op Msg_type Msg_text -database.t1 check status Table 'database.t1' is read only +database.t1 check status Table is already up to date +check table t1 quick; +Table Op Msg_type Msg_text +database.t1 check status OK +check table t1 extended; +Table Op Msg_type Msg_text +database.t1 check status OK analyze table t1; Table Op Msg_type Msg_text database.t1 analyze status Table 'database.t1' is read only @@ -91,9 +99,11 @@ Table Create Table t1 CREATE TABLE `t1` ( `a` int(11) DEFAULT NULL, `b` int(11) DEFAULT NULL, - KEY `a` (`a`) + `c` varchar(1000) DEFAULT NULL, + KEY `a` (`a`), + KEY `c` (`c`) ) ENGINE=Aria DEFAULT CHARSET=latin1 PAGE_CHECKSUM=1 -select * from t1 limit 10; +select a,b from t1 limit 10; a b 1 11 2 12 diff --git a/mysql-test/suite/s3/basic.test b/mysql-test/suite/s3/basic.test index 2e104444e0f..99c2d8adb5d 100644 --- a/mysql-test/suite/s3/basic.test +++ b/mysql-test/suite/s3/basic.test @@ -13,8 +13,8 @@ drop table if exists t1; --echo # Test simple create of s3 table --echo # -create or replace table t1 (a int, b int, key (a)) engine=aria; -insert into t1 select seq,seq+10 from seq_1_to_10000; +create or replace table t1 (a int, b int, c varchar(1000), key (a), key(c)) engine=aria; +insert into t1 select seq, seq+10, repeat(char(65+ mod(seq, 20)),mod(seq,1000)) from seq_1_to_10000; alter table t1 engine=s3; show create table t1; @@ -23,9 +23,9 @@ show create table t1; --eval select * from information_schema.tables where table_schema="$database" and table_name="t1"; --replace_column 8 # 12 # 13 # 14 # 19 # show table status like "t1"; -select * from t1 limit 10; +select a,b from t1 limit 10; select count(*) from t1; -select * from t1 where a between 10 and 20; +select a,b from t1 where a between 10 and 20; --replace_column 9 # explain select * from t1 where a between 10 and 20; --error ER_OPEN_AS_READONLY @@ -43,7 +43,11 @@ delete from t1 where a>10; set @@use_stat_tables='never'; truncate mysql.table_stats; --replace_result $database database -check table t1; +check table t1 fast; +--replace_result $database database +check table t1 quick; +--replace_result $database database +check table t1 extended; --replace_result $database database analyze table t1; --replace_result $database database @@ -61,7 +65,7 @@ select * from mysql.table_stats; alter table t1 engine=aria; show create table t1; -select * from t1 limit 10; +select a,b from t1 limit 10; select count(*) from t1; delete from t1 where a=1; drop table t1; diff --git a/mysql-test/suite/s3/partition.result b/mysql-test/suite/s3/partition.result index 08f61da012c..86a70b3c694 100644 --- a/mysql-test/suite/s3/partition.result +++ b/mysql-test/suite/s3/partition.result @@ -64,7 +64,7 @@ count(*) 6 ALTER TABLE t2 CHECK PARTITION p3; Table Op Msg_type Msg_text -s3.t2 check status Table 's3.t2' is read only +s3.t2 check status OK SELECT count(*) FROM t2; count(*) 6 diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc index 5bf6754ff9d..b93039190b0 100644 --- a/storage/maria/ha_maria.cc +++ b/storage/maria/ha_maria.cc @@ -1270,6 +1270,7 @@ int ha_maria::check(THD * thd, HA_CHECK_OPT * check_opt) if (!file || !param) return HA_ADMIN_INTERNAL_ERROR; unmap_file(file); + register_handler(file); maria_chk_init(param); param->thd= thd; param->op_name= "check"; @@ -1325,14 +1326,18 @@ int ha_maria::check(THD * thd, HA_CHECK_OPT * check_opt) { ulonglong old_testflag= param->testflag; param->testflag |= T_MEDIUM; - if (!(error= init_io_cache(¶m->read_cache, file->dfile.file, - my_default_record_cache_size, READ_CACHE, - share->pack.header_length, 1, MYF(MY_WME)))) - { + + /* BLOCK_RECORD does not need a cache as it is using the page cache */ + if (file->s->data_file_type != BLOCK_RECORD) + error= init_io_cache(¶m->read_cache, file->dfile.file, + my_default_record_cache_size, READ_CACHE, + share->pack.header_length, 1, MYF(MY_WME)); + if (!error) error= maria_chk_data_link(param, file, MY_TEST(param->testflag & T_EXTEND)); + + if (file->s->data_file_type != BLOCK_RECORD) end_io_cache(¶m->read_cache); - } param->testflag= old_testflag; } } diff --git a/storage/maria/ha_s3.h b/storage/maria/ha_s3.h index ca021022c7b..d4b9e954154 100644 --- a/storage/maria/ha_s3.h +++ b/storage/maria/ha_s3.h @@ -44,11 +44,6 @@ public: DBUG_ENTER("delete_row"); DBUG_RETURN(HA_ERR_TABLE_READONLY); } - int check(THD *, HA_CHECK_OPT *) override - { - DBUG_ENTER("check"); - DBUG_RETURN(HA_ERR_TABLE_READONLY); - } int analyze(THD *, HA_CHECK_OPT *) override { DBUG_ENTER("analyze"); diff --git a/storage/maria/ma_check.c b/storage/maria/ma_check.c index 2b65cd36418..358d563b63a 100644 --- a/storage/maria/ma_check.c +++ b/storage/maria/ma_check.c @@ -413,6 +413,12 @@ int maria_chk_size(HA_CHECK *param, register MARIA_HA *info) char buff[22],buff2[22]; DBUG_ENTER("maria_chk_size"); + if (info->s3) + { + /* We cannot check file sizes for S3 */ + DBUG_RETURN(0); + } + if (!(param->testflag & T_SILENT)) puts("- check file-size"); diff --git a/storage/maria/ma_pagecache.c b/storage/maria/ma_pagecache.c index bf646115bd9..a2e9a5cc172 100644 --- a/storage/maria/ma_pagecache.c +++ b/storage/maria/ma_pagecache.c @@ -2772,7 +2772,7 @@ retry: #ifdef WITH_S3_STORAGE_ENGINE static void read_big_block(PAGECACHE *pagecache, - PAGECACHE_BLOCK_LINK *block) + PAGECACHE_BLOCK_LINK *block) { int page_st; size_t big_block_size_in_pages; @@ -2810,6 +2810,11 @@ static void read_big_block(PAGECACHE *pagecache, if (block_to_read->status & PCBLOCK_ERROR) { /* We get first block with an error so all operation failed */ + DBUG_PRINT("error", ("Got error when reading first page")); + block->status|= PCBLOCK_ERROR; + block->error= block_to_read->error; + remove_reader(block_to_read); + unreg_request(pagecache, block_to_read, 1); DBUG_VOID_RETURN; } if (block_to_read->status & PCBLOCK_BIG_READ) @@ -3952,7 +3957,6 @@ void pagecache_set_write_on_delete_by_link(PAGECACHE_BLOCK_LINK *block) @retval 0 deleted or was not present at all @retval 1 error - */ static my_bool pagecache_delete_internal(PAGECACHE *pagecache, -- cgit v1.2.1 From 4690442411b75ea0fc1a6aacabe767d6fb1d1f62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 2 Sep 2021 17:17:18 +0300 Subject: MDEV-24258 fixup: Throttle purge with exclusive dict_sys.latch A performance regression was observed after commit 82b7c561b7919fa24e3d24b3f04a16046e24374f because purge tasks would end up waiting more elsewhere, most notably trx_purge_get_next_rec() and trx_purge_truncate_history(). row_purge_parse_undo_rec(): Prevent the performance regression by unnecessarily acquiring dict_sys.latch in exclusive mode after the table lookup. --- storage/innobase/row/row0purge.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/storage/innobase/row/row0purge.cc b/storage/innobase/row/row0purge.cc index 4588c1d2b0a..29ceb0dabc1 100644 --- a/storage/innobase/row/row0purge.cc +++ b/storage/innobase/row/row0purge.cc @@ -1038,6 +1038,12 @@ try_again: goto err_exit; } + /* FIXME: We are acquiring exclusive dict_sys.latch only to + avoid increased wait times in + trx_purge_get_next_rec() and trx_purge_truncate_history(). */ + dict_sys.lock(SRW_LOCK_CALL); + dict_sys.unlock(); + already_locked: ut_ad(!node->table->is_temporary()); -- cgit v1.2.1 From e38a05e20a5b7c3d0ac0ddb6975e2af9b103c799 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 2 Sep 2021 19:41:54 +0200 Subject: Fix create_background_thd() Allow the caller to have current_thd. Also do not store PSI_CALL_get_thread() in the new THD, it is a thread local storage variable that can become invalid any time, we do not control the lifetime of the caller's thread. --- sql/sql_class.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sql/sql_class.cc b/sql/sql_class.cc index df241afc838..0d9a8439ccb 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -4894,11 +4894,13 @@ void destroy_thd(MYSQL_THD thd) extern "C" pthread_key(struct st_my_thread_var *, THR_KEY_mysys); MYSQL_THD create_background_thd() { - DBUG_ASSERT(!current_thd); + auto save_thd = current_thd; + set_current_thd(nullptr); + auto save_mysysvar= pthread_getspecific(THR_KEY_mysys); /* - Allocate new mysys_var specifically this THD, + Allocate new mysys_var specifically new THD, so that e.g safemalloc, DBUG etc are happy. */ pthread_setspecific(THR_KEY_mysys, 0); @@ -4906,7 +4908,8 @@ MYSQL_THD create_background_thd() auto thd_mysysvar= pthread_getspecific(THR_KEY_mysys); auto thd= new THD(0); pthread_setspecific(THR_KEY_mysys, save_mysysvar); - thd->set_psi(PSI_CALL_get_thread()); + thd->set_psi(nullptr); + set_current_thd(save_thd); /* Workaround the adverse effect of incrementing thread_count -- cgit v1.2.1 From c4ebfe22f920f671a6ed27c78d57d992d342f482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 3 Sep 2021 18:48:38 +0300 Subject: MDEV-4750 fixup: main.backup_lock result difference This test may occasionally display MDL for the InnoDB persistent statistics tables. Filter them out. This fixes up commit 9608773f75e2ca21491ef6825c3616cdc96d1ca5. --- mysql-test/main/backup_lock.result | 27 ++++++++++++++++++--------- mysql-test/main/backup_lock.test | 21 ++++++++++++--------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/mysql-test/main/backup_lock.result b/mysql-test/main/backup_lock.result index e519d3d325a..16806148c67 100644 --- a/mysql-test/main/backup_lock.result +++ b/mysql-test/main/backup_lock.result @@ -2,23 +2,28 @@ # Testing which locks we get from all stages # BACKUP STAGE START; -SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; +SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info +WHERE TABLE_NAME NOT LIKE 'innodb_%_stats'; LOCK_MODE LOCK_TYPE TABLE_SCHEMA TABLE_NAME MDL_BACKUP_START Backup lock BACKUP STAGE FLUSH; -SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; +SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info +WHERE TABLE_NAME NOT LIKE 'innodb_%_stats'; LOCK_MODE LOCK_TYPE TABLE_SCHEMA TABLE_NAME MDL_BACKUP_FLUSH Backup lock BACKUP STAGE BLOCK_DDL; -SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; +SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info +WHERE TABLE_NAME NOT LIKE 'innodb_%_stats'; LOCK_MODE LOCK_TYPE TABLE_SCHEMA TABLE_NAME MDL_BACKUP_WAIT_DDL Backup lock BACKUP STAGE BLOCK_COMMIT; -SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; +SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info +WHERE TABLE_NAME NOT LIKE 'innodb_%_stats'; LOCK_MODE LOCK_TYPE TABLE_SCHEMA TABLE_NAME MDL_BACKUP_WAIT_COMMIT Backup lock BACKUP STAGE END; -SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; +SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info +WHERE TABLE_NAME NOT LIKE 'innodb_%_stats'; LOCK_MODE LOCK_TYPE TABLE_SCHEMA TABLE_NAME # # testing BACKUP STAGE LOCK's @@ -39,7 +44,8 @@ connection con1; alter table t1 add column (j int), algorithm copy; connection con2; backup stage flush; -SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; +SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info +WHERE TABLE_NAME NOT LIKE 'innodb_%_stats'; LOCK_MODE LOCK_TYPE TABLE_SCHEMA TABLE_NAME MDL_BACKUP_DDL Backup lock MDL_BACKUP_FLUSH Backup lock @@ -75,7 +81,8 @@ alter table t1 add column (j int); connection con2; backup stage start; backup stage flush; -SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; +SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info +WHERE TABLE_NAME NOT LIKE 'innodb_%_stats'; LOCK_MODE LOCK_TYPE TABLE_SCHEMA TABLE_NAME MDL_BACKUP_ALTER_COPY Backup lock MDL_BACKUP_FLUSH Backup lock @@ -110,7 +117,8 @@ backup stage flush; SET STATEMENT lock_wait_timeout=0 FOR SELECT * FROM t1; ERROR HY000: Lock wait timeout exceeded; try restarting transaction backup stage block_ddl; -SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; +SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info +WHERE TABLE_NAME NOT LIKE 'innodb_%_stats'; LOCK_MODE LOCK_TYPE TABLE_SCHEMA TABLE_NAME MDL_BACKUP_WAIT_DDL Backup lock MDL_SHARED_WRITE Table metadata lock test t1 @@ -130,7 +138,8 @@ connection con1; DROP TABLE t1; connection con2; connection con2; -SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; +SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info +WHERE TABLE_NAME NOT LIKE 'innodb_%_stats'; LOCK_MODE LOCK_TYPE TABLE_SCHEMA TABLE_NAME MDL_BACKUP_WAIT_DDL Backup lock SELECT * FROM t1; diff --git a/mysql-test/main/backup_lock.test b/mysql-test/main/backup_lock.test index f51b6ecdaad..9fdc42b54ea 100644 --- a/mysql-test/main/backup_lock.test +++ b/mysql-test/main/backup_lock.test @@ -10,16 +10,19 @@ --echo # Testing which locks we get from all stages --echo # +let $mdl= LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info +WHERE TABLE_NAME NOT LIKE 'innodb_%_stats'; + BACKUP STAGE START; -SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; +eval SELECT $mdl; BACKUP STAGE FLUSH; -SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; +eval SELECT $mdl; BACKUP STAGE BLOCK_DDL; -SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; +eval SELECT $mdl; BACKUP STAGE BLOCK_COMMIT; -SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; +eval SELECT $mdl; BACKUP STAGE END; -SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; +eval SELECT $mdl; --echo # --echo # testing BACKUP STAGE LOCK's @@ -54,7 +57,7 @@ let $wait_condition= where state = "Waiting for table metadata lock"; --source include/wait_condition.inc backup stage flush; -SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; +eval SELECT $mdl; # # Do first test with max_statement_time, other tests later are done with # lock_wait_timeout. This is mostly to ensure that both methods works @@ -103,7 +106,7 @@ let $wait_condition= --source include/wait_condition.inc backup stage start; backup stage flush; -SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; +eval SELECT $mdl; backup stage block_ddl; backup stage block_commit; connection default; @@ -146,7 +149,7 @@ let $wait_condition= SET STATEMENT lock_wait_timeout=0 FOR SELECT * FROM t1; backup stage block_ddl; -SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; +eval SELECT $mdl; backup stage end; connection default; @@ -170,7 +173,7 @@ let $wait_condition= where state = "Waiting for backup lock"; --source include/wait_condition.inc connection con2; -SELECT LOCK_MODE, LOCK_TYPE, TABLE_SCHEMA, TABLE_NAME FROM information_schema.metadata_lock_info; +eval SELECT $mdl; # Check that select's are not blocked SELECT * FROM t1; connection default; -- cgit v1.2.1 From a1b0f2358620335e9495242270cf47a1366c9a6a Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 4 Sep 2021 17:38:47 +0200 Subject: MDEV-26533 MariaDB 10.5 crashes with key_buffer_size > 4Gb on Windows x64 This is a side-effect of my_large_malloc() introduction,MDEV-18851 It removed a cast to size_t to variable 'blocks' in multiplication blocks * keycache->key_cache_block_size , creating ulong value instead of correct size_t. Replaced a couple of ulongs with appropriate data type, which is size_t. Also, fixed casts to ulongs in crash handler messages, so that people would not be confused by that, too. Interestingly, aria did not expose the same problem even if it contains copied and pasted code in ma_pagecache, because Aria had some ulongs removed when fixing a similar problem in MDEV-9256. --- mysys/mf_keycache.c | 28 ++++++++++++++-------------- sql/signal_handler.cc | 19 +++++++++---------- 2 files changed, 23 insertions(+), 24 deletions(-) diff --git a/mysys/mf_keycache.c b/mysys/mf_keycache.c index ec5e39938b6..127bfff4af2 100644 --- a/mysys/mf_keycache.c +++ b/mysys/mf_keycache.c @@ -164,18 +164,18 @@ typedef struct st_simple_key_cache_cb size_t key_cache_mem_size; /* specified size of the cache memory */ size_t allocated_mem_size; /* size of the memory actually allocated */ uint key_cache_block_size; /* size of the page buffer of a cache block */ - ulong min_warm_blocks; /* min number of warm blocks; */ - ulong age_threshold; /* age threshold for hot blocks */ + size_t min_warm_blocks; /* min number of warm blocks; */ + size_t age_threshold; /* age threshold for hot blocks */ ulonglong keycache_time; /* total number of block link operations */ uint hash_entries; /* max number of entries in the hash table */ uint changed_blocks_hash_size; /* Number of hash buckets for file blocks */ int hash_links; /* max number of hash links */ int hash_links_used; /* number of hash links currently used */ int disk_blocks; /* max number of blocks in the cache */ - ulong blocks_used; /* maximum number of concurrently used blocks */ - ulong blocks_unused; /* number of currently unused blocks */ - ulong blocks_changed; /* number of currently dirty blocks */ - ulong warm_blocks; /* number of blocks in warm sub-chain */ + size_t blocks_used; /* maximum number of concurrently used blocks */ + size_t blocks_unused; /* number of currently unused blocks */ + size_t blocks_changed; /* number of currently dirty blocks */ + size_t warm_blocks; /* number of blocks in warm sub-chain */ ulong cnt_for_resize_op; /* counter to block resize operation */ long blocks_available; /* number of blocks available in the LRU chain */ HASH_LINK **hash_root; /* arr. of entries into hash table buckets */ @@ -478,7 +478,7 @@ int init_simple_key_cache(SIMPLE_KEY_CACHE_CB *keycache, size_t use_mem, uint division_limit, uint age_threshold, uint changed_blocks_hash_size) { - ulong blocks, hash_links; + size_t blocks, hash_links; size_t length; int error; DBUG_ENTER("init_simple_key_cache"); @@ -519,8 +519,8 @@ int init_simple_key_cache(SIMPLE_KEY_CACHE_CB *keycache, DBUG_PRINT("info", ("key_cache_block_size: %u", key_cache_block_size)); - blocks= (ulong) (use_mem / (sizeof(BLOCK_LINK) + 2 * sizeof(HASH_LINK) + - sizeof(HASH_LINK*) * 5/4 + key_cache_block_size)); + blocks= use_mem / (sizeof(BLOCK_LINK) + 2 * sizeof(HASH_LINK) + + sizeof(HASH_LINK*) * 5/4 + key_cache_block_size); /* Changed blocks hash needs to be a power of 2 */ changed_blocks_hash_size= my_round_up_to_next_power(MY_MAX(changed_blocks_hash_size, @@ -532,7 +532,7 @@ int init_simple_key_cache(SIMPLE_KEY_CACHE_CB *keycache, for ( ; ; ) { /* Set my_hash_entries to the next bigger 2 power */ - if ((keycache->hash_entries= next_power(blocks)) < blocks * 5/4) + if ((keycache->hash_entries= next_power((uint)blocks)) < blocks * 5/4) keycache->hash_entries<<= 1; hash_links= 2 * blocks; #if defined(MAX_THREADS) @@ -543,8 +543,8 @@ int init_simple_key_cache(SIMPLE_KEY_CACHE_CB *keycache, ALIGN_SIZE(hash_links * sizeof(HASH_LINK)) + ALIGN_SIZE(sizeof(HASH_LINK*) * keycache->hash_entries) + - sizeof(BLOCK_LINK*)* (changed_blocks_hash_size*2))) + - ((size_t) blocks * keycache->key_cache_block_size) > use_mem && blocks > 8) + sizeof(BLOCK_LINK*)* ((size_t)changed_blocks_hash_size*2))) + + (blocks * keycache->key_cache_block_size) > use_mem && blocks > 8) blocks--; keycache->allocated_mem_size= blocks * keycache->key_cache_block_size; if ((keycache->block_mem= my_large_malloc(&keycache->allocated_mem_size, @@ -584,7 +584,7 @@ int init_simple_key_cache(SIMPLE_KEY_CACHE_CB *keycache, } keycache->blocks_unused= blocks; keycache->disk_blocks= (int) blocks; - keycache->hash_links= hash_links; + keycache->hash_links= (int)hash_links; keycache->hash_links_used= 0; keycache->free_hash_list= NULL; keycache->blocks_used= keycache->blocks_changed= 0; @@ -4854,7 +4854,7 @@ static int cache_empty(SIMPLE_KEY_CACHE_CB *keycache) } if (errcnt) { - fprintf(stderr, "blocks: %d used: %lu\n", + fprintf(stderr, "blocks: %d used: %zu\n", keycache->disk_blocks, keycache->blocks_used); fprintf(stderr, "hash_links: %d used: %d\n", keycache->hash_links, keycache->hash_links_used); diff --git a/sql/signal_handler.cc b/sql/signal_handler.cc index 0cfe55fe254..eddcc632972 100644 --- a/sql/signal_handler.cc +++ b/sql/signal_handler.cc @@ -173,19 +173,19 @@ extern "C" sig_handler handle_fatal_signal(int sig) my_safe_printf_stderr("Server version: %s\n", server_version); if (dflt_key_cache) - my_safe_printf_stderr("key_buffer_size=%lu\n", - (ulong) dflt_key_cache->key_cache_mem_size); + my_safe_printf_stderr("key_buffer_size=%zu\n", + dflt_key_cache->key_cache_mem_size); - my_safe_printf_stderr("read_buffer_size=%ld\n", - (long) global_system_variables.read_buff_size); + my_safe_printf_stderr("read_buffer_size=%lu\n", + global_system_variables.read_buff_size); my_safe_printf_stderr("max_used_connections=%lu\n", - (ulong) max_used_connections); + max_used_connections); if (thread_scheduler) my_safe_printf_stderr("max_threads=%u\n", - (uint) thread_scheduler->max_threads + - (uint) extra_max_connections); + thread_scheduler->max_threads + + extra_max_connections); my_safe_printf_stderr("thread_count=%u\n", THD_count::value()); @@ -194,11 +194,10 @@ extern "C" sig_handler handle_fatal_signal(int sig) my_safe_printf_stderr("It is possible that mysqld could use up to \n" "key_buffer_size + " "(read_buffer_size + sort_buffer_size)*max_threads = " - "%lu K bytes of memory\n", - (ulong) + "%zu K bytes of memory\n", (dflt_key_cache->key_cache_mem_size + (global_system_variables.read_buff_size + - global_system_variables.sortbuff_size) * + (size_t)global_system_variables.sortbuff_size) * (thread_scheduler->max_threads + extra_max_connections) + (max_connections + extra_max_connections) * sizeof(THD)) / 1024); -- cgit v1.2.1 From 5ae5453291d0c0ca01a73c794b79e267dc1b44e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 4 Sep 2021 19:08:14 +0300 Subject: MDEV-25919 fixup: MSAN and Valgrind errors related to statistics dict_table_close(): Fix a race condition around dict_stats_deinit(). This was not observed; it should have been caught by an assertion. dict_stats_deinit(): Slightly simplify the code. ha_innobase::info_low(): If the table is unreadable, initialize some dummy statistics. --- storage/innobase/dict/dict0dict.cc | 6 ++++-- storage/innobase/handler/ha_innodb.cc | 5 +++++ storage/innobase/include/dict0stats.ic | 8 +++----- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 447ff7929f0..98c619cb7d3 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -225,7 +225,8 @@ void dict_table_close(dict_table_t *table) if (table->release()) { table->stats_mutex_lock(); - dict_stats_deinit(table); + if (table->get_ref_count() == 0) + dict_stats_deinit(table); table->stats_mutex_unlock(); } dict_sys.unlock(); @@ -258,7 +259,8 @@ dict_table_close( that FLUSH TABLE can be used to forcibly fetch stats from disk if they have been manually modified. */ table->stats_mutex_lock(); - dict_stats_deinit(table); + if (table->get_ref_count() == 0) + dict_stats_deinit(table); table->stats_mutex_unlock(); } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 0ce32554232..56d9444016a 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -14532,7 +14532,12 @@ ha_innobase::info_low( DBUG_ASSERT(ib_table->get_ref_count() > 0); if (!ib_table->is_readable()) { + ib_table->stats_mutex_lock(); ib_table->stat_initialized = true; + ib_table->stat_n_rows = 0; + ib_table->stat_clustered_index_size = 0; + ib_table->stat_sum_of_other_index_sizes = 0; + ib_table->stats_mutex_unlock(); } if (flag & HA_STATUS_TIME) { diff --git a/storage/innobase/include/dict0stats.ic b/storage/innobase/include/dict0stats.ic index e49153eb099..dd516275156 100644 --- a/storage/innobase/include/dict0stats.ic +++ b/storage/innobase/include/dict0stats.ic @@ -175,16 +175,13 @@ dict_stats_deinit( dict_table_t* table) /*!< in/out: table */ { ut_ad(table->stats_mutex_is_owner()); + ut_ad(table->get_ref_count() == 0); - ut_a(table->get_ref_count() == 0); - +#ifdef HAVE_valgrind if (!table->stat_initialized) { return; } - table->stat_initialized = FALSE; - -#ifdef HAVE_valgrind MEM_UNDEFINED(&table->stat_n_rows, sizeof table->stat_n_rows); MEM_UNDEFINED(&table->stat_clustered_index_size, sizeof table->stat_clustered_index_size); @@ -218,4 +215,5 @@ dict_stats_deinit( sizeof(index->stat_n_leaf_pages)); } #endif /* HAVE_valgrind */ + table->stat_initialized = FALSE; } -- cgit v1.2.1 From 4ffcfe7c2a8a977c0b4a1f044bf9384260967a27 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 4 Sep 2021 18:52:10 +0200 Subject: MDEV-26538 Incorrect error condition check for ReadFile (named pipes) --- sql/threadpool_winsockets.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/threadpool_winsockets.cc b/sql/threadpool_winsockets.cc index 6b4758a451f..41167781283 100644 --- a/sql/threadpool_winsockets.cc +++ b/sql/threadpool_winsockets.cc @@ -201,7 +201,7 @@ DWORD win_aiosocket::begin_read() else { /* Do async read (named pipe) */ - if (ReadFile(m_handle, buf.buf, buf.len, 0, &m_overlapped)) + if (!ReadFile(m_handle, buf.buf, buf.len, 0, &m_overlapped)) err= GetLastError(); } -- cgit v1.2.1 From ae85835cc7fa88bb80a282106c1e905d85157923 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sun, 5 Sep 2021 19:37:13 +0200 Subject: Fix warnings from -DPLUGIN_PARTITION=NO, portably. Also fix Spider's cmake. --- sql/datadict.cc | 4 +++- sql/handler.cc | 4 +++- storage/spider/CMakeLists.txt | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/sql/datadict.cc b/sql/datadict.cc index 5bde9f1cb79..783229a47a6 100644 --- a/sql/datadict.cc +++ b/sql/datadict.cc @@ -151,7 +151,9 @@ Table_type dd_frm_type(THD *thd, char *path, LEX_CSTRING *engine_name, } } -cont: __attribute__((unused)); +#ifdef WITH_PARTITION_STORAGE_ENGINE +cont: +#endif /* read the true engine name */ { MY_STAT state; diff --git a/sql/handler.cc b/sql/handler.cc index 0c75d58c8ea..7a220068a7c 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -6201,7 +6201,9 @@ bool ha_table_exists(THD *thd, const LEX_CSTRING *db, DBUG_RETURN(TRUE); } -retry_from_frm: __attribute__((unused)); +#ifdef WITH_PARTITION_STORAGE_ENGINE +retry_from_frm: +#endif char path[FN_REFLEN + 1]; size_t path_len = build_table_filename(path, sizeof(path) - 1, db->str, table_name->str, "", 0); diff --git a/storage/spider/CMakeLists.txt b/storage/spider/CMakeLists.txt index 397478bfc40..898421fa532 100644 --- a/storage/spider/CMakeLists.txt +++ b/storage/spider/CMakeLists.txt @@ -73,7 +73,7 @@ IF(ORACLE_INCLUDE_DIR AND ORACLE_OCI_LIBRARY) ENDIF() ENDIF() -IF(MSVC) +IF(MSVC AND (TARGET spider)) IF (CMAKE_BUILD_TYPE STREQUAL "Debug") ADD_CUSTOM_COMMAND(TARGET spider POST_BUILD -- cgit v1.2.1 From 12c3d1e1d7eee013f2f2ee75965803822a95d43e Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sun, 5 Sep 2021 20:22:39 +0200 Subject: Fix Windows warnings and tests for -DPLUGIN_PERFSCHEMA=NO --- mysql-test/main/mysql_install_db_win.result | 6 ------ mysql-test/main/mysql_upgrade_noengine.test | 1 + mysql-test/std_data/mysql_install_db_win.ini | 6 ------ sql/mdl.cc | 4 ++++ storage/innobase/include/btr0sea.h | 10 ++++++++++ 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/mysql-test/main/mysql_install_db_win.result b/mysql-test/main/mysql_install_db_win.result index 936165bd052..2440ff351ea 100644 --- a/mysql-test/main/mysql_install_db_win.result +++ b/mysql-test/main/mysql_install_db_win.result @@ -26,12 +26,6 @@ tmpdir=BASEDIR/temp innodb_data_file_path=ibdata1:10M;ibdata2:10M:autoextend innodb_log_group_home_dir=BASEDIR/data skip-name-resolve - -#********************************************************* -# performance_schema -#********************************************************* -performance_schema=OFF - [client] port=3307 # restart diff --git a/mysql-test/main/mysql_upgrade_noengine.test b/mysql-test/main/mysql_upgrade_noengine.test index 5b063fb3d8c..efe36c355ab 100644 --- a/mysql-test/main/mysql_upgrade_noengine.test +++ b/mysql-test/main/mysql_upgrade_noengine.test @@ -1,6 +1,7 @@ # # MDEV-11942 BLACKHOLE is no longer active in 10.1 by default, mysql_upgrade not handling the situation # +source include/mysql_upgrade_preparation.inc; source include/have_innodb.inc; source include/not_embedded.inc; diff --git a/mysql-test/std_data/mysql_install_db_win.ini b/mysql-test/std_data/mysql_install_db_win.ini index e9b5e7a84b0..c375bdb4bf3 100644 --- a/mysql-test/std_data/mysql_install_db_win.ini +++ b/mysql-test/std_data/mysql_install_db_win.ini @@ -10,9 +10,3 @@ tmpdir=BASEDIR/temp innodb_data_file_path=ibdata1:10M;ibdata2:10M:autoextend innodb_log_group_home_dir=BASEDIR/data skip-name-resolve - -#********************************************************* -# performance_schema -#********************************************************* -performance_schema=OFF - diff --git a/sql/mdl.cc b/sql/mdl.cc index 67ebc70d860..dc1d658b0b5 100644 --- a/sql/mdl.cc +++ b/sql/mdl.cc @@ -2336,11 +2336,13 @@ MDL_context::acquire_lock(MDL_request *mdl_request, double lock_wait_timeout) mysql_prlock_unlock(&lock->m_rwlock); +#ifdef HAVE_PSI_INTERFACE PSI_metadata_locker_state state __attribute__((unused)); PSI_metadata_locker *locker= NULL; if (ticket->m_psi != NULL) locker= PSI_CALL_start_metadata_wait(&state, ticket->m_psi, __FILE__, __LINE__); +#endif will_wait_for(ticket); @@ -2387,8 +2389,10 @@ MDL_context::acquire_lock(MDL_request *mdl_request, double lock_wait_timeout) done_waiting_for(); +#ifdef HAVE_PSI_INTERFACE if (locker != NULL) PSI_CALL_end_metadata_wait(locker, 0); +#endif if (wait_status != MDL_wait::GRANTED) { diff --git a/storage/innobase/include/btr0sea.h b/storage/innobase/include/btr0sea.h index da7bbac1f83..31ca4ba53dd 100644 --- a/storage/innobase/include/btr0sea.h +++ b/storage/innobase/include/btr0sea.h @@ -243,10 +243,20 @@ struct btr_search_sys_t /** memory heap for table */ mem_heap_t *heap; +#ifdef _MSC_VER +#pragma warning(push) +// nonstandard extension - zero sized array, if perfschema is not compiled +#pragma warning(disable : 4200) +#endif + char pad[(CPU_LEVEL1_DCACHE_LINESIZE - sizeof(srw_lock) - sizeof(hash_table_t) - sizeof(mem_heap_t)) & (CPU_LEVEL1_DCACHE_LINESIZE - 1)]; +#ifdef _MSC_VER +#pragma warning(pop) +#endif + void init() { memset((void*) this, 0, sizeof *this); -- cgit v1.2.1 From b9e2002702a4b40bf68252256761b219d9dcf4ad Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 5 Sep 2021 11:53:36 +0200 Subject: cleanup: only do work when needed --- sql/item.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sql/item.cc b/sql/item.cc index a7975bfbd7a..f8237748bc8 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -2384,10 +2384,6 @@ left_is_superset(const DTCollation *left, const DTCollation *right) bool DTCollation::aggregate(const DTCollation &dt, uint flags) { - - THD *thd = current_thd; - myf utf8_flag= thd ? thd->get_utf8_flag() : - global_system_variables.old_behavior & OLD_MODE_UTF8_IS_UTF8MB3; if (!my_charset_same(collation, dt.collation)) { /* @@ -2475,6 +2471,9 @@ bool DTCollation::aggregate(const DTCollation &dt, uint flags) set(dt); return 0; } + THD *thd = current_thd; + myf utf8_flag= thd ? thd->get_utf8_flag() + : global_system_variables.old_behavior & OLD_MODE_UTF8_IS_UTF8MB3; CHARSET_INFO *bin= get_charset_by_csname(collation->cs_name.str, MY_CS_BINSORT,MYF(utf8_flag)); set(bin, DERIVATION_NONE); -- cgit v1.2.1 From 4c1ed54bfcb1ac08eb9b3221fba563ff55ac8f86 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 5 Sep 2021 13:09:02 +0200 Subject: fix Binary_string::c_ptr and c_ptr_safe if the Ptr="abc", then str_length=3, and for a C ptr it needs Ptr[3]=0; but it passes str_length+1 (=4) to realloc, and realloc allocates arg_length+1 bytes (that is 5) and does Ptr[arg_length]= 0; (Ptr[4]=0) --- sql/sql_string.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/sql_string.h b/sql/sql_string.h index d7661605492..fe57c8153bb 100644 --- a/sql/sql_string.h +++ b/sql/sql_string.h @@ -643,7 +643,7 @@ public: Ptr[str_length]=0; return Ptr; } - (void) realloc(str_length+1); /* This will add end \0 */ + (void) realloc(str_length); /* This will add end \0 */ return Ptr; } /* @@ -666,7 +666,7 @@ public: if (Ptr && str_length < Alloced_length) Ptr[str_length]=0; else - (void) realloc(str_length + 1); + (void) realloc(str_length); return Ptr; } -- cgit v1.2.1 From 1fda0544b9146ed608a59974c783ab1350b1dcfd Mon Sep 17 00:00:00 2001 From: Nayuta Yanagisawa Date: Wed, 11 Aug 2021 10:27:20 +0000 Subject: MDEV-25684 Crash in THD::find_temporary_table while calling spider_direct_sql UDF without temporary table created The server crashed when SPIDER_DIRECT_SQL UDF was called with non-existing temporary table. The bug has been introduced by 91ffdc8. The commit removed the check, from THD::open_temporary_table(), which ensure that the target temporary tables exist. We can fix the bug by adding the check before the call of THD::open_temporary_table(). --- .../mysql-test/spider/bugfix/r/mdev_25684.result | 18 ++++++++++++++++++ .../spider/mysql-test/spider/bugfix/t/mdev_25684.cnf | 3 +++ .../spider/mysql-test/spider/bugfix/t/mdev_25684.test | 19 +++++++++++++++++++ storage/spider/spd_sys_table.cc | 4 ++++ 4 files changed, 44 insertions(+) create mode 100644 storage/spider/mysql-test/spider/bugfix/r/mdev_25684.result create mode 100644 storage/spider/mysql-test/spider/bugfix/t/mdev_25684.cnf create mode 100644 storage/spider/mysql-test/spider/bugfix/t/mdev_25684.test diff --git a/storage/spider/mysql-test/spider/bugfix/r/mdev_25684.result b/storage/spider/mysql-test/spider/bugfix/r/mdev_25684.result new file mode 100644 index 00000000000..8d4bea79476 --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/r/mdev_25684.result @@ -0,0 +1,18 @@ +for master_1 +for child2 +child2_1 +child2_2 +child2_3 +for child3 + +MDEV-25684 Crash in THD::find_temporary_table while calling spider_direct_sql UDF without temporary table created + +connection master_1; +SELECT SPIDER_DIRECT_SQL('SELECT * FROM s', 'non_existing_temporary_table', 'srv "s_2_1"'); +ERROR HY000: Temporary table 'test.non_existing_temporary_table' is not found +for master_1 +for child2 +child2_1 +child2_2 +child2_3 +for child3 diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_25684.cnf b/storage/spider/mysql-test/spider/bugfix/t/mdev_25684.cnf new file mode 100644 index 00000000000..05dfd8a0bce --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_25684.cnf @@ -0,0 +1,3 @@ +!include include/default_mysqld.cnf +!include ../my_1_1.cnf +!include ../my_2_1.cnf diff --git a/storage/spider/mysql-test/spider/bugfix/t/mdev_25684.test b/storage/spider/mysql-test/spider/bugfix/t/mdev_25684.test new file mode 100644 index 00000000000..243031e52dd --- /dev/null +++ b/storage/spider/mysql-test/spider/bugfix/t/mdev_25684.test @@ -0,0 +1,19 @@ +--disable_query_log +--disable_result_log +--source ../../t/test_init.inc +--enable_result_log +--enable_query_log + +--echo +--echo MDEV-25684 Crash in THD::find_temporary_table while calling spider_direct_sql UDF without temporary table created +--echo + +--connection master_1 +--error 12703 +SELECT SPIDER_DIRECT_SQL('SELECT * FROM s', 'non_existing_temporary_table', 'srv "s_2_1"'); + +--disable_query_log +--disable_result_log +--source ../../t/test_deinit.inc +--enable_result_log +--enable_query_log diff --git a/storage/spider/spd_sys_table.cc b/storage/spider/spd_sys_table.cc index 0ed8640eb48..a467c2c00fa 100644 --- a/storage/spider/spd_sys_table.cc +++ b/storage/spider/spd_sys_table.cc @@ -3818,6 +3818,10 @@ TABLE *spider_find_temporary_table( ) { DBUG_ENTER("spider_find_temporary_table"); #ifdef SPIDER_open_temporary_table + if (!thd->has_temporary_tables()) + { + DBUG_RETURN(NULL); + } if (thd->open_temporary_table(table_list)) { DBUG_RETURN(NULL); -- cgit v1.2.1 From 7d351f1aa007244ec7b730500018221f4a6c294b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 6 Sep 2021 10:14:17 +0300 Subject: MDEV-26533 fixup: GCC -Wformat --- sql/signal_handler.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/signal_handler.cc b/sql/signal_handler.cc index eddcc632972..e5fda378f15 100644 --- a/sql/signal_handler.cc +++ b/sql/signal_handler.cc @@ -1,5 +1,5 @@ /* Copyright (c) 2011, 2012, Oracle and/or its affiliates. - Copyright (c) 2011, 2014, SkySQL Ab. + Copyright (c) 2011, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -183,7 +183,7 @@ extern "C" sig_handler handle_fatal_signal(int sig) max_used_connections); if (thread_scheduler) - my_safe_printf_stderr("max_threads=%u\n", + my_safe_printf_stderr("max_threads=%lu\n", thread_scheduler->max_threads + extra_max_connections); -- cgit v1.2.1 From 84c578c7952161fe5068fea003f0f8b1d62caa89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 6 Sep 2021 10:14:24 +0300 Subject: MDEV-26547 Restoring InnoDB buffer pool dump is single-threaded for no reason buf_read_page_background(): Remove the parameter "bool sync" and always actually initiate a page read in the background. buf_load(): Always submit asynchronous reads. This allows page checksums to be verified in concurrent threads as soon as the reads are completed. --- storage/innobase/btr/btr0cur.cc | 16 ++++++++-------- storage/innobase/buf/buf0dump.cc | 4 ++-- storage/innobase/buf/buf0rea.cc | 7 +++---- storage/innobase/include/buf0rea.h | 7 +++---- 4 files changed, 16 insertions(+), 18 deletions(-) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index a3351800231..cfff322f547 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -3341,16 +3341,16 @@ static void btr_cur_prefetch_siblings(const buf_block_t *block, uint32_t prev= mach_read_from_4(my_assume_aligned<4>(page + FIL_PAGE_PREV)); uint32_t next= mach_read_from_4(my_assume_aligned<4>(page + FIL_PAGE_NEXT)); + fil_space_t *space= index->table->space; + if (prev == FIL_NULL); - else if (index->table->space->acquire()) - buf_read_page_background(index->table->space, - page_id_t(block->page.id().space(), prev), - block->zip_size(), false); + else if (space->acquire()) + buf_read_page_background(space, page_id_t(space->id, prev), + block->zip_size()); if (next == FIL_NULL); - else if (index->table->space->acquire()) - buf_read_page_background(index->table->space, - page_id_t(block->page.id().space(), next), - block->zip_size(), false); + else if (space->acquire()) + buf_read_page_background(space, page_id_t(space->id, next), + block->zip_size()); } /*************************************************************//** diff --git a/storage/innobase/buf/buf0dump.cc b/storage/innobase/buf/buf0dump.cc index c6ddcb4fa48..2e6365eb39f 100644 --- a/storage/innobase/buf/buf0dump.cc +++ b/storage/innobase/buf/buf0dump.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2020, MariaDB Corporation. +Copyright (c) 2017, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -678,7 +678,7 @@ buf_load() } space->reacquire(); - buf_read_page_background(space, dump[i], zip_size, true); + buf_read_page_background(space, dump[i], zip_size); if (buf_load_abort_flag) { if (space) { diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index 42b3f674f81..2303ef625e7 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -501,14 +501,13 @@ an exclusive lock on the buffer frame. The flag is cleared and the x-lock released by the i/o-handler thread. @param[in,out] space tablespace @param[in] page_id page id -@param[in] zip_size ROW_FORMAT=COMPRESSED page size, or 0 -@param[in] sync true if synchronous aio is desired */ +@param[in] zip_size ROW_FORMAT=COMPRESSED page size, or 0 */ void buf_read_page_background(fil_space_t *space, const page_id_t page_id, - ulint zip_size, bool sync) + ulint zip_size) { dberr_t err; - if (buf_read_page_low(&err, space, sync, BUF_READ_ANY_PAGE, + if (buf_read_page_low(&err, space, false, BUF_READ_ANY_PAGE, page_id, zip_size, false)) { srv_stats.buf_pool_reads.add(1); } diff --git a/storage/innobase/include/buf0rea.h b/storage/innobase/include/buf0rea.h index 87c6b5d7e75..8d6b28194dc 100644 --- a/storage/innobase/include/buf0rea.h +++ b/storage/innobase/include/buf0rea.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2020, MariaDB Corporation. +Copyright (c) 2015, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -48,10 +48,9 @@ an exclusive lock on the buffer frame. The flag is cleared and the x-lock released by the i/o-handler thread. @param[in,out] space tablespace @param[in] page_id page id -@param[in] zip_size ROW_FORMAT=COMPRESSED page size, or 0 -@param[in] sync true if synchronous aio is desired */ +@param[in] zip_size ROW_FORMAT=COMPRESSED page size, or 0 */ void buf_read_page_background(fil_space_t *space, const page_id_t page_id, - ulint zip_size, bool sync) + ulint zip_size) MY_ATTRIBUTE((nonnull)); /** Applies a random read-ahead in buf_pool if there are at least a threshold -- cgit v1.2.1 From a73eedbf3fabd19ca7183b738056c30e3f7bbe35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 6 Sep 2021 12:16:26 +0300 Subject: MDEV-26467 Unnecessary compare-and-swap loop in srw_mutex srw_mutex::wait_and_lock(): In the spin loop, we will try to poll for non-conflicting lock word state by reads, avoiding any writes. We invoke explicit std::atomic_thread_fence(std::memory_order_acquire) before returning. The individual operations on the lock word can use memory_order_relaxed. srw_mutex::lock: Document that the value for a single writer is HOLDER+1 instead of HOLDER. srw_mutex::wr_lock_try(), srw_mutex::wr_unlock(): Adjust the value of the lock word of a single writer from HOLDER to HOLDER+1. --- storage/innobase/include/srw_lock.h | 9 ++++---- storage/innobase/sync/srw_lock.cc | 42 +++++++++++++++++++++---------------- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/storage/innobase/include/srw_lock.h b/storage/innobase/include/srw_lock.h index fdac659d494..fa3368d5c1b 100644 --- a/storage/innobase/include/srw_lock.h +++ b/storage/innobase/include/srw_lock.h @@ -36,7 +36,8 @@ public: /** Futex-based mutex */ class srw_mutex final { - /** The lock word, containing HOLDER and a count of waiters */ + /** The lock word, containing HOLDER + 1 if the lock is being held, + plus the number of waiters */ std::atomic lock; /** Identifies that the lock is being held */ static constexpr uint32_t HOLDER= 1U << 31; @@ -62,7 +63,7 @@ public: bool wr_lock_try() { uint32_t lk= 0; - return lock.compare_exchange_strong(lk, HOLDER, + return lock.compare_exchange_strong(lk, HOLDER + 1, std::memory_order_acquire, std::memory_order_relaxed); } @@ -70,8 +71,8 @@ public: void wr_lock() { if (!wr_lock_try()) wait_and_lock(); } void wr_unlock() { - const uint32_t lk= lock.fetch_and(~HOLDER, std::memory_order_release); - if (lk != HOLDER) + const uint32_t lk= lock.fetch_sub(HOLDER + 1, std::memory_order_release); + if (lk != HOLDER + 1) { DBUG_ASSERT(lk & HOLDER); wake(); diff --git a/storage/innobase/sync/srw_lock.cc b/storage/innobase/sync/srw_lock.cc index b76194c89e3..780add3f705 100644 --- a/storage/innobase/sync/srw_lock.cc +++ b/storage/innobase/sync/srw_lock.cc @@ -233,33 +233,39 @@ void ssux_lock_low::wake() { SRW_FUTEX(&readers, WAKE, 1); } void srw_mutex::wait_and_lock() { uint32_t lk= 1 + lock.fetch_add(1, std::memory_order_relaxed); - for (auto spin= srv_n_spin_wait_rounds; spin; spin--) + for (auto spin= srv_n_spin_wait_rounds;;) { - lk&= ~HOLDER; - DBUG_ASSERT(lk); - while (!lock.compare_exchange_weak(lk, HOLDER | (lk - 1), - std::memory_order_acquire, - std::memory_order_relaxed)) - if (lk & HOLDER) - goto occupied; - return; -occupied: + DBUG_ASSERT(~HOLDER & lk); + if (lk & HOLDER) + lk= lock.load(std::memory_order_relaxed); + else + { + lk= lock.fetch_or(HOLDER, std::memory_order_relaxed); + if (!(lk & HOLDER)) + goto acquired; + } ut_delay(srv_spin_wait_delay); + if (!--spin) + break; } - for (;;) + for (;; wait(lk)) { - lk= lock.load(std::memory_order_relaxed); - while (!(lk & HOLDER)) + if (lk & HOLDER) { + lk= lock.load(std::memory_order_relaxed); + if (lk & HOLDER) + continue; + } + lk= lock.fetch_or(HOLDER, std::memory_order_relaxed); + if (!(lk & HOLDER)) + { +acquired: DBUG_ASSERT(lk); - if (lock.compare_exchange_weak(lk, HOLDER | (lk - 1), - std::memory_order_acquire, - std::memory_order_relaxed)) - return; + std::atomic_thread_fence(std::memory_order_acquire); + return; } DBUG_ASSERT(lk > HOLDER); - wait(lk); } } -- cgit v1.2.1 From 0f0b7e47bc794d0af0b7f758a6fe1518b8388e3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 6 Sep 2021 12:22:33 +0300 Subject: MDEV-26467: Avoid re-reading srv_spin_wait_delay inside a loop Invoking ut_delay(srv_wpin_wait_delay) inside a spinloop would cause a read of 2 global variables as well as multiplication. Let us loop around MY_RELAX_CPU() using a precomputed loop count to keep the loops simpler, to help them scale better. We also tried precomputing the delay into a global variable, but that appeared to result in slightly worse throughput. --- storage/innobase/buf/buf0buf.cc | 4 ++-- storage/innobase/sync/srw_lock.cc | 50 +++++++++++++++++++++++++++++---------- storage/innobase/trx/trx0trx.cc | 2 +- 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 58820c7b906..426b21a8308 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -282,7 +282,7 @@ void page_hash_latch::read_lock_wait() /* First, try busy spinning for a while. */ for (auto spin= srv_n_spin_wait_rounds; spin--; ) { - ut_delay(srv_spin_wait_delay); + LF_BACKOFF(); if (read_trylock()) return; } @@ -301,7 +301,7 @@ void page_hash_latch::write_lock_wait() { if (write_lock_poll()) return; - ut_delay(srv_spin_wait_delay); + LF_BACKOFF(); } /* Fall back to yielding to other threads. */ diff --git a/storage/innobase/sync/srw_lock.cc b/storage/innobase/sync/srw_lock.cc index 780add3f705..b267969c3df 100644 --- a/storage/innobase/sync/srw_lock.cc +++ b/storage/innobase/sync/srw_lock.cc @@ -20,6 +20,21 @@ this program; if not, write to the Free Software Foundation, Inc., #include "srv0srv.h" #include "my_cpu.h" +/** @return the parameter for srw_pause() */ +static inline unsigned srw_pause_delay() +{ + return my_cpu_relax_multiplier / 4 * srv_spin_wait_delay; +} + +/** Pause the CPU for some time, with no memory accesses. */ +static inline void srw_pause(unsigned delay) +{ + HMT_low(); + while (delay--) + MY_RELAX_CPU(); + HMT_medium(); +} + #ifdef SUX_LOCK_GENERIC void ssux_lock_low::init() { @@ -90,15 +105,17 @@ void ssux_lock_low::read_lock(uint32_t l) pthread_mutex_unlock(&mutex); continue; } - else - for (auto spin= srv_n_spin_wait_rounds; spin; spin--) - { - ut_delay(srv_spin_wait_delay); - if (read_trylock(l)) - return; - else if (l == WRITER_WAITING) - goto wake_writer; - } + + const unsigned delay= srw_pause_delay(); + + for (auto spin= srv_n_spin_wait_rounds; spin; spin--) + { + srw_pause(delay); + if (read_trylock(l)) + return; + else if (l == WRITER_WAITING) + goto wake_writer; + } readers_wait(l); } @@ -128,14 +145,18 @@ void ssux_lock_low::update_lock(uint32_t l) continue; } else + { + const unsigned delay= srw_pause_delay(); + for (auto spin= srv_n_spin_wait_rounds; spin; spin--) { - ut_delay(srv_spin_wait_delay); + srw_pause(delay); if (update_trylock(l)) return; else if ((l | UPDATER) == (UPDATER | WRITER_WAITING)) goto wake_writer; } + } readers_wait(l); } @@ -146,6 +167,8 @@ void ssux_lock_low::update_lock(uint32_t l) @param holding_u whether we already hold u_lock() */ void ssux_lock_low::write_lock(bool holding_u) { + const unsigned delay= srw_pause_delay(); + for (;;) { uint32_t l= write_lock_wait_start(); @@ -157,7 +180,7 @@ void ssux_lock_low::write_lock(bool holding_u) return; if (!(l & WRITER_WAITING)) l= write_lock_wait_start(); - ut_delay(srv_spin_wait_delay); + srw_pause(delay); } const uint32_t e= holding_u ? WRITER_WAITING | UPDATER : WRITER_WAITING; @@ -233,6 +256,9 @@ void ssux_lock_low::wake() { SRW_FUTEX(&readers, WAKE, 1); } void srw_mutex::wait_and_lock() { uint32_t lk= 1 + lock.fetch_add(1, std::memory_order_relaxed); + + const unsigned delay= srw_pause_delay(); + for (auto spin= srv_n_spin_wait_rounds;;) { DBUG_ASSERT(~HOLDER & lk); @@ -244,7 +270,7 @@ void srw_mutex::wait_and_lock() if (!(lk & HOLDER)) goto acquired; } - ut_delay(srv_spin_wait_delay); + srw_pause(delay); if (!--spin) break; } diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index bb0b9882419..ad49d3e9c8e 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -1275,7 +1275,7 @@ inline void trx_t::commit_in_memory(const mtr_t *mtr) /* Wait for any implicit-to-explicit lock conversions to cease, so that there will be no race condition in lock_release(). */ while (UNIV_UNLIKELY(is_referenced())) - ut_delay(srv_spin_wait_delay); + LF_BACKOFF(); } else ut_ad(read_only || !rsegs.m_redo.rseg); -- cgit v1.2.1 From 277ba134ad1c994bda50de574a87a06a071fbecb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 6 Sep 2021 12:32:24 +0300 Subject: MDEV-26467: Avoid futile spin loops Typically, index_lock and fil_space_t::latch will be held for a longer time than the spin loop in latch acquisition would be waiting for. Let us avoid spin loops for those as well as dict_sys.latch, which could be held in exclusive mode for a longer time (while loading metadata into the buffer pool and the dictionary cache). Performance testing on a dual Intel Xeon E5-2630 v4 (2 NUMA nodes) suggests that the buffer pool page latch (block_lock) benefits from a spin loop in both read-only and read-write workloads where the working set is slightly larger than the buffer pool. Presumably, most contention would occur on leaf page latches. Contention on upper level pages in the buffer pool should intuitively last longer. We introduce srw_spin_lock and srw_spin_mutex to allow users of srw_lock or srw_mutex to opt in for the spin loop. On Microsoft Windows, a spin loop variant was and will not be available; srw_mutex and srw_lock will simply wrap SRWLOCK. That is, on Microsoft Windows, the parameters innodb_sync_spin_loops and innodb_spin_wait_delay will only affect block_lock. --- storage/innobase/btr/btr0cur.cc | 6 +- storage/innobase/btr/btr0sea.cc | 12 +-- storage/innobase/include/btr0cur.h | 4 +- storage/innobase/include/btr0pcur.h | 2 +- storage/innobase/include/btr0pcur.ic | 4 +- storage/innobase/include/btr0sea.h | 16 +-- storage/innobase/include/dict0mem.h | 4 +- storage/innobase/include/lock0lock.h | 4 +- storage/innobase/include/rw_lock.h | 4 +- storage/innobase/include/srw_lock.h | 50 +++++++-- storage/innobase/include/sux_lock.h | 44 ++++---- storage/innobase/include/trx0purge.h | 2 +- storage/innobase/include/trx0rseg.h | 2 +- storage/innobase/include/trx0trx.h | 2 +- storage/innobase/row/row0sel.cc | 2 +- storage/innobase/sync/srw_lock.cc | 166 +++++++++++++++++++++-------- storage/innobase/unittest/innodb_sync-t.cc | 4 +- 17 files changed, 213 insertions(+), 115 deletions(-) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 3a041ba11db..e533f93b199 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -1247,7 +1247,7 @@ btr_cur_search_to_nth_level_func( btr_cur_t* cursor, /*!< in/out: tree cursor; the cursor page is s- or x-latched, but see also above! */ #ifdef BTR_CUR_HASH_ADAPT - srw_lock* ahi_latch, + srw_spin_lock* ahi_latch, /*!< in: currently held AHI rdlock, or NULL */ #endif /* BTR_CUR_HASH_ADAPT */ mtr_t* mtr, /*!< in: mtr */ @@ -3611,7 +3611,7 @@ fail_err: ut_ad(flags == BTR_NO_LOCKING_FLAG); } else if (index->table->is_temporary()) { } else { - srw_lock* ahi_latch = btr_search_sys.get_latch(*index); + srw_spin_lock* ahi_latch = btr_search_sys.get_latch(*index); if (!reorg && cursor->flag == BTR_CUR_HASH) { btr_search_update_hash_node_on_insert( cursor, ahi_latch); @@ -4331,7 +4331,7 @@ btr_cur_update_in_place( #ifdef BTR_CUR_HASH_ADAPT { - srw_lock* ahi_latch = block->index + srw_spin_lock* ahi_latch = block->index ? btr_search_sys.get_latch(*index) : NULL; if (ahi_latch) { /* TO DO: Can we skip this if none of the fields diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 6fb4a01951a..b95fbbe694a 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -1015,7 +1015,7 @@ btr_search_guess_on_hash( ulint mode, ulint latch_mode, btr_cur_t* cursor, - srw_lock* ahi_latch, + srw_spin_lock* ahi_latch, mtr_t* mtr) { ulint fold; @@ -1460,7 +1460,7 @@ void btr_search_build_page_hash_index( dict_index_t* index, buf_block_t* block, - srw_lock* ahi_latch, + srw_spin_lock* ahi_latch, uint16_t n_fields, uint16_t n_bytes, bool left_side) @@ -1660,7 +1660,7 @@ exit_func: @param[in,out] cursor cursor which was just positioned */ void btr_search_info_update_slow(btr_search_t *info, btr_cur_t *cursor) { - srw_lock* ahi_latch = &btr_search_sys.get_part(*cursor->index) + srw_spin_lock* ahi_latch = &btr_search_sys.get_part(*cursor->index) ->latch; buf_block_t* block = btr_cur_get_block(cursor); @@ -1727,7 +1727,7 @@ btr_search_move_or_delete_hash_entries( assert_block_ahi_valid(block); assert_block_ahi_valid(new_block); - srw_lock* ahi_latch = index + srw_spin_lock* ahi_latch = index ? &btr_search_sys.get_part(*index)->latch : nullptr; @@ -1852,7 +1852,7 @@ void btr_search_update_hash_on_delete(btr_cur_t *cursor) inserted next to the cursor. @param[in] ahi_latch the adaptive hash index latch */ void btr_search_update_hash_node_on_insert(btr_cur_t *cursor, - srw_lock *ahi_latch) + srw_spin_lock *ahi_latch) { buf_block_t* block; dict_index_t* index; @@ -1925,7 +1925,7 @@ func_exit: to the cursor @param[in] ahi_latch the adaptive hash index latch */ void btr_search_update_hash_on_insert(btr_cur_t *cursor, - srw_lock *ahi_latch) + srw_spin_lock *ahi_latch) { buf_block_t* block; dict_index_t* index; diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index 6668fb6ca67..c7f25aff4b7 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2019, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, 2020, MariaDB Corporation. +Copyright (c) 2017, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -201,7 +201,7 @@ btr_cur_search_to_nth_level_func( btr_cur_t* cursor, /*!< in/out: tree cursor; the cursor page is s- or x-latched, but see also above! */ #ifdef BTR_CUR_HASH_ADAPT - srw_lock* ahi_latch, + srw_spin_lock* ahi_latch, /*!< in: currently held AHI rdlock, or NULL */ #endif /* BTR_CUR_HASH_ADAPT */ mtr_t* mtr, /*!< in/out: mini-transaction */ diff --git a/storage/innobase/include/btr0pcur.h b/storage/innobase/include/btr0pcur.h index 312e51971bb..62f82632c62 100644 --- a/storage/innobase/include/btr0pcur.h +++ b/storage/innobase/include/btr0pcur.h @@ -135,7 +135,7 @@ btr_pcur_open_with_no_init_func( that the ahi_latch protects the record! */ btr_pcur_t* cursor, /*!< in: memory buffer for persistent cursor */ #ifdef BTR_CUR_HASH_ADAPT - srw_lock* ahi_latch, + srw_spin_lock* ahi_latch, /*!< in: currently held AHI rdlock, or NULL */ #endif /* BTR_CUR_HASH_ADAPT */ mtr_t* mtr); /*!< in: mtr */ diff --git a/storage/innobase/include/btr0pcur.ic b/storage/innobase/include/btr0pcur.ic index 5840c34eecb..3853db88a8e 100644 --- a/storage/innobase/include/btr0pcur.ic +++ b/storage/innobase/include/btr0pcur.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2020, MariaDB Corporation. +Copyright (c) 2015, 2021, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -479,7 +479,7 @@ btr_pcur_open_with_no_init_func( that the ahi_latch protects the record! */ btr_pcur_t* cursor, /*!< in: memory buffer for persistent cursor */ #ifdef BTR_CUR_HASH_ADAPT - srw_lock* ahi_latch, + srw_spin_lock* ahi_latch, /*!< in: currently held AHI rdlock, or NULL */ #endif /* BTR_CUR_HASH_ADAPT */ mtr_t* mtr) /*!< in: mtr */ diff --git a/storage/innobase/include/btr0sea.h b/storage/innobase/include/btr0sea.h index 31ca4ba53dd..4339c895400 100644 --- a/storage/innobase/include/btr0sea.h +++ b/storage/innobase/include/btr0sea.h @@ -81,7 +81,7 @@ btr_search_guess_on_hash( ulint mode, ulint latch_mode, btr_cur_t* cursor, - srw_lock* ahi_latch, + srw_spin_lock* ahi_latch, mtr_t* mtr); /** Move or delete hash entries for moved records, usually in a page split. @@ -114,7 +114,7 @@ void btr_search_drop_page_hash_when_freed(const page_id_t page_id); inserted next to the cursor. @param[in] ahi_latch the adaptive hash index latch */ void btr_search_update_hash_node_on_insert(btr_cur_t *cursor, - srw_lock *ahi_latch); + srw_spin_lock *ahi_latch); /** Updates the page hash index when a single record is inserted on a page. @param[in,out] cursor cursor which was positioned to the @@ -123,7 +123,7 @@ void btr_search_update_hash_node_on_insert(btr_cur_t *cursor, to the cursor @param[in] ahi_latch the adaptive hash index latch */ void btr_search_update_hash_on_insert(btr_cur_t *cursor, - srw_lock *ahi_latch); + srw_spin_lock *ahi_latch); /** Updates the page hash index when a single record is deleted from a page. @param[in] cursor cursor which was positioned on the record to delete @@ -237,7 +237,7 @@ struct btr_search_sys_t struct partition { /** latches protecting hash_table */ - srw_lock latch; + srw_spin_lock latch; /** mapping of dtuple_fold() to rec_t* in buf_block_t::frame */ hash_table_t table; /** memory heap for table */ @@ -249,8 +249,8 @@ struct btr_search_sys_t #pragma warning(disable : 4200) #endif - char pad[(CPU_LEVEL1_DCACHE_LINESIZE - sizeof(srw_lock) - - sizeof(hash_table_t) - sizeof(mem_heap_t)) & + char pad[(CPU_LEVEL1_DCACHE_LINESIZE - sizeof latch - + sizeof table - sizeof heap) & (CPU_LEVEL1_DCACHE_LINESIZE - 1)]; #ifdef _MSC_VER @@ -306,7 +306,7 @@ struct btr_search_sys_t } /** Get the search latch for the adaptive hash index partition */ - srw_lock *get_latch(const dict_index_t &index) const + srw_spin_lock *get_latch(const dict_index_t &index) const { return &get_part(index)->latch; } /** Create and initialize at startup */ @@ -351,7 +351,7 @@ inline ulint dict_index_t::n_ahi_pages() const { if (!btr_search_enabled) return 0; - srw_lock *latch= &btr_search_sys.get_part(*this)->latch; + srw_spin_lock *latch= &btr_search_sys.get_part(*this)->latch; latch->rd_lock(SRW_LOCK_CALL); ulint ref_count= search_info->ref_count; latch->rd_unlock(); diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index 417a65f1b09..04debaa7869 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -2268,10 +2268,10 @@ public: lock_t* autoinc_lock; /** Mutex protecting autoinc and freed_indexes. */ - srw_mutex autoinc_mutex; + srw_spin_mutex autoinc_mutex; private: /** Mutex protecting locks on this table. */ - srw_mutex lock_mutex; + srw_spin_mutex lock_mutex; #ifdef UNIV_DEBUG /** The owner of lock_mutex (0 if none) */ Atomic_relaxed lock_mutex_owner{0}; diff --git a/storage/innobase/include/lock0lock.h b/storage/innobase/include/lock0lock.h index 42e8bf4ad22..859441afcc0 100644 --- a/storage/innobase/include/lock0lock.h +++ b/storage/innobase/include/lock0lock.h @@ -584,7 +584,7 @@ class lock_sys_t #else { private: - srw_lock_low lock; + srw_spin_lock_low lock; public: /** Try to acquire a lock */ bool try_acquire() { return lock.wr_lock_try(); } @@ -666,7 +666,7 @@ private: bool m_initialised; /** mutex proteting the locks */ - MY_ALIGNED(CPU_LEVEL1_DCACHE_LINESIZE) srw_lock latch; + MY_ALIGNED(CPU_LEVEL1_DCACHE_LINESIZE) srw_spin_lock latch; #ifdef UNIV_DEBUG /** The owner of exclusive latch (0 if none); protected by latch */ std::atomic writer{0}; diff --git a/storage/innobase/include/rw_lock.h b/storage/innobase/include/rw_lock.h index 3f1d76d8d97..cd176f0b35b 100644 --- a/storage/innobase/include/rw_lock.h +++ b/storage/innobase/include/rw_lock.h @@ -124,8 +124,8 @@ protected: } DBUG_ASSERT((l & ~WRITER_WAITING) == UPDATER); /* Any thread that had set WRITER_WAITING will eventually be woken - up by ssux_lock_low::x_unlock() or ssux_lock_low::u_unlock() - (not ssux_lock_low::wr_u_downgrade() to keep the code simple). */ + up by ssux_lock_impl::x_unlock() or ssux_lock_impl::u_unlock() + (not ssux_lock_impl::wr_u_downgrade() to keep the code simple). */ return true; } /** Downgrade an exclusive lock to an update lock. */ diff --git a/storage/innobase/include/srw_lock.h b/storage/innobase/include/srw_lock.h index fa3368d5c1b..b24e0a30857 100644 --- a/storage/innobase/include/srw_lock.h +++ b/storage/innobase/include/srw_lock.h @@ -32,9 +32,11 @@ public: void wr_unlock() { pthread_mutex_unlock(&lock); } bool wr_lock_try() { return !pthread_mutex_trylock(&lock); } }; +typedef srw_mutex srw_spin_mutex; #else /** Futex-based mutex */ -class srw_mutex final +template +class srw_mutex_impl final { /** The lock word, containing HOLDER + 1 if the lock is being held, plus the number of waiters */ @@ -79,10 +81,19 @@ public: } } }; + +typedef srw_mutex_impl srw_spin_mutex; +typedef srw_mutex_impl srw_mutex; +#endif + +# if defined _WIN32 || defined SUX_LOCK_GENERIC +# else +template class srw_lock_impl; #endif /** Slim shared-update-exclusive lock with no recursion */ -class ssux_lock_low final +template +class ssux_lock_impl final #ifdef SUX_LOCK_GENERIC : private rw_lock #endif @@ -92,7 +103,7 @@ class ssux_lock_low final # ifdef SUX_LOCK_GENERIC # elif defined _WIN32 # else - friend class srw_lock; + friend srw_lock_impl; # endif #endif #ifdef SUX_LOCK_GENERIC @@ -259,7 +270,7 @@ public: class srw_lock_low { # ifdef UNIV_PFS_RWLOCK - friend class srw_lock; + friend class srw_lock_impl; # endif SRWLOCK lock; public: @@ -272,12 +283,14 @@ public: bool wr_lock_try() { return TryAcquireSRWLockExclusive(&lock); } void wr_unlock() { ReleaseSRWLockExclusive(&lock); } }; + +typedef srw_lock_low srw_spin_lock_low; #elif defined SUX_LOCK_GENERIC /** Slim read-write lock */ class srw_lock_low { # ifdef UNIV_PFS_RWLOCK - friend class srw_lock; + friend class srw_lock_impl; # endif rw_lock_t lock; public: @@ -290,8 +303,10 @@ public: bool wr_lock_try() { return !rw_trywrlock(&lock); } void wr_unlock() { rw_unlock(&lock); } }; +typedef srw_lock_low srw_spin_lock_low; #else -typedef ssux_lock_low srw_lock_low; +typedef ssux_lock_impl srw_lock_low; +typedef ssux_lock_impl srw_spin_lock_low; #endif #ifndef UNIV_PFS_RWLOCK @@ -299,7 +314,7 @@ typedef ssux_lock_low srw_lock_low; # define SRW_LOCK_ARGS(file, line) /* nothing */ # define SRW_LOCK_CALL /* nothing */ typedef srw_lock_low srw_lock; -typedef ssux_lock_low ssux_lock; +typedef srw_spin_lock_low srw_spin_lock; #else # define SRW_LOCK_INIT(key) init(key) # define SRW_LOCK_ARGS(file, line) file, line @@ -309,7 +324,7 @@ typedef ssux_lock_low ssux_lock; class ssux_lock { PSI_rwlock *pfs_psi; - ssux_lock_low lock; + ssux_lock_impl lock; ATTRIBUTE_NOINLINE void psi_rd_lock(const char *file, unsigned line); ATTRIBUTE_NOINLINE void psi_wr_lock(const char *file, unsigned line); @@ -383,10 +398,18 @@ public: }; /** Slim reader-writer lock with PERFORMANCE_SCHEMA instrumentation */ -class srw_lock +# if defined _WIN32 || defined SUX_LOCK_GENERIC +# else +template +# endif +class srw_lock_impl { PSI_rwlock *pfs_psi; +# if defined _WIN32 || defined SUX_LOCK_GENERIC srw_lock_low lock; +# else + ssux_lock_impl lock; +# endif ATTRIBUTE_NOINLINE void psi_rd_lock(const char *file, unsigned line); ATTRIBUTE_NOINLINE void psi_wr_lock(const char *file, unsigned line); @@ -434,4 +457,13 @@ public: bool rd_lock_try() { return lock.rd_lock_try(); } bool wr_lock_try() { return lock.wr_lock_try(); } }; + +# if defined _WIN32 || defined SUX_LOCK_GENERIC +typedef srw_lock_impl srw_lock; +typedef srw_lock_impl srw_spin_lock; +# else +typedef srw_lock_impl srw_lock; +typedef srw_lock_impl srw_spin_lock; +# endif + #endif diff --git a/storage/innobase/include/sux_lock.h b/storage/innobase/include/sux_lock.h index f967e8c81cd..c09915cf6de 100644 --- a/storage/innobase/include/sux_lock.h +++ b/storage/innobase/include/sux_lock.h @@ -27,12 +27,12 @@ this program; if not, write to the Free Software Foundation, Inc., /** A "fat" rw-lock that supports S (shared), U (update, or shared-exclusive), and X (exclusive) modes as well as recursive U and X latch acquisition -@tparam srw ssux_lock_low or ssux_lock */ -template +@tparam ssux ssux_lock_impl or ssux_lock */ +template class sux_lock final { /** The underlying non-recursive lock */ - srw lock; + ssux lock; /** Numbers of U and X locks. Protected by lock. */ uint32_t recursive; /** The owner of the U or X lock (0 if none); protected by lock */ @@ -270,20 +270,14 @@ public: bool is_waiting() const { return lock.is_waiting(); } }; -/** needed for dict_index_t::clone() */ -template<> inline void sux_lock::operator=(const sux_lock&) -{ - memset((void*) this, 0, sizeof *this); -} - -typedef sux_lock block_lock; +typedef sux_lock> block_lock; #ifndef UNIV_PFS_RWLOCK -typedef block_lock index_lock; +typedef sux_lock> index_lock; #else typedef sux_lock index_lock; -template<> inline void sux_lock::init() +template<> inline void sux_lock>::init() { lock.init(); ut_ad(!writer.load(std::memory_order_relaxed)); @@ -340,8 +334,13 @@ inline void sux_lock::u_x_upgrade(const char *file, unsigned line) } #endif -template<> -inline void sux_lock::s_lock() +/** needed for dict_index_t::clone() */ +template<> inline void index_lock::operator=(const sux_lock&) +{ + memset((void*) this, 0, sizeof *this); +} + +template inline void sux_lock::s_lock() { ut_ad(!have_x()); ut_ad(!have_s()); @@ -349,8 +348,7 @@ inline void sux_lock::s_lock() ut_d(s_lock_register()); } -template<> -inline void sux_lock::u_lock() +template inline void sux_lock::u_lock() { os_thread_id_t id= os_thread_get_curr_id(); if (writer.load(std::memory_order_relaxed) == id) @@ -364,8 +362,7 @@ inline void sux_lock::u_lock() } } -template<> -inline void sux_lock::x_lock(bool for_io) +template inline void sux_lock::x_lock(bool for_io) { os_thread_id_t id= os_thread_get_curr_id(); if (writer.load(std::memory_order_relaxed) == id) @@ -382,15 +379,14 @@ inline void sux_lock::x_lock(bool for_io) } } -template<> -inline void sux_lock::u_x_upgrade() +template inline void sux_lock::u_x_upgrade() { ut_ad(have_u_not_x()); lock.u_wr_upgrade(); recursive/= RECURSIVE_U; } -template<> inline bool sux_lock::x_lock_upgraded() +template inline bool sux_lock::x_lock_upgraded() { os_thread_id_t id= os_thread_get_curr_id(); if (writer.load(std::memory_order_relaxed) == id) @@ -417,8 +413,7 @@ template<> inline bool sux_lock::x_lock_upgraded() } } -template<> -inline bool sux_lock::u_lock_try(bool for_io) +template inline bool sux_lock::u_lock_try(bool for_io) { os_thread_id_t id= os_thread_get_curr_id(); if (writer.load(std::memory_order_relaxed) == id) @@ -438,8 +433,7 @@ inline bool sux_lock::u_lock_try(bool for_io) return false; } -template<> -inline bool sux_lock::x_lock_try() +template inline bool sux_lock::x_lock_try() { os_thread_id_t id= os_thread_get_curr_id(); if (writer.load(std::memory_order_relaxed) == id) diff --git a/storage/innobase/include/trx0purge.h b/storage/innobase/include/trx0purge.h index c2c755e183a..417c6688e83 100644 --- a/storage/innobase/include/trx0purge.h +++ b/storage/innobase/include/trx0purge.h @@ -125,7 +125,7 @@ class purge_sys_t { public: /** latch protecting view, m_enabled */ - MY_ALIGNED(CACHE_LINE_SIZE) mutable srw_lock latch; + MY_ALIGNED(CACHE_LINE_SIZE) mutable srw_spin_lock latch; private: /** The purge will not remove undo logs which are >= this view */ ReadViewBase view; diff --git a/storage/innobase/include/trx0rseg.h b/storage/innobase/include/trx0rseg.h index 02e6f290c56..7d0147b3752 100644 --- a/storage/innobase/include/trx0rseg.h +++ b/storage/innobase/include/trx0rseg.h @@ -84,7 +84,7 @@ struct MY_ALIGNED(CPU_LEVEL1_DCACHE_LINESIZE) trx_rseg_t /** tablespace containing the rollback segment; constant after init() */ fil_space_t *space; /** latch protecting everything except page_no, space */ - srw_lock_low latch; + srw_spin_lock_low latch; /** rollback segment header page number; constant after init() */ uint32_t page_no; /** length of the TRX_RSEG_HISTORY list (number of transactions) */ diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index d64fd019b85..343e7d42dc4 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -565,7 +565,7 @@ public: private: /** mutex protecting state and some of lock (some are protected by lock_sys.latch) */ - srw_mutex mutex; + srw_spin_mutex mutex; #ifdef UNIV_DEBUG /** The owner of mutex (0 if none); protected by mutex */ std::atomic mutex_owner{0}; diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 79da2b522d7..4a839082a6b 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -3930,7 +3930,7 @@ row_sel_try_search_shortcut_for_mysql( ut_ad(!prebuilt->templ_contains_blob); ut_ad(trx->read_view.is_open()); - srw_lock* ahi_latch = btr_search_sys.get_latch(*index); + srw_spin_lock* ahi_latch = btr_search_sys.get_latch(*index); ahi_latch->rd_lock(SRW_LOCK_CALL); btr_pcur_open_with_no_init(index, search_tuple, PAGE_CUR_GE, BTR_SEARCH_LEAF, pcur, ahi_latch, mtr); diff --git a/storage/innobase/sync/srw_lock.cc b/storage/innobase/sync/srw_lock.cc index b267969c3df..4cd4b56bdd0 100644 --- a/storage/innobase/sync/srw_lock.cc +++ b/storage/innobase/sync/srw_lock.cc @@ -36,7 +36,8 @@ static inline void srw_pause(unsigned delay) } #ifdef SUX_LOCK_GENERIC -void ssux_lock_low::init() +template +void ssux_lock_impl::init() { DBUG_ASSERT(!is_locked_or_waiting()); pthread_mutex_init(&mutex, nullptr); @@ -44,7 +45,8 @@ void ssux_lock_low::init() pthread_cond_init(&cond_exclusive, nullptr); } -void ssux_lock_low::destroy() +template +void ssux_lock_impl::destroy() { DBUG_ASSERT(!is_locked_or_waiting()); pthread_mutex_destroy(&mutex); @@ -52,7 +54,8 @@ void ssux_lock_low::destroy() pthread_cond_destroy(&cond_exclusive); } -inline void ssux_lock_low::writer_wait(uint32_t l) +template +inline void ssux_lock_impl::writer_wait(uint32_t l) { pthread_mutex_lock(&mutex); while (value() == l) @@ -60,7 +63,8 @@ inline void ssux_lock_low::writer_wait(uint32_t l) pthread_mutex_unlock(&mutex); } -inline void ssux_lock_low::readers_wait(uint32_t l) +template +inline void ssux_lock_impl::readers_wait(uint32_t l) { pthread_mutex_lock(&mutex); while (value() == l) @@ -68,7 +72,8 @@ inline void ssux_lock_low::readers_wait(uint32_t l) pthread_mutex_unlock(&mutex); } -inline void ssux_lock_low::wake() +template +inline void ssux_lock_impl::wake() { pthread_mutex_lock(&mutex); uint32_t l= value(); @@ -85,7 +90,8 @@ inline void ssux_lock_low::wake() /** Wait for a read lock. @param lock word value from a failed read_trylock() */ -void ssux_lock_low::read_lock(uint32_t l) +template +void ssux_lock_impl::read_lock(uint32_t l) { do { @@ -105,16 +111,18 @@ void ssux_lock_low::read_lock(uint32_t l) pthread_mutex_unlock(&mutex); continue; } - - const unsigned delay= srw_pause_delay(); - - for (auto spin= srv_n_spin_wait_rounds; spin; spin--) + else if (spinloop) { - srw_pause(delay); - if (read_trylock(l)) - return; - else if (l == WRITER_WAITING) - goto wake_writer; + const unsigned delay= srw_pause_delay(); + + for (auto spin= srv_n_spin_wait_rounds; spin; spin--) + { + srw_pause(delay); + if (read_trylock(l)) + return; + else if (l == WRITER_WAITING) + goto wake_writer; + } } readers_wait(l); @@ -124,7 +132,8 @@ void ssux_lock_low::read_lock(uint32_t l) /** Wait for an update lock. @param lock word value from a failed update_trylock() */ -void ssux_lock_low::update_lock(uint32_t l) +template +void ssux_lock_impl::update_lock(uint32_t l) { do { @@ -144,7 +153,7 @@ void ssux_lock_low::update_lock(uint32_t l) pthread_mutex_unlock(&mutex); continue; } - else + else if (spinloop) { const unsigned delay= srw_pause_delay(); @@ -165,23 +174,12 @@ void ssux_lock_low::update_lock(uint32_t l) /** Wait for a write lock after a failed write_trylock() or upgrade_trylock() @param holding_u whether we already hold u_lock() */ -void ssux_lock_low::write_lock(bool holding_u) +template +void ssux_lock_impl::write_lock(bool holding_u) { - const unsigned delay= srw_pause_delay(); - for (;;) { uint32_t l= write_lock_wait_start(); - /* We are the first writer to be granted the lock. Spin for a while. */ - for (auto spin= srv_n_spin_wait_rounds; spin; spin--) - { - l= holding_u ? WRITER_WAITING | UPDATER : WRITER_WAITING; - if (write_lock_wait_try(l)) - return; - if (!(l & WRITER_WAITING)) - l= write_lock_wait_start(); - srw_pause(delay); - } const uint32_t e= holding_u ? WRITER_WAITING | UPDATER : WRITER_WAITING; l= e; @@ -213,21 +211,34 @@ void ssux_lock_low::write_lock(bool holding_u) } } -void ssux_lock_low::rd_unlock() { if (read_unlock()) wake(); } -void ssux_lock_low::u_unlock() { update_unlock(); wake(); } -void ssux_lock_low::wr_unlock() { write_unlock(); wake(); } +template +void ssux_lock_impl::rd_unlock() { if (read_unlock()) wake(); } +template +void ssux_lock_impl::u_unlock() { update_unlock(); wake(); } +template +void ssux_lock_impl::wr_unlock() { write_unlock(); wake(); } + +template void ssux_lock_impl::init(); +template void ssux_lock_impl::destroy(); +template void ssux_lock_impl::rd_unlock(); +template void ssux_lock_impl::u_unlock(); +template void ssux_lock_impl::wr_unlock(); #else /* SUX_LOCK_GENERIC */ static_assert(4 == sizeof(rw_lock), "ABI"); # ifdef _WIN32 # include -inline void srw_mutex::wait(uint32_t lk) +template +inline void srw_mutex_impl::wait(uint32_t lk) { WaitOnAddress(&lock, &lk, 4, INFINITE); } -void srw_mutex::wake() { WakeByAddressSingle(&lock); } +template +void srw_mutex_impl::wake() { WakeByAddressSingle(&lock); } -inline void ssux_lock_low::wait(uint32_t lk) +template +inline void ssux_lock_impl::wait(uint32_t lk) { WaitOnAddress(&readers, &lk, 4, INFINITE); } -void ssux_lock_low::wake() { WakeByAddressSingle(&readers); } +template +void ssux_lock_impl::wake() { WakeByAddressSingle(&readers); } # else # ifdef __linux__ @@ -244,16 +255,27 @@ void ssux_lock_low::wake() { WakeByAddressSingle(&readers); } # error "no futex support" # endif -inline void srw_mutex::wait(uint32_t lk) { SRW_FUTEX(&lock, WAIT, lk); } -void srw_mutex::wake() { SRW_FUTEX(&lock, WAKE, 1); } +template +inline void srw_mutex_impl::wait(uint32_t lk) +{ SRW_FUTEX(&lock, WAIT, lk); } +template +void srw_mutex_impl::wake() { SRW_FUTEX(&lock, WAKE, 1); } -inline void ssux_lock_low::wait(uint32_t lk) { SRW_FUTEX(&readers, WAIT, lk); } -void ssux_lock_low::wake() { SRW_FUTEX(&readers, WAKE, 1); } +template +inline void ssux_lock_impl::wait(uint32_t lk) +{ SRW_FUTEX(&readers, WAIT, lk); } +template +void ssux_lock_impl::wake() { SRW_FUTEX(&readers, WAKE, 1); } # endif +template void srw_mutex_impl::wake(); +template void ssux_lock_impl::wake(); +template void srw_mutex_impl::wake(); +template void ssux_lock_impl::wake(); -void srw_mutex::wait_and_lock() +template<> +void srw_mutex_impl::wait_and_lock() { uint32_t lk= 1 + lock.fetch_add(1, std::memory_order_relaxed); @@ -295,7 +317,31 @@ acquired: } } -void ssux_lock_low::wr_wait(uint32_t lk) +template<> +void srw_mutex_impl::wait_and_lock() +{ + uint32_t lk= 1 + lock.fetch_add(1, std::memory_order_relaxed); + for (;; wait(lk)) + { + if (lk & HOLDER) + { + lk= lock.load(std::memory_order_relaxed); + if (lk & HOLDER) + continue; + } + lk= lock.fetch_or(HOLDER, std::memory_order_relaxed); + if (!(lk & HOLDER)) + { + DBUG_ASSERT(lk); + std::atomic_thread_fence(std::memory_order_acquire); + return; + } + DBUG_ASSERT(lk > HOLDER); + } +} + +template +void ssux_lock_impl::wr_wait(uint32_t lk) { DBUG_ASSERT(writer.is_locked()); DBUG_ASSERT(lk); @@ -310,7 +356,11 @@ void ssux_lock_low::wr_wait(uint32_t lk) while (lk != WRITER); } -void ssux_lock_low::rd_wait() +template void ssux_lock_impl::wr_wait(uint32_t); +template void ssux_lock_impl::wr_wait(uint32_t); + +template +void ssux_lock_impl::rd_wait() { for (;;) { @@ -329,10 +379,22 @@ void ssux_lock_low::rd_wait() } writer.wr_unlock(); } + +template void ssux_lock_impl::rd_wait(); +template void ssux_lock_impl::rd_wait(); #endif /* SUX_LOCK_GENERIC */ #ifdef UNIV_PFS_RWLOCK -void srw_lock::psi_rd_lock(const char *file, unsigned line) +# if defined _WIN32 || defined SUX_LOCK_GENERIC +# define void_srw_lock void srw_lock_impl +# else +# define void_srw_lock template void srw_lock_impl +template void srw_lock_impl::psi_rd_lock(const char*, unsigned); +template void srw_lock_impl::psi_wr_lock(const char*, unsigned); +template void srw_lock_impl::psi_rd_lock(const char*, unsigned); +template void srw_lock_impl::psi_wr_lock(const char*, unsigned); +# endif +void_srw_lock::psi_rd_lock(const char *file, unsigned line) { PSI_rwlock_locker_state state; const bool nowait= lock.rd_lock_try(); @@ -348,7 +410,7 @@ void srw_lock::psi_rd_lock(const char *file, unsigned line) lock.rd_lock(); } -void srw_lock::psi_wr_lock(const char *file, unsigned line) +void_srw_lock::psi_wr_lock(const char *file, unsigned line) { PSI_rwlock_locker_state state; const bool nowait= lock.wr_lock_try(); @@ -428,7 +490,7 @@ void ssux_lock::psi_u_wr_upgrade(const char *file, unsigned line) DBUG_ASSERT(lock.writer.is_locked()); uint32_t lk= 1; const bool nowait= - lock.readers.compare_exchange_strong(lk, ssux_lock_low::WRITER, + lock.readers.compare_exchange_strong(lk, ssux_lock_impl::WRITER, std::memory_order_acquire, std::memory_order_relaxed); if (PSI_rwlock_locker *locker= PSI_RWLOCK_CALL(start_rwlock_wrwait) @@ -444,4 +506,14 @@ void ssux_lock::psi_u_wr_upgrade(const char *file, unsigned line) else if (!nowait) lock.u_wr_upgrade(); } +#else /* UNIV_PFS_RWLOCK */ +template void ssux_lock_impl::rd_lock(); +# ifdef SUX_LOCK_GENERIC +template void ssux_lock_impl::write_lock(bool); +template void ssux_lock_impl::update_lock(uint32_t); +# else +template void ssux_lock_impl::rd_unlock(); +template void ssux_lock_impl::u_unlock(); +template void ssux_lock_impl::wr_unlock(); +# endif #endif /* UNIV_PFS_RWLOCK */ diff --git a/storage/innobase/unittest/innodb_sync-t.cc b/storage/innobase/unittest/innodb_sync-t.cc index 6cf0e648d9b..d0289086b24 100644 --- a/storage/innobase/unittest/innodb_sync-t.cc +++ b/storage/innobase/unittest/innodb_sync-t.cc @@ -62,7 +62,7 @@ static void test_srw_lock() } } -static ssux_lock_low ssux; +static ssux_lock_impl ssux; static void test_ssux_lock() { @@ -95,7 +95,7 @@ static void test_ssux_lock() } } -static sux_lock sux; +static sux_lock> sux; static void test_sux_lock() { -- cgit v1.2.1 From 2e39987fda252b2686146301509ee15d132fb3e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 6 Sep 2021 14:28:02 +0300 Subject: MDEV-26467: GCC 4.8.5 internal compiler error on ARMv8 We will disable some optimizations, because the function row_ins_clust_index_entry_low() would fail to compile ever since commit a73eedbf3fabd19ca7183b738056c30e3f7bbe35 changed the definition of srw_mutex::wr_unlock() to use fetch_sub() instead of fetch_and(). For some reason, applying this work-around does not fix the "could not split insn" error for that commit, while it does work for commit 277ba134ad1c994bda50de574a87a06a071fbecb. --- storage/innobase/row/row0ins.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index 6f228142cba..b67c1212271 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -2501,6 +2501,13 @@ extern "C" int thd_is_slave(const MYSQL_THD thd); # define thd_is_slave(thd) 0 #endif +#if defined __aarch64__&&defined __GNUC__&&__GNUC__==4&&!defined __clang__ +/* Avoid GCC 4.8.5 internal compiler error due to srw_mutex::wr_unlock(). +We would only need this for row_ins_clust_index_entry_low(), +but GCC 4.8.5 does not support pop_options. */ +# pragma GCC optimize ("no-expensive-optimizations") +#endif + /***************************************************************//** Tries to insert an entry into a clustered index, ignoring foreign key constraints. If a record with the same unique key is found, the other -- cgit v1.2.1 From 16131a7e46bb6c120b4e0ddb27b03c0bb7164928 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 4 Sep 2021 20:24:37 +0200 Subject: MDEV-26527 speedup appveyor build Allow mysql_client_test to execute even if server/client auth plugins are not built. There is a single small test that requires qa_auth_client. --- mysql-test/main/mysql_client_test.test | 2 -- tests/mysql_client_test.c | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mysql-test/main/mysql_client_test.test b/mysql-test/main/mysql_client_test.test index 9fb7bcd81c9..bcb246a8a06 100644 --- a/mysql-test/main/mysql_client_test.test +++ b/mysql-test/main/mysql_client_test.test @@ -1,7 +1,5 @@ # This test should work in embedded server after we fix mysqltest -- source include/not_embedded.inc -# need to have the dynamic loading turned on for the client plugin tests ---source include/have_plugin_auth.inc # Run test with default character set --source include/default_charset.inc diff --git a/tests/mysql_client_test.c b/tests/mysql_client_test.c index 9c18667614b..8f4e2b2fdad 100644 --- a/tests/mysql_client_test.c +++ b/tests/mysql_client_test.c @@ -19203,6 +19203,9 @@ static void test_bug11766854() struct st_mysql_client_plugin *plugin; DBUG_ENTER("test_bug11766854"); + if (!getenv("QA_AUTH_CLIENT_SO")) + DBUG_VOID_RETURN; + myheader("test_bug11766854"); plugin= mysql_load_plugin(mysql, "foo", -1, 0); -- cgit v1.2.1 From 999d254cf289ba122d40f3e2590bf57561d47d21 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 4 Sep 2021 20:33:01 +0200 Subject: MDEV-26527 speedup appveyor build Introduce -DFAST_BUILD parameter for a little faster build or test if set, - do not compile with /d2OptimizeHugeFunctions, this makes compilation of bison output much slower on optimized build - do not use runtime checks on debug build (RTC1). This slows down tests considerably --- cmake/os/Windows.cmake | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmake/os/Windows.cmake b/cmake/os/Windows.cmake index 232560ff7b1..d9a0dfba46a 100644 --- a/cmake/os/Windows.cmake +++ b/cmake/os/Windows.cmake @@ -116,7 +116,7 @@ ENDMACRO() IF(MSVC) IF(MSVC_VERSION LESS 1920) - MESSAGE(FATAL_ERROR "Visual Studio q2019 or later is required") + MESSAGE(FATAL_ERROR "Visual Studio 2019 or later is required") ENDIF() # Disable mingw based pkg-config found in Strawberry perl SET(PKG_CONFIG_EXECUTABLE 0 CACHE INTERNAL "") @@ -254,12 +254,12 @@ IF(MSVC) ENDFOREACH() ENDFOREACH() ENDIF() - IF(MSVC_VERSION LESS 1910) - # Noisy warning C4800: 'type': forcing value to bool 'true' or 'false' (performance warning), - # removed in VS2017 - STRING(APPEND CMAKE_CXX_FLAGS " /wd4800") + + IF(FAST_BUILD) + STRING (REGEX REPLACE "/RTC(su|[1su])" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") ELSEIF (NOT CLANG_CL) - STRING(APPEND CMAKE_CXX_FLAGS " /d2OptimizeHugeFunctions") + STRING(APPEND CMAKE_CXX_FLAGS_RELEASE " /d2OptimizeHugeFunctions") + STRING(APPEND CMAKE_CXX_FLAGS_RELWITHDEBINFO " /d2OptimizeHugeFunctions") ENDIF() ENDIF() -- cgit v1.2.1 From 7dd85c0ffdd3bc12d8b83b1f6c43f75d30cb5e37 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sat, 4 Sep 2021 20:37:40 +0200 Subject: MDEV-26527 speedup appveyor build - set clone_depth to 1. Otherwise, appveyor spends 4.5 minutes for the "git clone" alone - Use Ninja instead of msbuild, it is (a bit) faster - do not compile perfschema, or dynamic plugins to save time on compilation, or test execution. - use -DFAST_BUILD=1 to speedup build a little - increase number of parallel jobs used in build and test. - Exclude some slow tests from the main suite. --- appveyor.yml | 29 ++++++++++++++++------------- win/appveyor_skip_tests.txt | 14 ++++++++++++++ 2 files changed, 30 insertions(+), 13 deletions(-) create mode 100644 win/appveyor_skip_tests.txt diff --git a/appveyor.yml b/appveyor.yml index 355c7f5aeeb..79ce36410f5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,21 +1,24 @@ version: build-{build}~branch-{branch} -before_build: - - md %APPVEYOR_BUILD_FOLDER%\win_build - - cd %APPVEYOR_BUILD_FOLDER%\win_build - - cmake .. -DWITH_UNIT_TESTS=0 -DWITH_MARIABACKUP=0 -DMYSQL_MAINTAINER_MODE=ERR -DPLUGIN_ROCKSDB=NO -DPLUGIN_CONNECT=NO -DBISON_EXECUTABLE=C:\cygwin64\bin\bison +clone_depth: 1 -build: - project: win_build\MySQL.sln - parallel: true - verbosity: minimal - -configuration: RelWithDebInfo -platform: x64 +build_script: + # dump some system info + - echo processor='%PROCESSOR_IDENTIFIER%' , processor count= %NUMBER_OF_PROCESSORS% + - cd %APPVEYOR_BUILD_FOLDER% + - mkdir _build + - cd _build + - set BUILD_TYPE=MinSizeRel + - set GENERATOR=-GNinja + - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat" + - cmake -E time cmake %GENERATOR% .. -DCMAKE_BUILD_TYPE=%BUILD_TYPE% -DMYSQL_MAINTAINER_MODE=ERR -DFAST_BUILD=1 -DBISON_EXECUTABLE=C:\cygwin64\bin\bison -DWITHOUT_DYNAMIC_PLUGINS=1 -DPLUGIN_PERFSCHEMA=NO -DPLUGIN_FEEDBACK=NO -DWITH_UNIT_TESTS=0 -DWITH_MARIABACKUP=0 + - set /A jobs=2*%NUMBER_OF_PROCESSORS% + - cmake -E time cmake --build . -j %jobs% --config %BUILD_TYPE% test_script: - set PATH=C:\Strawberry\perl\bin;%PATH%;C:\Program Files (x86)\Windows Kits\10\Debuggers\x64 - - cd %APPVEYOR_BUILD_FOLDER%\win_build\mysql-test - - perl mysql-test-run.pl --force --max-test-fail=10 --parallel=4 --testcase-timeout=10 --suite=main + - cd %APPVEYOR_BUILD_FOLDER%\_build\mysql-test + - set /A parallel=4*%NUMBER_OF_PROCESSORS% + - perl mysql-test-run.pl --force --max-test-fail=10 --retry=2 -parallel=%parallel% --testcase-timeout=3 --suite=main --skip-test-list=%APPVEYOR_BUILD_FOLDER%\win\appveyor_skip_tests.txt --mysqld=--loose-innodb-flush-log-at-trx-commit=2 image: Visual Studio 2019 diff --git a/win/appveyor_skip_tests.txt b/win/appveyor_skip_tests.txt new file mode 100644 index 00000000000..3f0a0874064 --- /dev/null +++ b/win/appveyor_skip_tests.txt @@ -0,0 +1,14 @@ +main.mysql_upgrade : Takes long time on Appveyor +main.mysqlslap : Takes long time +mysql.upgrade_view : Takes long time +main.check : Takes long time on Appveyor +main.mrr_icp_extra : Takes long time on Appveyor +main.derived_opt : Takes long time on Appveyor +main.trigger : Takes long time on Appveyor +main.index_merge_myisam : Takes long time on Appveyor +main.mysqldump : Takes long time on Appveyor +main.derived : Takes long time on Appveyor +main.multi_update : Takes long time on Appveyor +main.index_merge_innodb : Takes long time on Appveyor +main.count_distinct2 : Takes long time on Appveyor +main.mysqltest : Takes long time on Appveyor -- cgit v1.2.1 From d8943b0cc3ecb10dc68542e7846f9411a35f0d69 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Sun, 5 Sep 2021 19:35:21 +0200 Subject: MDEV-26527 speedup appveyor build Define custom target for minimal testable build use it in appveyor.yml --- CMakeLists.txt | 59 ++++++++++++++++++++++++++++++++++++++++++---------------- appveyor.yml | 4 ++-- 2 files changed, 45 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b8b3f5c7d12..5301dc563af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,22 +33,11 @@ PROJECT(MySQL) # in RPM's: #set(CPACK_RPM_SPEC_MORE_DEFINE "%define __spec_install_post /bin/true") - -IF(POLICY CMP0022) - CMAKE_POLICY(SET CMP0022 NEW) -ENDIF() -IF(POLICY CMP0048) - CMAKE_POLICY(SET CMP0048 NEW) -ENDIF() -IF(POLICY CMP0054) - CMAKE_POLICY(SET CMP0054 NEW) -ENDIF() -IF(POLICY CMP0075) - CMAKE_POLICY(SET CMP0075 NEW) -ENDIF() -IF(POLICY CMP0069) - CMAKE_POLICY(SET CMP0069 NEW) -ENDIF() +FOREACH(p CMP0022 CMP0046 CMP0040 CMP0048 CMP0054 CMP0075 CMP0069) + IF(POLICY ${p}) + CMAKE_POLICY(SET ${p} NEW) + ENDIF() +ENDFOREACH() MESSAGE(STATUS "Running cmake version ${CMAKE_VERSION}") @@ -563,3 +552,41 @@ IF(NON_DISTRIBUTABLE_WARNING) MESSAGE(WARNING " You have linked MariaDB with ${NON_DISTRIBUTABLE_WARNING} libraries! You may not distribute the resulting binary. If you do, you will put yourself into a legal problem with the Free Software Foundation.") ENDIF() + +IF(NOT WITHOUT_SERVER) + # Define target for minimal mtr-testable build + ADD_CUSTOM_TARGET(minbuild) + ADD_DEPENDENCIES(minbuild + aria_chk + aria_pack + mariadb + mariadb-admin + mariadb-binlog + mariadb-check + mariadb-client-test + mariadb-conv + mariadb-dump + mariadb-import + mariadb-plugin + mariadb-show + mariadb-slap + mariadb-test + mariadb-tzinfo-to-sql + mariadb-upgrade + mariadbd + my_print_defaults + my_safe_process + myisam_ftdump + myisamchk + myisamlog + myisampack + perror + replace) + IF(WIN32) + ADD_DEPENDENCIES(minbuild echo mariadb-install-db my_safe_kill) + ENDIF() + ADD_CUSTOM_TARGET(smoketest + COMMAND perl ./mysql-test-run.pl main.1st + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/mysql-test) + ADD_DEPENDENCIES(smoketest minbuild) +ENDIF() diff --git a/appveyor.yml b/appveyor.yml index 79ce36410f5..c4abddf6665 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,9 +11,9 @@ build_script: - set BUILD_TYPE=MinSizeRel - set GENERATOR=-GNinja - call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat" - - cmake -E time cmake %GENERATOR% .. -DCMAKE_BUILD_TYPE=%BUILD_TYPE% -DMYSQL_MAINTAINER_MODE=ERR -DFAST_BUILD=1 -DBISON_EXECUTABLE=C:\cygwin64\bin\bison -DWITHOUT_DYNAMIC_PLUGINS=1 -DPLUGIN_PERFSCHEMA=NO -DPLUGIN_FEEDBACK=NO -DWITH_UNIT_TESTS=0 -DWITH_MARIABACKUP=0 + - cmake -E time cmake %GENERATOR% .. -DCMAKE_BUILD_TYPE=%BUILD_TYPE% -DMYSQL_MAINTAINER_MODE=ERR -DFAST_BUILD=1 -DBISON_EXECUTABLE=C:\cygwin64\bin\bison -DPLUGIN_PERFSCHEMA=NO -DPLUGIN_FEEDBACK=NO - set /A jobs=2*%NUMBER_OF_PROCESSORS% - - cmake -E time cmake --build . -j %jobs% --config %BUILD_TYPE% + - cmake -E time cmake --build . -j %jobs% --config %BUILD_TYPE% --target minbuild test_script: - set PATH=C:\Strawberry\perl\bin;%PATH%;C:\Program Files (x86)\Windows Kits\10\Debuggers\x64 -- cgit v1.2.1