diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/CMakeLists.txt | 6 | ||||
| -rwxr-xr-x | scripts/galera_new_cluster.sh | 5 | ||||
| -rw-r--r-- | scripts/mysql_install_db.sh | 7 | ||||
| -rw-r--r-- | scripts/mysql_secure_installation.sh | 6 | ||||
| -rw-r--r-- | scripts/mysql_system_tables.sql | 63 | ||||
| -rw-r--r-- | scripts/mysql_system_tables_data.sql | 26 | ||||
| -rw-r--r-- | scripts/mysql_system_tables_fix.sql | 87 | ||||
| -rw-r--r-- | scripts/mysql_test_db.sql | 4 | ||||
| -rw-r--r-- | scripts/mysqld_safe.sh | 11 | ||||
| -rw-r--r-- | scripts/mytop.sh | 19 | ||||
| -rwxr-xr-x | scripts/wsrep_sst_common.sh | 32 | ||||
| -rw-r--r-- | scripts/wsrep_sst_mariabackup.sh | 155 | ||||
| -rw-r--r-- | scripts/wsrep_sst_mysqldump.sh | 5 | ||||
| -rw-r--r-- | scripts/wsrep_sst_rsync.sh | 61 | ||||
| -rw-r--r-- | scripts/wsrep_sst_xtrabackup-v2.sh | 1236 | ||||
| -rw-r--r-- | scripts/wsrep_sst_xtrabackup.sh | 692 |
16 files changed, 386 insertions, 2029 deletions
diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 464c00d57da..eca5b7ad3e5 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -31,7 +31,7 @@ ENDIF() IF(CAT_EXECUTABLE) SET(CAT_COMMAND COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_SOURCE_DIR} - ${CAT_EXECUTABLE} mysql_system_tables.sql mysql_system_tables_fix.sql mysql_performance_tables.sql > + ${CAT_EXECUTABLE} mysql_system_tables_fix.sql mysql_system_tables.sql mysql_performance_tables.sql > ${CMAKE_CURRENT_BINARY_DIR}/mysql_fix_privilege_tables.sql ) ELSEIF(WIN32) @@ -39,7 +39,7 @@ ELSEIF(WIN32) native_outfile ) SET(CAT_COMMAND COMMAND ${CMAKE_COMMAND} -E chdir ${CMAKE_CURRENT_SOURCE_DIR} - cmd /c copy /b mysql_system_tables.sql + mysql_system_tables_fix.sql + mysql_performance_tables.sql + cmd /c copy /b mysql_system_tables_fix.sql + mysql_system_tables.sql + mysql_performance_tables.sql ${native_outfile} ) ELSE() MESSAGE(FATAL_ERROR "Cannot concatenate files") @@ -274,8 +274,6 @@ ELSE() SET(WSREP_SCRIPTS wsrep_sst_mysqldump wsrep_sst_rsync - wsrep_sst_xtrabackup - wsrep_sst_xtrabackup-v2 wsrep_sst_mariabackup ) # The following script is sourced from other SST scripts, so it should diff --git a/scripts/galera_new_cluster.sh b/scripts/galera_new_cluster.sh index e4e2ac39b52..e0763ed516a 100755 --- a/scripts/galera_new_cluster.sh +++ b/scripts/galera_new_cluster.sh @@ -21,9 +21,6 @@ EOF exit 0 fi -VERSION="@VERSION@@MYSQL_SERVER_SUFFIX@" -COMPILATION_COMMENT="@COMPILATION_COMMENT@" - systemctl set-environment _WSREP_NEW_CLUSTER='--wsrep-new-cluster' && \ systemctl start ${1:-mariadb} @@ -31,4 +28,4 @@ extcode=$? systemctl set-environment _WSREP_NEW_CLUSTER='' -return $extcode +exit $extcode diff --git a/scripts/mysql_install_db.sh b/scripts/mysql_install_db.sh index 8a0ade99d69..82a8e175753 100644 --- a/scripts/mysql_install_db.sh +++ b/scripts/mysql_install_db.sh @@ -473,6 +473,13 @@ then args="$args --user=$user" fi +if test -f "$ldata/mysql/user.frm" +then + echo "mysql.user table already exists!" + echo "Run mysql_upgrade, not mysql_install_db" + exit 0 +fi + # When doing a "cross bootstrap" install, no reference to the current # host should be added to the system tables. So we filter out any # lines which contain the current host name. diff --git a/scripts/mysql_secure_installation.sh b/scripts/mysql_secure_installation.sh index 57e4d43dfad..1bd21513d4d 100644 --- a/scripts/mysql_secure_installation.sh +++ b/scripts/mysql_secure_installation.sh @@ -304,7 +304,7 @@ set_root_password() { fi esc_pass=`basic_single_escape "$password1"` - do_query "UPDATE mysql.user SET Password=PASSWORD('$esc_pass') WHERE User='root';" + do_query "UPDATE mysql.global_priv SET priv=json_set(priv, '$.plugin', 'mysql_native_password', '$.authentication_string', PASSWORD('$esc_pass')) WHERE User='root';" if [ $? -eq 0 ]; then echo "Password updated successfully!" echo "Reloading privilege tables.." @@ -324,7 +324,7 @@ set_root_password() { } remove_anonymous_users() { - do_query "DELETE FROM mysql.user WHERE User='';" + do_query "DELETE FROM mysql.global_priv WHERE User='';" if [ $? -eq 0 ]; then echo " ... Success!" else @@ -336,7 +336,7 @@ remove_anonymous_users() { } remove_remote_root() { - do_query "DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1');" + do_query "DELETE FROM mysql.global_priv WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1');" if [ $? -eq 0 ]; then echo " ... Success!" else diff --git a/scripts/mysql_system_tables.sql b/scripts/mysql_system_tables.sql index 0d59862f26f..f788f5d67d5 100644 --- a/scripts/mysql_system_tables.sql +++ b/scripts/mysql_system_tables.sql @@ -33,7 +33,57 @@ CREATE TABLE IF NOT EXISTS db ( Host char(60) binary DEFAULT '' NOT NULL, Db c -- Remember for later if db table already existed set @had_db_table= @@warning_count != 0; -CREATE TABLE IF NOT EXISTS user ( Host char(60) binary DEFAULT '' NOT NULL, User char(80) binary DEFAULT '' NOT NULL, Password char(41) character set latin1 collate latin1_bin DEFAULT '' NOT NULL, Select_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Insert_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Update_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Delete_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Create_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Drop_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Reload_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Shutdown_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Process_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, File_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Grant_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, References_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Index_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Alter_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Show_db_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Super_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Create_tmp_table_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Lock_tables_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Execute_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Repl_slave_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Repl_client_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Create_view_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Show_view_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Create_routine_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Alter_routine_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Create_user_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Event_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Trigger_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Create_tablespace_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, Delete_history_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, ssl_type enum('','ANY','X509', 'SPECIFIED') COLLATE utf8_general_ci DEFAULT '' NOT NULL, ssl_cipher BLOB NOT NULL, x509_issuer BLOB NOT NULL, x509_subject BLOB NOT NULL, max_questions int(11) unsigned DEFAULT 0 NOT NULL, max_updates int(11) unsigned DEFAULT 0 NOT NULL, max_connections int(11) unsigned DEFAULT 0 NOT NULL, max_user_connections int(11) DEFAULT 0 NOT NULL, plugin char(64) CHARACTER SET latin1 DEFAULT '' NOT NULL, authentication_string TEXT NOT NULL, password_expired ENUM('N', 'Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, is_role enum('N', 'Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL, default_role char(80) binary DEFAULT '' NOT NULL, max_statement_time decimal(12,6) DEFAULT 0 NOT NULL, PRIMARY KEY Host (Host,User) ) engine=Aria transactional=1 CHARACTER SET utf8 COLLATE utf8_bin comment='Users and global privileges'; +CREATE TABLE IF NOT EXISTS global_priv (Host char(60) binary DEFAULT '', User char(80) binary DEFAULT '', Priv JSON NOT NULL DEFAULT '{}' CHECK(JSON_VALID(Priv)), PRIMARY KEY Host (Host,User)) engine=Aria transactional=1 CHARACTER SET utf8 COLLATE utf8_bin comment='Users and global privileges'; + +CREATE DEFINER=root@localhost SQL SECURITY DEFINER VIEW IF NOT EXISTS user AS SELECT + Host, + User, + IF(JSON_VALUE(Priv, '$.plugin') IN ('mysql_native_password', 'mysql_old_password'), IFNULL(JSON_VALUE(Priv, '$.authentication_string'), ''), '') AS Password, + IF(JSON_VALUE(Priv, '$.access') & 1, 'Y', 'N') AS Select_priv, + IF(JSON_VALUE(Priv, '$.access') & 2, 'Y', 'N') AS Insert_priv, + IF(JSON_VALUE(Priv, '$.access') & 4, 'Y', 'N') AS Update_priv, + IF(JSON_VALUE(Priv, '$.access') & 8, 'Y', 'N') AS Delete_priv, + IF(JSON_VALUE(Priv, '$.access') & 16, 'Y', 'N') AS Create_priv, + IF(JSON_VALUE(Priv, '$.access') & 32, 'Y', 'N') AS Drop_priv, + IF(JSON_VALUE(Priv, '$.access') & 64, 'Y', 'N') AS Reload_priv, + IF(JSON_VALUE(Priv, '$.access') & 128, 'Y', 'N') AS Shutdown_priv, + IF(JSON_VALUE(Priv, '$.access') & 256, 'Y', 'N') AS Process_priv, + IF(JSON_VALUE(Priv, '$.access') & 512, 'Y', 'N') AS File_priv, + IF(JSON_VALUE(Priv, '$.access') & 1024, 'Y', 'N') AS Grant_priv, + IF(JSON_VALUE(Priv, '$.access') & 2048, 'Y', 'N') AS References_priv, + IF(JSON_VALUE(Priv, '$.access') & 4096, 'Y', 'N') AS Index_priv, + IF(JSON_VALUE(Priv, '$.access') & 8192, 'Y', 'N') AS Alter_priv, + IF(JSON_VALUE(Priv, '$.access') & 16384, 'Y', 'N') AS Show_db_priv, + IF(JSON_VALUE(Priv, '$.access') & 32768, 'Y', 'N') AS Super_priv, + IF(JSON_VALUE(Priv, '$.access') & 65536, 'Y', 'N') AS Create_tmp_table_priv, + IF(JSON_VALUE(Priv, '$.access') & 131072, 'Y', 'N') AS Lock_tables_priv, + IF(JSON_VALUE(Priv, '$.access') & 262144, 'Y', 'N') AS Execute_priv, + IF(JSON_VALUE(Priv, '$.access') & 524288, 'Y', 'N') AS Repl_slave_priv, + IF(JSON_VALUE(Priv, '$.access') & 1048576, 'Y', 'N') AS Repl_client_priv, + IF(JSON_VALUE(Priv, '$.access') & 2097152, 'Y', 'N') AS Create_view_priv, + IF(JSON_VALUE(Priv, '$.access') & 4194304, 'Y', 'N') AS Show_view_priv, + IF(JSON_VALUE(Priv, '$.access') & 8388608, 'Y', 'N') AS Create_routine_priv, + IF(JSON_VALUE(Priv, '$.access') & 16777216, 'Y', 'N') AS Alter_routine_priv, + IF(JSON_VALUE(Priv, '$.access') & 33554432, 'Y', 'N') AS Create_user_priv, + IF(JSON_VALUE(Priv, '$.access') & 67108864, 'Y', 'N') AS Event_priv, + IF(JSON_VALUE(Priv, '$.access') & 134217728, 'Y', 'N') AS Trigger_priv, + IF(JSON_VALUE(Priv, '$.access') & 268435456, 'Y', 'N') AS Create_tablespace_priv, + IF(JSON_VALUE(Priv, '$.access') & 536870912, 'Y', 'N') AS Delete_history_priv, + ELT(IFNULL(JSON_VALUE(Priv, '$.ssl_type'), 0) + 1, '', 'ANY','X509', 'SPECIFIED') AS ssl_type, + IFNULL(JSON_VALUE(Priv, '$.ssl_cipher'), '') AS ssl_cipher, + IFNULL(JSON_VALUE(Priv, '$.x509_issuer'), '') AS x509_issuer, + IFNULL(JSON_VALUE(Priv, '$.x509_subject'), '') AS x509_subject, + CAST(IFNULL(JSON_VALUE(Priv, '$.max_questions'), 0) AS UNSIGNED) AS max_questions, + CAST(IFNULL(JSON_VALUE(Priv, '$.max_updates'), 0) AS UNSIGNED) AS max_updates, + CAST(IFNULL(JSON_VALUE(Priv, '$.max_connections'), 0) AS UNSIGNED) AS max_connections, + CAST(IFNULL(JSON_VALUE(Priv, '$.max_user_connections'), 0) AS SIGNED) AS max_user_connections, + IFNULL(JSON_VALUE(Priv, '$.plugin'), '') AS plugin, + IFNULL(JSON_VALUE(Priv, '$.authentication_string'), '') AS authentication_string, + 'N' AS password_expired, + ELT(IFNULL(JSON_VALUE(Priv, '$.is_role'), 0) + 1, 'N', 'Y') AS is_role, + IFNULL(JSON_VALUE(Priv, '$.default_role'), '') AS default_role, + CAST(IFNULL(JSON_VALUE(Priv, '$.max_statement_time'), 0.0) AS DECIMAL(12,6)) AS max_statement_time + FROM global_priv; -- Remember for later if user table already existed set @had_user_table= @@warning_count != 0; @@ -80,7 +130,7 @@ CREATE TABLE IF NOT EXISTS time_zone_transition_type ( Time_zone_id int unsign CREATE TABLE IF NOT EXISTS time_zone_leap_second ( Transition_time bigint signed NOT NULL, Correction int signed NOT NULL, PRIMARY KEY TranTime (Transition_time) ) engine=Aria transactional=1 CHARACTER SET utf8 comment='Leap seconds information for time zones'; -CREATE TABLE IF NOT EXISTS proc (db char(64) collate utf8_bin DEFAULT '' NOT NULL, name char(64) DEFAULT '' NOT NULL, type enum('FUNCTION','PROCEDURE','PACKAGE','PACKAGE BODY') NOT NULL, specific_name char(64) DEFAULT '' NOT NULL, language enum('SQL') DEFAULT 'SQL' NOT NULL, sql_data_access enum( 'CONTAINS_SQL', 'NO_SQL', 'READS_SQL_DATA', 'MODIFIES_SQL_DATA') DEFAULT 'CONTAINS_SQL' NOT NULL, is_deterministic enum('YES','NO') DEFAULT 'NO' NOT NULL, security_type enum('INVOKER','DEFINER') DEFAULT 'DEFINER' NOT NULL, param_list blob NOT NULL, returns longblob NOT NULL, body longblob NOT NULL, definer char(141) collate utf8_bin DEFAULT '' NOT NULL, created timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, modified timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', sql_mode set( 'REAL_AS_FLOAT', 'PIPES_AS_CONCAT', 'ANSI_QUOTES', 'IGNORE_SPACE', 'IGNORE_BAD_TABLE_OPTIONS', 'ONLY_FULL_GROUP_BY', 'NO_UNSIGNED_SUBTRACTION', 'NO_DIR_IN_CREATE', 'POSTGRESQL', 'ORACLE', 'MSSQL', 'DB2', 'MAXDB', 'NO_KEY_OPTIONS', 'NO_TABLE_OPTIONS', 'NO_FIELD_OPTIONS', 'MYSQL323', 'MYSQL40', 'ANSI', 'NO_AUTO_VALUE_ON_ZERO', 'NO_BACKSLASH_ESCAPES', 'STRICT_TRANS_TABLES', 'STRICT_ALL_TABLES', 'NO_ZERO_IN_DATE', 'NO_ZERO_DATE', 'INVALID_DATES', 'ERROR_FOR_DIVISION_BY_ZERO', 'TRADITIONAL', 'NO_AUTO_CREATE_USER', 'HIGH_NOT_PRECEDENCE', 'NO_ENGINE_SUBSTITUTION', 'PAD_CHAR_TO_FULL_LENGTH', 'EMPTY_STRING_IS_NULL', 'SIMULTANEOUS_ASSIGNMENT') DEFAULT '' NOT NULL, comment text collate utf8_bin NOT NULL, character_set_client char(32) collate utf8_bin, collation_connection char(32) collate utf8_bin, db_collation char(32) collate utf8_bin, body_utf8 longblob, aggregate enum('NONE', 'GROUP') DEFAULT 'NONE' NOT NULL, PRIMARY KEY (db,name,type)) engine=Aria transactional=1 character set utf8 comment='Stored Procedures'; +CREATE TABLE IF NOT EXISTS proc (db char(64) collate utf8_bin DEFAULT '' NOT NULL, name char(64) DEFAULT '' NOT NULL, type enum('FUNCTION','PROCEDURE','PACKAGE','PACKAGE BODY') NOT NULL, specific_name char(64) DEFAULT '' NOT NULL, language enum('SQL') DEFAULT 'SQL' NOT NULL, sql_data_access enum( 'CONTAINS_SQL', 'NO_SQL', 'READS_SQL_DATA', 'MODIFIES_SQL_DATA') DEFAULT 'CONTAINS_SQL' NOT NULL, is_deterministic enum('YES','NO') DEFAULT 'NO' NOT NULL, security_type enum('INVOKER','DEFINER') DEFAULT 'DEFINER' NOT NULL, param_list blob NOT NULL, returns longblob NOT NULL, body longblob NOT NULL, definer char(141) collate utf8_bin DEFAULT '' NOT NULL, created timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, modified timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', sql_mode set( 'REAL_AS_FLOAT', 'PIPES_AS_CONCAT', 'ANSI_QUOTES', 'IGNORE_SPACE', 'IGNORE_BAD_TABLE_OPTIONS', 'ONLY_FULL_GROUP_BY', 'NO_UNSIGNED_SUBTRACTION', 'NO_DIR_IN_CREATE', 'POSTGRESQL', 'ORACLE', 'MSSQL', 'DB2', 'MAXDB', 'NO_KEY_OPTIONS', 'NO_TABLE_OPTIONS', 'NO_FIELD_OPTIONS', 'MYSQL323', 'MYSQL40', 'ANSI', 'NO_AUTO_VALUE_ON_ZERO', 'NO_BACKSLASH_ESCAPES', 'STRICT_TRANS_TABLES', 'STRICT_ALL_TABLES', 'NO_ZERO_IN_DATE', 'NO_ZERO_DATE', 'INVALID_DATES', 'ERROR_FOR_DIVISION_BY_ZERO', 'TRADITIONAL', 'NO_AUTO_CREATE_USER', 'HIGH_NOT_PRECEDENCE', 'NO_ENGINE_SUBSTITUTION', 'PAD_CHAR_TO_FULL_LENGTH', 'EMPTY_STRING_IS_NULL', 'SIMULTANEOUS_ASSIGNMENT', 'TIME_ROUND_FRACTIONAL') DEFAULT '' NOT NULL, comment text collate utf8_bin NOT NULL, character_set_client char(32) collate utf8_bin, collation_connection char(32) collate utf8_bin, db_collation char(32) collate utf8_bin, body_utf8 longblob, aggregate enum('NONE', 'GROUP') DEFAULT 'NONE' NOT NULL, PRIMARY KEY (db,name,type)) engine=Aria transactional=1 character set utf8 comment='Stored Procedures'; CREATE TABLE IF NOT EXISTS procs_priv ( Host char(60) binary DEFAULT '' NOT NULL, Db char(64) binary DEFAULT '' NOT NULL, User char(80) binary DEFAULT '' NOT NULL, Routine_name char(64) COLLATE utf8_general_ci DEFAULT '' NOT NULL, Routine_type enum('FUNCTION','PROCEDURE','PACKAGE','PACKAGE BODY') NOT NULL, Grantor char(141) DEFAULT '' NOT NULL, Proc_priv set('Execute','Alter Routine','Grant') COLLATE utf8_general_ci DEFAULT '' NOT NULL, Timestamp timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (Host,Db,User,Routine_name,Routine_type), KEY Grantor (Grantor) ) engine=Aria transactional=1 CHARACTER SET utf8 COLLATE utf8_bin comment='Procedure privileges'; @@ -101,7 +151,7 @@ PREPARE stmt FROM @str; EXECUTE stmt; DROP PREPARE stmt; -CREATE TABLE IF NOT EXISTS event ( db char(64) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL default '', name char(64) CHARACTER SET utf8 NOT NULL default '', body longblob NOT NULL, definer char(141) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL default '', execute_at DATETIME default NULL, interval_value int(11) default NULL, interval_field ENUM('YEAR','QUARTER','MONTH','DAY','HOUR','MINUTE','WEEK','SECOND','MICROSECOND','YEAR_MONTH','DAY_HOUR','DAY_MINUTE','DAY_SECOND','HOUR_MINUTE','HOUR_SECOND','MINUTE_SECOND','DAY_MICROSECOND','HOUR_MICROSECOND','MINUTE_MICROSECOND','SECOND_MICROSECOND') default NULL, created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, modified TIMESTAMP NOT NULL DEFAULT '0000-00-00 00:00:00', last_executed DATETIME default NULL, starts DATETIME default NULL, ends DATETIME default NULL, status ENUM('ENABLED','DISABLED','SLAVESIDE_DISABLED') NOT NULL default 'ENABLED', on_completion ENUM('DROP','PRESERVE') NOT NULL default 'DROP', sql_mode set('REAL_AS_FLOAT','PIPES_AS_CONCAT','ANSI_QUOTES','IGNORE_SPACE','IGNORE_BAD_TABLE_OPTIONS','ONLY_FULL_GROUP_BY','NO_UNSIGNED_SUBTRACTION','NO_DIR_IN_CREATE','POSTGRESQL','ORACLE','MSSQL','DB2','MAXDB','NO_KEY_OPTIONS','NO_TABLE_OPTIONS','NO_FIELD_OPTIONS','MYSQL323','MYSQL40','ANSI','NO_AUTO_VALUE_ON_ZERO','NO_BACKSLASH_ESCAPES','STRICT_TRANS_TABLES','STRICT_ALL_TABLES','NO_ZERO_IN_DATE','NO_ZERO_DATE','INVALID_DATES','ERROR_FOR_DIVISION_BY_ZERO','TRADITIONAL','NO_AUTO_CREATE_USER','HIGH_NOT_PRECEDENCE','NO_ENGINE_SUBSTITUTION','PAD_CHAR_TO_FULL_LENGTH','EMPTY_STRING_IS_NULL','SIMULTANEOUS_ASSIGNMENT') DEFAULT '' NOT NULL, comment char(64) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL default '', originator INTEGER UNSIGNED NOT NULL, time_zone char(64) CHARACTER SET latin1 NOT NULL DEFAULT 'SYSTEM', character_set_client char(32) collate utf8_bin, collation_connection char(32) collate utf8_bin, db_collation char(32) collate utf8_bin, body_utf8 longblob, PRIMARY KEY (db, name) ) engine=Aria transactional=1 DEFAULT CHARSET=utf8 COMMENT 'Events'; +CREATE TABLE IF NOT EXISTS event ( db char(64) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL default '', name char(64) CHARACTER SET utf8 NOT NULL default '', body longblob NOT NULL, definer char(141) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL default '', execute_at DATETIME default NULL, interval_value int(11) default NULL, interval_field ENUM('YEAR','QUARTER','MONTH','DAY','HOUR','MINUTE','WEEK','SECOND','MICROSECOND','YEAR_MONTH','DAY_HOUR','DAY_MINUTE','DAY_SECOND','HOUR_MINUTE','HOUR_SECOND','MINUTE_SECOND','DAY_MICROSECOND','HOUR_MICROSECOND','MINUTE_MICROSECOND','SECOND_MICROSECOND') default NULL, created TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, modified TIMESTAMP NOT NULL DEFAULT '0000-00-00 00:00:00', last_executed DATETIME default NULL, starts DATETIME default NULL, ends DATETIME default NULL, status ENUM('ENABLED','DISABLED','SLAVESIDE_DISABLED') NOT NULL default 'ENABLED', on_completion ENUM('DROP','PRESERVE') NOT NULL default 'DROP', sql_mode set('REAL_AS_FLOAT','PIPES_AS_CONCAT','ANSI_QUOTES','IGNORE_SPACE','IGNORE_BAD_TABLE_OPTIONS','ONLY_FULL_GROUP_BY','NO_UNSIGNED_SUBTRACTION','NO_DIR_IN_CREATE','POSTGRESQL','ORACLE','MSSQL','DB2','MAXDB','NO_KEY_OPTIONS','NO_TABLE_OPTIONS','NO_FIELD_OPTIONS','MYSQL323','MYSQL40','ANSI','NO_AUTO_VALUE_ON_ZERO','NO_BACKSLASH_ESCAPES','STRICT_TRANS_TABLES','STRICT_ALL_TABLES','NO_ZERO_IN_DATE','NO_ZERO_DATE','INVALID_DATES','ERROR_FOR_DIVISION_BY_ZERO','TRADITIONAL','NO_AUTO_CREATE_USER','HIGH_NOT_PRECEDENCE','NO_ENGINE_SUBSTITUTION','PAD_CHAR_TO_FULL_LENGTH','EMPTY_STRING_IS_NULL','SIMULTANEOUS_ASSIGNMENT','TIME_ROUND_FRACTIONAL') DEFAULT '' NOT NULL, comment char(64) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL default '', originator INTEGER UNSIGNED NOT NULL, time_zone char(64) CHARACTER SET latin1 NOT NULL DEFAULT 'SYSTEM', character_set_client char(32) collate utf8_bin, collation_connection char(32) collate utf8_bin, db_collation char(32) collate utf8_bin, body_utf8 longblob, PRIMARY KEY (db, name) ) engine=Aria transactional=1 DEFAULT CHARSET=utf8 COMMENT 'Events'; SET @create_innodb_table_stats="CREATE TABLE IF NOT EXISTS innodb_table_stats ( database_name VARCHAR(64) NOT NULL, @@ -232,6 +282,13 @@ CREATE TABLE IF NOT EXISTS proxies_priv (Host char(60) binary DEFAULT '' NOT NUL -- Remember for later if proxies_priv table already existed set @had_proxies_priv_table= @@warning_count != 0; +-- The following needs to be done both for new installations +-- and for upgrades +CREATE TEMPORARY TABLE tmp_proxies_priv LIKE proxies_priv; +INSERT INTO tmp_proxies_priv VALUES ('localhost', 'root', '', '', TRUE, '', now()); +INSERT INTO proxies_priv SELECT * FROM tmp_proxies_priv WHERE @had_proxies_priv_table=0; +DROP TABLE tmp_proxies_priv; + -- -- Tables unique for MariaDB -- diff --git a/scripts/mysql_system_tables_data.sql b/scripts/mysql_system_tables_data.sql index 4761fe51dcc..60867fc2bc3 100644 --- a/scripts/mysql_system_tables_data.sql +++ b/scripts/mysql_system_tables_data.sql @@ -25,28 +25,28 @@ -- add escape character in front of wildcard character to convert "_" or "%" to -- a plain character SELECT LOWER( REPLACE((SELECT REPLACE(@@hostname,'_','\_')),'%','\%') )INTO @current_hostname; +SELECT json_object('access',cast(-1 as unsigned)) INTO @all_privileges; --- Fill "user" table with default users allowing root access --- from local machine if "user" table didn't exist before -CREATE TEMPORARY TABLE tmp_user_nopasswd LIKE user; -CREATE TEMPORARY TABLE tmp_user_socket LIKE user; +-- Fill "global_priv" table with default users allowing root access +-- from local machine if "global_priv" table didn't exist before +CREATE TEMPORARY TABLE tmp_user_nopasswd LIKE global_priv; +CREATE TEMPORARY TABLE tmp_user_socket LIKE global_priv; -- Classic passwordless root account. -INSERT INTO tmp_user_nopasswd VALUES ('localhost','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N', 'N','', 0); -REPLACE INTO tmp_user_nopasswd SELECT @current_hostname,'root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N','',0 FROM dual WHERE @current_hostname != 'localhost'; -REPLACE INTO tmp_user_nopasswd VALUES ('127.0.0.1','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N','',0); -REPLACE INTO tmp_user_nopasswd VALUES ('::1','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N', '', 0); +INSERT INTO tmp_user_nopasswd VALUES ('localhost','root',@all_privileges); +REPLACE INTO tmp_user_nopasswd SELECT @current_hostname,'root',@all_privileges FROM dual WHERE @current_hostname != 'localhost'; +REPLACE INTO tmp_user_nopasswd VALUES ('127.0.0.1','root',@all_privileges); +REPLACE INTO tmp_user_nopasswd VALUES ('::1','root',@all_privileges); -- More secure root account using unix socket auth. -INSERT INTO tmp_user_socket VALUES ('localhost',IFNULL(@auth_root_socket, 'root'),'','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'unix_socket','','N', 'N','', 0); +INSERT INTO tmp_user_socket VALUES ('localhost',IFNULL(@auth_root_socket, 'root'),json_set(@all_privileges, '$.plugin', 'unix_socket')); IF @auth_root_socket is not null THEN IF not exists(select 1 from information_schema.plugins where plugin_name='unix_socket') THEN INSTALL SONAME 'auth_socket'; END IF; END IF; -INSERT INTO user SELECT * FROM tmp_user_nopasswd WHERE @had_user_table=0 AND @skip_auth_root_nopasswd IS NULL; -INSERT INTO user SELECT * FROM tmp_user_socket WHERE @had_user_table=0 AND @auth_root_socket IS NOT NULL; +INSERT INTO global_priv SELECT * FROM tmp_user_nopasswd WHERE @had_user_table=0 AND @skip_auth_root_nopasswd IS NULL; +INSERT INTO global_priv SELECT * FROM tmp_user_socket WHERE @had_user_table=0 AND @auth_root_socket IS NOT NULL; DROP TABLE tmp_user_nopasswd, tmp_user_socket; CREATE TEMPORARY TABLE tmp_proxies_priv LIKE proxies_priv; -INSERT INTO tmp_proxies_priv VALUES ('localhost', 'root', '', '', TRUE, '', now()); -REPLACE INTO tmp_proxies_priv SELECT @current_hostname, 'root', '', '', TRUE, '', now() FROM DUAL WHERE @current_hostname != 'localhost'; +INSERT INTO tmp_proxies_priv SELECT @current_hostname, 'root', '', '', TRUE, '', now() FROM DUAL WHERE @current_hostname != 'localhost'; INSERT INTO proxies_priv SELECT * FROM tmp_proxies_priv WHERE @had_proxies_priv_table=0; DROP TABLE tmp_proxies_priv; diff --git a/scripts/mysql_system_tables_fix.sql b/scripts/mysql_system_tables_fix.sql index 90b84732392..e80569d6636 100644 --- a/scripts/mysql_system_tables_fix.sql +++ b/scripts/mysql_system_tables_fix.sql @@ -28,6 +28,8 @@ set sql_mode=''; set storage_engine=Aria; set enforce_storage_engine=NULL; +set @have_innodb= (select count(engine) from information_schema.engines where engine='INNODB' and support != 'NO'); + ALTER TABLE user add File_priv enum('N','Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL; # Detect whether or not we had the Grant_priv column @@ -417,19 +419,17 @@ ALTER TABLE proc MODIFY name char(64) DEFAULT '' NOT NULL, 'NO_ENGINE_SUBSTITUTION', 'PAD_CHAR_TO_FULL_LENGTH', 'EMPTY_STRING_IS_NULL', - 'SIMULTANEOUS_ASSIGNMENT' + 'SIMULTANEOUS_ASSIGNMENT', + 'TIME_ROUND_FRACTIONAL' ) DEFAULT '' NOT NULL, DEFAULT CHARACTER SET utf8; # Correct the character set and collation -ALTER TABLE proc CONVERT TO CHARACTER SET utf8; # Reset some fields after the conversion -ALTER TABLE proc MODIFY db - char(64) collate utf8_bin DEFAULT '' NOT NULL, - MODIFY definer - char(141) collate utf8_bin DEFAULT '' NOT NULL, - MODIFY comment - text collate utf8_bin NOT NULL; +ALTER TABLE proc CONVERT TO CHARACTER SET utf8, + MODIFY db char(64) binary DEFAULT '' NOT NULL, + MODIFY definer char(141) binary DEFAULT '' NOT NULL, + MODIFY comment text binary NOT NULL; ALTER TABLE proc ADD character_set_client char(32) collate utf8_bin DEFAULT NULL @@ -556,7 +556,8 @@ ALTER TABLE event MODIFY sql_mode 'NO_ENGINE_SUBSTITUTION', 'PAD_CHAR_TO_FULL_LENGTH', 'EMPTY_STRING_IS_NULL', - 'SIMULTANEOUS_ASSIGNMENT' + 'SIMULTANEOUS_ASSIGNMENT', + 'TIME_ROUND_FRACTIONAL' ) DEFAULT '' NOT NULL AFTER on_completion; ALTER TABLE event MODIFY name char(64) CHARACTER SET utf8 NOT NULL default ''; @@ -650,30 +651,18 @@ ALTER TABLE user ADD max_statement_time decimal(12,6) DEFAULT 0 NOT NULL; ALTER TABLE user MODIFY password_expired ENUM('N', 'Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL; ALTER TABLE user MODIFY is_role enum('N', 'Y') COLLATE utf8_general_ci DEFAULT 'N' NOT NULL; --- Need to pre-fill mysql.proxies_priv with access for root even when upgrading from --- older versions - -CREATE TEMPORARY TABLE tmp_proxies_priv LIKE proxies_priv; -INSERT INTO tmp_proxies_priv VALUES ('localhost', 'root', '', '', TRUE, '', now()); -INSERT INTO proxies_priv SELECT * FROM tmp_proxies_priv WHERE @had_proxies_priv_table=0; -DROP TABLE tmp_proxies_priv; - -- Checking for any duplicate hostname and username combination are exists. -- If exits we will throw error. -DROP PROCEDURE IF EXISTS mysql.count_duplicate_host_names; DELIMITER // -CREATE PROCEDURE mysql.count_duplicate_host_names() -BEGIN +BEGIN NOT ATOMIC SET @duplicate_hosts=(SELECT count(*) FROM mysql.user GROUP BY user, lower(host) HAVING count(*) > 1 LIMIT 1); IF @duplicate_hosts > 1 THEN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = 'Multiple accounts exist for @user_name, @host_name that differ only in Host lettercase; remove all except one of them'; END IF; END // DELIMITER ; -CALL mysql.count_duplicate_host_names(); -- Get warnings (if any) SHOW WARNINGS; -DROP PROCEDURE mysql.count_duplicate_host_names; # Convering the host name to lower case for existing users UPDATE user SET host=LOWER( host ) WHERE LOWER( host ) <> host; @@ -761,7 +750,7 @@ ALTER TABLE proc ENGINE=Aria transactional=1; ALTER TABLE event ENGINE=Aria transactional=1; ALTER TABLE proxies_priv ENGINE=Aria transactional=1; --- The folloing tables doesn't have to be transactional +-- The following tables doesn't have to be transactional ALTER TABLE help_topic ENGINE=Aria transactional=0; ALTER TABLE help_category ENGINE=Aria transactional=0; ALTER TABLE help_relation ENGINE=Aria transactional=0; @@ -769,3 +758,55 @@ ALTER TABLE help_keyword ENGINE=Aria transactional=0; ALTER TABLE table_stats ENGINE=Aria transactional=0; ALTER TABLE column_stats ENGINE=Aria transactional=0; ALTER TABLE index_stats ENGINE=Aria transactional=0; + +DELIMITER // +IF 'BASE TABLE' = (select table_type from information_schema.tables where table_name='user') THEN + CREATE TABLE IF NOT EXISTS global_priv (Host char(60) binary DEFAULT '', User char(80) binary DEFAULT '', Priv JSON NOT NULL DEFAULT '{}' CHECK(JSON_VALID(Priv)), PRIMARY KEY Host (Host,User)) engine=Aria transactional=1 CHARACTER SET utf8 COLLATE utf8_bin comment='Users and global privileges' + SELECT Host, User, JSON_COMPACT(JSON_OBJECT('access', + 1*('Y'=Select_priv)+ + 2*('Y'=Insert_priv)+ + 4*('Y'=Update_priv)+ + 8*('Y'=Delete_priv)+ + 16*('Y'=Create_priv)+ + 32*('Y'=Drop_priv)+ + 64*('Y'=Reload_priv)+ + 128*('Y'=Shutdown_priv)+ + 256*('Y'=Process_priv)+ + 512*('Y'=File_priv)+ + 1024*('Y'=Grant_priv)+ + 2048*('Y'=References_priv)+ + 4096*('Y'=Index_priv)+ + 8192*('Y'=Alter_priv)+ + 16384*('Y'=Show_db_priv)+ + 32768*('Y'=Super_priv)+ + 65536*('Y'=Create_tmp_table_priv)+ + 131072*('Y'=Lock_tables_priv)+ + 262144*('Y'=Execute_priv)+ + 524288*('Y'=Repl_slave_priv)+ + 1048576*('Y'=Repl_client_priv)+ + 2097152*('Y'=Create_view_priv)+ + 4194304*('Y'=Show_view_priv)+ + 8388608*('Y'=Create_routine_priv)+ + 16777216*('Y'=Alter_routine_priv)+ + 33554432*('Y'=Create_user_priv)+ + 67108864*('Y'=Event_priv)+ + 134217728*('Y'=Trigger_priv)+ + 268435456*('Y'=Create_tablespace_priv)+ + 536870912*('Y'=Delete_history_priv), + 'ssl_type', ssl_type-1, + 'ssl_cipher', ssl_cipher, + 'x509_issuer', x509_issuer, + 'x509_subject', x509_subject, + 'max_questions', max_questions, + 'max_updates', max_updates, + 'max_connections', max_connections, + 'max_user_connections', max_user_connections, + 'max_statement_time', max_statement_time, + 'plugin', if(plugin>'',plugin,if(length(password)=16,'mysql_old_password','mysql_native_password')), + 'authentication_string', if(plugin>'' and authentication_string>'',authentication_string,password), + 'default_role', default_role, + 'is_role', 'Y'=is_role)) as Priv + FROM user; + DROP TABLE user; +END IF// +DELIMITER ; diff --git a/scripts/mysql_test_db.sql b/scripts/mysql_test_db.sql index c1bb3661ec3..9f8a0cf604c 100644 --- a/scripts/mysql_test_db.sql +++ b/scripts/mysql_test_db.sql @@ -24,8 +24,8 @@ INSERT INTO db SELECT * FROM tmp_db WHERE @had_db_table=0; DROP TABLE tmp_db; -- Anonymous user with no privileges. -CREATE TEMPORARY TABLE tmp_user_anonymous LIKE user; +CREATE TEMPORARY TABLE tmp_user_anonymous LIKE global_priv; INSERT INTO tmp_user_anonymous (host,user) VALUES ('localhost',''); INSERT INTO tmp_user_anonymous (host,user) SELECT @current_hostname,'' FROM dual WHERE @current_hostname != 'localhost'; -INSERT INTO user SELECT * FROM tmp_user_anonymous WHERE @had_user_table=0; +INSERT INTO global_priv SELECT * FROM tmp_user_anonymous WHERE @had_user_table=0; DROP TABLE tmp_user_anonymous; diff --git a/scripts/mysqld_safe.sh b/scripts/mysqld_safe.sh index 5797bdc68d7..5f09ac3c235 100644 --- a/scripts/mysqld_safe.sh +++ b/scripts/mysqld_safe.sh @@ -265,7 +265,16 @@ wsrep_recover_position() { wsrep_start_position_opt="--wsrep_start_position=$start_pos" fi - [ $ret -eq 0 ] && rm $wr_logfile + if [ $ret -eq 0 ] ; then + local wr_logfile_permanent="$DATADIR/wsrep_recovery.ok" + else + local wr_logfile_permanent="$DATADIR/wsrep_recovery.fail" + fi + touch $wr_logfile_permanent + [ "$euid" = "0" ] && chown $user $wr_logfile_permanent + chmod 600 $wr_logfile_permanent + cat "$wr_logfile" >> $wr_logfile_permanent + rm -f "$wr_logfile" return $ret } diff --git a/scripts/mytop.sh b/scripts/mytop.sh index 17a87e9efd2..3ef0a59f27f 100644 --- a/scripts/mytop.sh +++ b/scripts/mytop.sh @@ -437,7 +437,7 @@ while (1) if ($key eq 'C') { - if ( $HAS_COLOR ) + if ( $HAS_COLOR ) { $HAS_COLOR = 0; } @@ -817,11 +817,11 @@ sub GetData() if ($config{header}) { my @recs = ""; - if ( $db_release > 4 ) + if ( $db_release > 4 ) { @recs = Hashes("show global status"); - } - else + } + else { @recs = Hashes("show status"); } @@ -978,7 +978,7 @@ sub GetData() # print("q_diff: $STATUS{Questions} - $OLD_STATUS{Questions} / $t_delta = $q_diff\n"); printf(" Sorts: %5.0f qps now: %4.0f Slow qps: %3.1f Threads: %4.0f (%4.0f/%4.0f) %02.0f/%02.0f/%02.0f/%02.0f\n", - ( $STATUS{Sort_rows} - $OLD_STATUS{Sort_rows} ) / $t_delta, + ( $STATUS{Sort_rows} - $OLD_STATUS{Sort_rows} ) / $t_delta, ( $STATUS{Questions} - $OLD_STATUS{Questions} ) / $t_delta, ( # slow now (qps) ($STATUS{Slow_queries} ) ? @@ -989,7 +989,7 @@ sub GetData() $STATUS{Threads_running}, $STATUS{Threads_cached}, - (100 * ($STATUS{Com_select} - $OLD_STATUS{Com_select} + + (100 * ($STATUS{Com_select} - $OLD_STATUS{Com_select} + ($STATUS{Qcache_hits}||0) - ($OLD_STATUS{Qcache_hits}||0) ) ) / ($q_diff ), (100 * ($STATUS{Com_insert} - $OLD_STATUS{Com_insert} + @@ -1075,7 +1075,7 @@ sub GetData() $t_delta, ($STATUS{Rows_tmp_read} - $OLD_STATUS{Rows_tmp_read}) / $t_delta, - ($STATUS{Handler_tmp_write} + ($STATUS{Handler_tmp_write} -$OLD_STATUS{Handler_tmp_write})/$t_delta, ($STATUS{Handler_tmp_update} - $OLD_STATUS{Handler_tmp_update})/$t_delta); @@ -1119,6 +1119,7 @@ sub GetData() } } print " Replication "; + print "Master:$data->{Master_Host} "; print "IO:$data->{Slave_IO_Running} "; print "SQL:$data->{Slave_SQL_Running} "; print RESET() if ($HAS_COLOR); @@ -1225,9 +1226,9 @@ sub GetData() $thread->{State} ||= ""; $thread->{Progress} ||= 0; - ## alter double hyphen comments so they don't break + ## alter double hyphen comments so they don't break ## the query when newlines are removed - http://freshmeat.net/users/jerjones - $thread->{Info} =~ s~\s--(.*)$~ /* $1 */ ~mg; + $thread->{Info} =~ s~\s--(.*)$~ /* $1 */ ~mg; ## Normalize spaces -- mostly disabled for now. This can ## break EXPLAIN if you try to explain a mangled query. It diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 313821f522d..3e82d379a6d 100755 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -20,6 +20,7 @@ set -u WSREP_SST_OPT_BYPASS=0 WSREP_SST_OPT_BINLOG="" +WSREP_SST_OPT_BINLOG_INDEX="" WSREP_SST_OPT_DATA="" WSREP_SST_OPT_AUTH=${WSREP_SST_OPT_AUTH:-} WSREP_SST_OPT_USER=${WSREP_SST_OPT_USER:-} @@ -28,6 +29,7 @@ WSREP_SST_OPT_DEFAULT="" WSREP_SST_OPT_EXTRA_DEFAULT="" WSREP_SST_OPT_SUFFIX_DEFAULT="" WSREP_SST_OPT_SUFFIX_VALUE="" +INNODB_DATA_HOME_DIR_ARG="" while [ $# -gt 0 ]; do case "$1" in @@ -42,22 +44,33 @@ case "$1" in addr_no_bracket=${WSREP_SST_OPT_ADDR#\[} readonly WSREP_SST_OPT_HOST_UNESCAPED=${addr_no_bracket%%\]*} readonly WSREP_SST_OPT_HOST="[${WSREP_SST_OPT_HOST_UNESCAPED}]" + readonly WSREP_SST_OPT_HOST_ESCAPED="\\[${WSREP_SST_OPT_HOST_UNESCAPED}\\]" ;; *) readonly WSREP_SST_OPT_HOST=${WSREP_SST_OPT_ADDR%%[:/]*} readonly WSREP_SST_OPT_HOST_UNESCAPED=$WSREP_SST_OPT_HOST + readonly WSREP_SST_OPT_HOST_ESCAPED=$WSREP_SST_OPT_HOST ;; esac - remain=${WSREP_SST_OPT_ADDR#${WSREP_SST_OPT_HOST}} + remain=${WSREP_SST_OPT_ADDR#${WSREP_SST_OPT_HOST_ESCAPED}} remain=${remain#:} readonly WSREP_SST_OPT_ADDR_PORT=${remain%%/*} remain=${remain#*/} readonly WSREP_SST_OPT_MODULE=${remain%%/*} readonly WSREP_SST_OPT_PATH=${WSREP_SST_OPT_ADDR#*/} remain=${WSREP_SST_OPT_PATH#*/} - readonly WSREP_SST_OPT_LSN=${remain%%/*} - remain=${remain#*/} - readonly WSREP_SST_OPT_SST_VER=${remain%%/*} + if [ "$remain" != "${WSREP_SST_OPT_PATH}" ]; then + readonly WSREP_SST_OPT_LSN=${remain%%/*} + remain=${remain#*/} + if [ "$remain" != "${WSREP_SST_OPT_LSN}" ]; then + readonly WSREP_SST_OPT_SST_VER=${remain%%/*} + else + readonly WSREP_SST_OPT_SST_VER="" + fi + else + readonly WSREP_SST_OPT_LSN="" + readonly WSREP_SST_OPT_SST_VER="" + fi shift ;; '--bypass') @@ -67,6 +80,10 @@ case "$1" in readonly WSREP_SST_OPT_DATA="$2" shift ;; + '--innodb-data-home-dir') + readonly INNODB_DATA_HOME_DIR_ARG="$2" + shift + ;; '--defaults-file') readonly WSREP_SST_OPT_DEFAULT="$1=$2" shift @@ -120,6 +137,10 @@ case "$1" in WSREP_SST_OPT_BINLOG="$2" shift ;; + '--binlog-index') + WSREP_SST_OPT_BINLOG_INDEX="$2" + shift + ;; '--gtid-domain-id') readonly WSREP_SST_OPT_GTID_DOMAIN_ID="$2" shift @@ -133,6 +154,7 @@ shift done readonly WSREP_SST_OPT_BYPASS readonly WSREP_SST_OPT_BINLOG +readonly WSREP_SST_OPT_BINLOG_INDEX if [ -n "${WSREP_SST_OPT_ADDR_PORT:-}" ]; then if [ -n "${WSREP_SST_OPT_PORT:-}" ]; then @@ -254,7 +276,7 @@ wsrep_check_programs() } # -# user can specify xtrabackup specific settings that will be used during sst +# user can specify mariabackup specific settings that will be used during sst # process like encryption, etc..... # parse such configuration option. (group for xb settings is [sst] in my.cnf # diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 8173db26291..5362389a59d 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -22,6 +22,7 @@ . $(dirname $0)/wsrep_sst_common +OS=$(uname) ealgo="" ekey="" ekeyfile="" @@ -82,7 +83,7 @@ fi pcmd="pv $pvopts" declare -a RC -INNOBACKUPEX_BIN=mariabackup +INNOBACKUPEX_BIN=$(which mariabackup) XBSTREAM_BIN=mbstream XBCRYPT_BIN=xbcrypt # Not available in MariaBackup @@ -90,6 +91,9 @@ DATA="${WSREP_SST_OPT_DATA}" INFO_FILE="xtrabackup_galera_info" IST_FILE="xtrabackup_ist" MAGIC_FILE="${DATA}/${INFO_FILE}" +INNOAPPLYLOG="${DATA}/mariabackup.prepare.log" +INNOMOVELOG="${DATA}/mariabackup.move.log" +INNOBACKUPLOG="${DATA}/mariabackup.backup.log" # Setting the path for ss and ip export PATH="/usr/sbin:/sbin:$PATH" @@ -327,6 +331,7 @@ read_cnf() rebuild=$(parse_cnf sst rebuild 0) ttime=$(parse_cnf sst time 0) cpat=$(parse_cnf sst cpat '.*galera\.cache$\|.*sst_in_progress$\|.*\.sst$\|.*gvwstate\.dat$\|.*grastate\.dat$\|.*\.err$\|.*\.log$\|.*RPM_UPGRADE_MARKER$\|.*RPM_UPGRADE_HISTORY$') + [[ $OS == "FreeBSD" ]] && cpat=$(parse_cnf sst cpat '.*galera\.cache$|.*sst_in_progress$|.*\.sst$|.*gvwstate\.dat$|.*grastate\.dat$|.*\.err$|.*\.log$|.*RPM_UPGRADE_MARKER$|.*RPM_UPGRADE_HISTORY$') ealgo=$(parse_cnf xtrabackup encrypt "") ekey=$(parse_cnf xtrabackup encrypt-key "") ekeyfile=$(parse_cnf xtrabackup encrypt-key-file "") @@ -350,6 +355,8 @@ read_cnf() ssyslog=$(parse_cnf sst sst-syslog 0) ssystag=$(parse_cnf mysqld_safe syslog-tag "${SST_SYSLOG_TAG:-}") ssystag+="-" + sstlogarchive=$(parse_cnf sst sst-log-archive 1) + sstlogarchivedir=$(parse_cnf sst sst-log-archive-dir "/tmp/sst_log_archive") if [[ $speciald -eq 0 ]];then wsrep_log_error "sst-special-dirs equal to 0 is not supported, falling back to 1" @@ -509,12 +516,24 @@ kill_xtrabackup() setup_ports() { if [[ "$WSREP_SST_OPT_ROLE" == "donor" ]];then - SST_PORT=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $2 }') - REMOTEIP=$(echo $WSREP_SST_OPT_ADDR | awk -F ':' '{ print $1 }') - lsn=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $4 }') - sst_ver=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $5 }') + if [[ ${WSREP_SST_OPT_ADDR:0:1} == '[' ]];then + remain=$(echo $WSREP_SST_OPT_ADDR | awk -F '\\][:/]' '{ print $2 }') + REMOTEIP=$(echo $WSREP_SST_OPT_ADDR | awk -F '\\]:' '{ print $1 }')"]" + SST_PORT=$(echo $remain | awk -F '[:/]' '{ print $1 }') + lsn=$(echo $remain | awk -F '[:/]' '{ print $3 }') + sst_ver=$(echo $remain | awk -F '[:/]' '{ print $4 }') + else + SST_PORT=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $2 }') + REMOTEIP=$(echo $WSREP_SST_OPT_ADDR | awk -F ':' '{ print $1 }') + lsn=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $4 }') + sst_ver=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $5 }') + fi else - SST_PORT=$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $2 }') + if [[ ${WSREP_SST_OPT_ADDR:0:1} == '[' ]];then + SST_PORT=$(echo ${WSREP_SST_OPT_ADDR} | awk -F '\\]:' '{ print $2 }') + else + SST_PORT=$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $2 }') + fi fi } @@ -527,7 +546,11 @@ wait_for_listen() local MODULE=$3 for i in {1..50} do - ss -p state listening "( sport = :$PORT )" | grep -qE 'socat|nc' && break + if [ "$OS" = "FreeBSD" ];then + sockstat -46lp $PORT | grep -qE "^[^ ]* *(socat|nc) *[^ ]* *[^ ]* *[^ ]* *[^ ]*:$PORT" && break + else + ss -p state listening "( sport = :$PORT )" | grep -qE 'socat|nc' && break + fi sleep 0.2 done echo "ready ${ADDR}/${MODULE}//$sst_ver" @@ -639,13 +662,12 @@ monitor_process() while true ; do - if ! ps --pid "${WSREP_SST_OPT_PARENT}" &>/dev/null; then + if ! ps -p "${WSREP_SST_OPT_PARENT}" &>/dev/null; then wsrep_log_error "Parent mysqld process (PID:${WSREP_SST_OPT_PARENT}) terminated unexpectedly." - kill -- -"${WSREP_SST_OPT_PARENT}" exit 32 fi - if ! ps --pid "${sst_stream_pid}" &>/dev/null; then + if ! ps -p "${sst_stream_pid}" &>/dev/null; then break fi @@ -678,6 +700,30 @@ fi INNOEXTRA="" +INNODB_DATA_HOME_DIR=${INNODB_DATA_HOME_DIR:-""} +# Try to set INNODB_DATA_HOME_DIR from the command line: +if [ ! -z "$INNODB_DATA_HOME_DIR_ARG" ]; then + INNODB_DATA_HOME_DIR=$INNODB_DATA_HOME_DIR_ARG +fi +# if INNODB_DATA_HOME_DIR env. variable is not set, try to get it from my.cnf +if [ -z "$INNODB_DATA_HOME_DIR" ]; then + INNODB_DATA_HOME_DIR=$(parse_cnf mysqld$WSREP_SST_OPT_SUFFIX_VALUE innodb-data-home-dir '') +fi +if [ -z "$INNODB_DATA_HOME_DIR" ]; then + INNODB_DATA_HOME_DIR=$(parse_cnf --mysqld innodb-data-home-dir "") +fi +if [ ! -z "$INNODB_DATA_HOME_DIR" ]; then + INNOEXTRA+=" --innodb-data-home-dir=$INNODB_DATA_HOME_DIR" +fi + +if [ -n "$INNODB_DATA_HOME_DIR" ]; then + # handle both relative and absolute paths + INNODB_DATA_HOME_DIR=$(cd $DATA; mkdir -p "$INNODB_DATA_HOME_DIR"; cd $INNODB_DATA_HOME_DIR; pwd -P) +else + # default to datadir + INNODB_DATA_HOME_DIR=$(cd $DATA; pwd -P) +fi + if [[ $ssyslog -eq 1 ]];then if ! command -v logger >/dev/null;then @@ -698,15 +744,73 @@ if [[ $ssyslog -eq 1 ]];then logger -p daemon.info -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" } - INNOAPPLY="${INNOBACKUPEX_BIN} --innobackupex $disver $iapts --apply-log \$rebuildcmd \${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-apply " + INNOAPPLY="${INNOBACKUPEX_BIN} --innobackupex $disver $iapts \$INNOEXTRA --apply-log \$rebuildcmd \${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-apply " INNOMOVE="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $impts --move-back --force-non-empty-directories \${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move " INNOBACKUP="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)" fi -else - INNOAPPLY="${INNOBACKUPEX_BIN} --innobackupex $disver $iapts --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log" - INNOMOVE="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $impts --move-back --force-non-empty-directories \${DATA} &>\${DATA}/innobackup.move.log" - INNOBACKUP="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2>\${DATA}/innobackup.backup.log" +else + +if [[ "$sstlogarchive" -eq 1 ]] +then + ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S") + newfile="" + + if [[ ! -z "$sstlogarchivedir" ]] + then + if [[ ! -d "$sstlogarchivedir" ]] + then + mkdir -p "$sstlogarchivedir" + fi + fi + + if [ -e "${INNOAPPLYLOG}" ] + then + if [[ ! -z "$sstlogarchivedir" ]] + then + newfile=$sstlogarchivedir/$(basename "${INNOAPPLYLOG}").${ARCHIVETIMESTAMP} + else + newfile=${INNOAPPLYLOG}.${ARCHIVETIMESTAMP} + fi + + wsrep_log_info "Moving ${INNOAPPLYLOG} to ${newfile}" + mv "${INNOAPPLYLOG}" "${newfile}" + gzip "${newfile}" + fi + + if [ -e "${INNOMOVELOG}" ] + then + if [[ ! -z "$sstlogarchivedir" ]] + then + newfile=$sstlogarchivedir/$(basename "${INNOMOVELOG}").${ARCHIVETIMESTAMP} + else + newfile=${INNOMOVELOG}.${ARCHIVETIMESTAMP} + fi + + wsrep_log_info "Moving ${INNOMOVELOG} to ${newfile}" + mv "${INNOMOVELOG}" "${newfile}" + gzip "${newfile}" + fi + + if [ -e "${INNOBACKUPLOG}" ] + then + if [[ ! -z "$sstlogarchivedir" ]] + then + newfile=$sstlogarchivedir/$(basename "${INNOBACKUPLOG}").${ARCHIVETIMESTAMP} + else + newfile=${INNOBACKUPLOG}.${ARCHIVETIMESTAMP} + fi + + wsrep_log_info "Moving ${INNOBACKUPLOG} to ${newfile}" + mv "${INNOBACKUPLOG}" "${newfile}" + gzip "${newfile}" + fi + +fi + + INNOAPPLY="${INNOBACKUPEX_BIN} --innobackupex $disver $iapts \$INNOEXTRA --apply-log \$rebuildcmd \${DATA} &> ${INNOAPPLYLOG}" + INNOMOVE="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $impts --move-back --force-non-empty-directories \${DATA} &> ${INNOMOVELOG}" + INNOBACKUP="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2> ${INNOBACKUPLOG}" fi get_stream @@ -803,7 +907,7 @@ then if [ ${RC[0]} -ne 0 ]; then wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \ - "Check ${DATA}/innobackup.backup.log" + "Check syslog or ${INNOBACKUPLOG} for details" exit 22 elif [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then wsrep_log_error "$tcmd finished with error: ${RC[1]}" @@ -847,7 +951,7 @@ then [[ -e $SST_PROGRESS_FILE ]] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" [[ -n $SST_PROGRESS_FILE ]] && touch $SST_PROGRESS_FILE - ib_home_dir=$(parse_cnf mysqld innodb-data-home-dir "") + ib_home_dir=$INNODB_DATA_HOME_DIR ib_log_dir=$(parse_cnf mysqld innodb-log-group-home-dir "") ib_undo_dir=$(parse_cnf mysqld innodb-undo-directory "") @@ -868,7 +972,11 @@ then if [ -z "${SST_PORT}" ] then SST_PORT=4444 - ADDR="$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $1 }'):${SST_PORT}" + if [[ ${ADDR:0:1} == '[' ]];then + ADDR="$(echo ${WSREP_SST_OPT_ADDR} | awk -F '\\]:' '{ print $1 }')]:${SST_PORT}" + else + ADDR="$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $1 }'):${SST_PORT}" + fi fi wait_for_listen ${SST_PORT} ${ADDR} ${MODULE} & @@ -917,7 +1025,11 @@ then wsrep_log_info "Cleaning the existing datadir and innodb-data/log directories" - find $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -regex $cpat -prune -o -exec rm -rfv {} 1>&2 \+ + if [ "${OS}" = "FreeBSD" ]; then + find -E $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -prune -regex $cpat -o -exec rm -rfv {} 1>&2 \+ + else + find $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -prune -regex $cpat -o -exec rm -rfv {} 1>&2 \+ + fi tempdir=$(parse_cnf mysqld log-bin "") if [[ -n ${tempdir:-} ]];then @@ -1024,13 +1136,12 @@ then if [ $? -ne 0 ]; then - wsrep_log_error "${INNOBACKUPEX_BIN} apply finished with errors. Check ${DATA}/innobackup.prepare.log" + wsrep_log_error "${INNOBACKUPEX_BIN} apply finished with errors. Check syslog or ${INNOAPPLYLOG} for details" exit 22 fi MAGIC_FILE="${TDATA}/${INFO_FILE}" set +e - rm $TDATA/innobackup.prepare.log $TDATA/innobackup.move.log set -e wsrep_log_info "Moving the backup to ${TDATA}" timeit "Xtrabackup move stage" "$INNOMOVE" @@ -1040,7 +1151,7 @@ then DATA=${TDATA} else wsrep_log_error "Move failed, keeping ${DATA} for further diagnosis" - wsrep_log_error "Check ${DATA}/innobackup.move.log for details" + wsrep_log_error "Check syslog or ${INNOMOVELOG} for details" exit 22 fi diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index faa3f10639b..d36deb5759f 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -25,6 +25,7 @@ EINVAL=22 local_ip() { [ "$1" = "127.0.0.1" ] && return 0 + [ "$1" = "127.0.0.2" ] && return 0 [ "$1" = "localhost" ] && return 0 [ "$1" = "[::1]" ] && return 0 [ "$1" = "$(hostname -s)" ] && return 0 @@ -138,8 +139,8 @@ then # turned off for the session so that gtid state does not get altered while # the dump gets replayed on joiner. if [[ "$LOG_BIN" == 'ON' ]]; then - RESET_MASTER="RESET MASTER;" - SET_GTID_BINLOG_STATE="SET @@global.gtid_binlog_state='$GTID_BINLOG_STATE';" + RESET_MASTER="SET GLOBAL wsrep_on=OFF; RESET MASTER; SET GLOBAL wsrep_on=ON;" + SET_GTID_BINLOG_STATE="SET GLOBAL wsrep_on=OFF; SET @@global.gtid_binlog_state='$GTID_BINLOG_STATE'; SET GLOBAL wsrep_on=ON;" SQL_LOG_BIN_OFF="SET @@session.sql_log_bin=OFF;" fi fi diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 1bfdb83633a..1fecb6f09d9 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -1,4 +1,4 @@ -#!/bin/sh -ue +#!/bin/bash -ue # Copyright (C) 2010-2014 Codership Oy # @@ -88,7 +88,7 @@ check_pid_and_port() local is_listening_all="$(echo $port_info | \ grep "*:$rsync_port" 2>/dev/null)" local is_listening_addr="$(echo $port_info | \ - grep "$rsync_addr:$rsync_port" 2>/dev/null)" + grep -F "$rsync_addr:$rsync_port" 2>/dev/null)" if [ ! -z "$is_listening_all" -o ! -z "$is_listening_addr" ]; then if [ -z "$is_rsync" ]; then @@ -119,7 +119,7 @@ is_local_ip() address="$address " fi - $get_addr_bin | grep "$address" > /dev/null + $get_addr_bin | grep -F "$address" > /dev/null } STUNNEL_CONF="$WSREP_SST_OPT_DATA/stunnel.conf" @@ -139,6 +139,14 @@ if ! [ -z $WSREP_SST_OPT_BINLOG ] then BINLOG_DIRNAME=$(dirname $WSREP_SST_OPT_BINLOG) BINLOG_FILENAME=$(basename $WSREP_SST_OPT_BINLOG) + BINLOG_INDEX_DIRNAME=$(dirname $WSREP_SST_OPT_BINLOG) + BINLOG_INDEX_FILENAME=$(basename $WSREP_SST_OPT_BINLOG) +fi + +if ! [ -z $WSREP_SST_OPT_BINLOG_INDEX ] +then + BINLOG_INDEX_DIRNAME=$(dirname $WSREP_SST_OPT_BINLOG_INDEX) + BINLOG_INDEX_FILENAME=$(basename $WSREP_SST_OPT_BINLOG_INDEX) fi WSREP_LOG_DIR=${WSREP_LOG_DIR:-""} @@ -156,10 +164,17 @@ else fi INNODB_DATA_HOME_DIR=${INNODB_DATA_HOME_DIR:-""} +# Try to set INNODB_DATA_HOME_DIR from the command line: +if [ ! -z "$INNODB_DATA_HOME_DIR_ARG" ]; then + INNODB_DATA_HOME_DIR=$INNODB_DATA_HOME_DIR_ARG +fi # if INNODB_DATA_HOME_DIR env. variable is not set, try to get it from my.cnf if [ -z "$INNODB_DATA_HOME_DIR" ]; then INNODB_DATA_HOME_DIR=$(parse_cnf mysqld$WSREP_SST_OPT_SUFFIX_VALUE innodb-data-home-dir '') fi +if [ -z "$INNODB_DATA_HOME_DIR" ]; then + INNODB_DATA_HOME_DIR=$(parse_cnf --mysqld innodb-data-home-dir "") +fi if [ -n "$INNODB_DATA_HOME_DIR" ]; then # handle both relative and absolute paths @@ -176,8 +191,15 @@ fi # --exclude '*.[0-9][0-9][0-9][0-9][0-9][0-9]' --exclude '*.index') # New filter - exclude everything except dirs (schemas) and innodb files -FILTER="-f '- /lost+found' -f '- /.fseventsd' -f '- /.Trashes' - -f '+ /wsrep_sst_binlog.tar' -f '- $INNODB_DATA_HOME_DIR/ib_lru_dump' -f '- $INNODB_DATA_HOME_DIR/ibdata*' -f '+ /*/' -f '- /*'" +FILTER="-f '- /lost+found' + -f '- /.fseventsd' + -f '- /.Trashes' + -f '+ /wsrep_sst_binlog.tar' + -f '- $INNODB_DATA_HOME_DIR/ib_lru_dump' + -f '- $INNODB_DATA_HOME_DIR/ibdata*' + -f '+ /undo*' + -f '+ /*/' + -f '- /*'" SSTKEY=$(parse_cnf sst tkey "") SSTCERT=$(parse_cnf sst tcert "") @@ -246,12 +268,21 @@ EOF OLD_PWD="$(pwd)" cd $BINLOG_DIRNAME - binlog_files_full=$(tail -n $BINLOG_N_FILES ${BINLOG_FILENAME}.index) + if ! [ -z $WSREP_SST_OPT_BINLOG_INDEX ] + then + binlog_files_full=$(tail -n $BINLOG_N_FILES ${BINLOG_FILENAME}.index) + else + cd $BINLOG_INDEX_DIRNAME + binlog_files_full=$(tail -n $BINLOG_N_FILES ${BINLOG_INDEX_FILENAME}.index) + fi + + cd $BINLOG_DIRNAME binlog_files="" for ii in $binlog_files_full do binlog_files="$binlog_files $(basename $ii)" done + if ! [ -z "$binlog_files" ] then wsrep_log_info "Preparing binlog files for transfer:" @@ -368,12 +399,17 @@ then rm -rf "$RSYNC_PID" ADDR=$WSREP_SST_OPT_ADDR - RSYNC_PORT=$(echo $ADDR | awk -F ':' '{ print $2 }') - RSYNC_ADDR=$(echo $ADDR | awk -F ':' '{ print $1 }') + if [[ ${ADDR:0:1} == '[' ]]; then + RSYNC_PORT=$(echo $ADDR | awk -F '\\]:' '{ print $2 }') + RSYNC_ADDR=$(echo $ADDR | awk -F '\\]:' '{ print $1 }')"]" + else + RSYNC_PORT=$(echo $ADDR | awk -F ':' '{ print $2 }') + RSYNC_ADDR=$(echo $ADDR | awk -F ':' '{ print $1 }') + fi if [ -z "$RSYNC_PORT" ] then RSYNC_PORT=4444 - ADDR="$(echo $ADDR | awk -F ':' '{ print $1 }'):$RSYNC_PORT" + ADDR="$RSYNC_ADDR:$RSYNC_PORT" fi trap "exit 32" HUP PIPE @@ -477,7 +513,12 @@ EOF tar -xvf $BINLOG_TAR_FILE >&2 for ii in $(ls -1 ${BINLOG_FILENAME}.*) do - echo ${BINLOG_DIRNAME}/${ii} >> ${BINLOG_FILENAME}.index + if ! [ -z $WSREP_SST_OPT_BINLOG_INDEX ] + then + echo ${BINLOG_DIRNAME}/${ii} >> ${BINLOG_FILENAME}.index + else + echo ${BINLOG_DIRNAME}/${ii} >> ${BINLOG_INDEX_DIRNAME}/${BINLOG_INDEX_FILENAME}.index + fi done fi cd "$OLD_PWD" diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh deleted file mode 100644 index 11977249569..00000000000 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ /dev/null @@ -1,1236 +0,0 @@ -#!/bin/bash -ue -# Copyright (C) 2013 Percona Inc -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; see the file COPYING. If not, write to the -# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston -# MA 02110-1301 USA. - -# Documentation: http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html -# Make sure to read that before proceeding! - - - - -. $(dirname $0)/wsrep_sst_common - -ealgo="" -ekey="" -ekeyfile="" -encrypt=0 -nproc=1 -ecode=0 -ssyslog="" -ssystag="" -XTRABACKUP_PID="" -tca="" -tcert="" -tkey="" -sockopt="" -progress="" -ttime=0 -totime=0 -ecmd="" -rlimit="" -# Initially -stagemsg="${WSREP_SST_OPT_ROLE}" -cpat="" -ib_home_dir="" -ib_log_dir="" -ib_undo_dir="" - -sfmt="tar" -strmcmd="" -tfmt="" -tcmd="" -rebuild=0 -rebuildcmd="" -payload=0 -pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p' " -pvopts="-f -i 10 -N $WSREP_SST_OPT_ROLE " -STATDIR="" -uextra=0 -disver="" - -tmpopts="" -itmpdir="" -xtmpdir="" - -scomp="" -sdecomp="" -ssl_dhparams="" - -ssl_cert="" -ssl_ca="" -ssl_key="" - -if pv --help 2>/dev/null | grep -q FORMAT;then - pvopts+=$pvformat -fi -pcmd="pv $pvopts" -declare -a RC - -INNOBACKUPEX_BIN=innobackupex -DATA="${WSREP_SST_OPT_DATA}" -INFO_FILE="xtrabackup_galera_info" -IST_FILE="xtrabackup_ist" -MAGIC_FILE="${DATA}/${INFO_FILE}" - -# Setting the path for ss and ip -export PATH="/usr/sbin:/sbin:$PATH" - -OS=$(uname) - -if ! which lsof > /dev/null; then - wsrep_log_error "lsof tool not found in PATH! Make sure you have it installed." - exit 2 # ENOENT -fi - -timeit(){ - local stage=$1 - shift - local cmd="$@" - local x1 x2 took extcode - - if [[ $ttime -eq 1 ]];then - x1=$(date +%s) - wsrep_log_info "Evaluating $cmd" - eval "$cmd" - extcode=$? - x2=$(date +%s) - took=$(( x2-x1 )) - wsrep_log_info "NOTE: $stage took $took seconds" - totime=$(( totime+took )) - else - wsrep_log_info "Evaluating $cmd" - eval "$cmd" - extcode=$? - fi - return $extcode -} - -get_keys() -{ - # $encrypt -eq 1 is for internal purposes only - if [[ $encrypt -ge 2 || $encrypt -eq -1 ]];then - return - fi - - if [[ $encrypt -eq 0 ]];then - if $MY_PRINT_DEFAULTS xtrabackup | grep -q encrypt;then - wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html " - fi - return - fi - - if [[ $sfmt == 'tar' ]];then - wsrep_log_info "NOTE: Xtrabackup-based encryption - encrypt=1 - cannot be enabled with tar format" - encrypt=-1 - return - fi - - wsrep_log_info "Xtrabackup based encryption enabled in my.cnf - Supported only from Xtrabackup 2.1.4" - - if [[ -z $ealgo ]];then - wsrep_log_error "FATAL: Encryption algorithm empty from my.cnf, bailing out" - exit 3 - fi - - if [[ -z $ekey && ! -r $ekeyfile ]];then - wsrep_log_error "FATAL: Either key or keyfile must be readable" - exit 3 - fi - - if [[ -z $ekey ]];then - ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key-file=$ekeyfile" - else - wsrep_log_warning "Using the 'encrypt-key' option causes the encryption key" - wsrep_log_warning "to be set via the command-line and is considered insecure." - wsrep_log_warning "It is recommended to use the 'encrypt-key-file' option instead." - - ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key=$ekey" - fi - - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - ecmd+=" -d" - fi - - stagemsg+="-XB-Encrypted" -} - -# -# If the ssl_dhparams variable is already set, uses that as a source -# of dh parameters for OpenSSL. Otherwise, looks for dhparams.pem in the -# datadir, and creates it there if it can't find the file. -# No input parameters -# -check_for_dhparams() -{ - if [[ -z "$ssl_dhparams" ]]; then - if ! [[ -r "$DATA/dhparams.pem" ]]; then - wsrep_check_programs openssl - wsrep_log_info "Could not find dhparams file, creating $DATA/dhparams.pem" - - if ! openssl dhparam -out "$DATA/dhparams.pem" 2048 >/dev/null 2>&1 - then - wsrep_log_error "******** FATAL ERROR ********************************* " - wsrep_log_error "* Could not create the dhparams.pem file with OpenSSL. " - wsrep_log_error "****************************************************** " - exit 22 - fi - fi - ssl_dhparams="$DATA/dhparams.pem" - fi -} - -# -# verifies that the certificate matches the private key -# doing this will save us having to wait for a timeout that would -# otherwise occur. -# -# 1st param: path to the cert -# 2nd param: path to the private key -# -verify_cert_matches_key() -{ - local cert_path=$1 - local key_path=$2 - - wsrep_check_programs openssl diff - - # generate the public key from the cert and the key - # they should match (otherwise we can't create an SSL connection) - if ! diff <(openssl x509 -in "$cert_path" -pubkey -noout) <(openssl rsa -in "$key_path" -pubout 2>/dev/null) >/dev/null 2>&1 - then - wsrep_log_error "******** FATAL ERROR ************************* " - wsrep_log_error "* The certifcate and private key do not match. " - wsrep_log_error "* Please check your certificate and key files. " - wsrep_log_error "********************************************** " - exit 22 - fi -} - -# Checks to see if the file exists -# If the file does not exist (or cannot be read), issues an error -# and exits -# -# 1st param: file name to be checked (for read access) -# 2nd param: 1st error message (header) -# 3rd param: 2nd error message (footer, optional) -# -verify_file_exists() -{ - local file_path=$1 - local error_message1=$2 - local error_message2=$3 - - if ! [[ -r "$file_path" ]]; then - wsrep_log_error "******** FATAL ERROR ************************* " - wsrep_log_error "* $error_message1 " - wsrep_log_error "* Could not find/access : $file_path " - - if ! [[ -z "$error_message2" ]]; then - wsrep_log_error "* $error_message2 " - fi - - wsrep_log_error "********************************************** " - exit 22 - fi -} - -get_transfer() -{ - TSST_PORT=${WSREP_SST_OPT_PORT:-4444} - - if [[ $tfmt == 'nc' ]];then - wsrep_check_programs nc - - if [[ $encrypt -eq 2 || $encrypt -eq 3 || $encrypt -eq 4 ]]; then - wsrep_log_error "******** FATAL ERROR *********************** " - wsrep_log_error "* Using SSL encryption (encrypt= 2, 3, or 4) " - wsrep_log_error "* is not supported when using nc(netcat). " - wsrep_log_error "******************************************** " - exit 22 - fi - - wsrep_log_info "Using netcat as streamer" - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - if nc -h 2>&1 | grep -q ncat; then - # Ncat - tcmd="nc $sockopt -l ${TSST_PORT}" - elif nc -h 2>&1 | grep -q -- '-d\>';then - # Debian netcat - tcmd="nc $sockopt -dl ${TSST_PORT}" - else - # traditional netcat - tcmd="nc $sockopt -l -p ${TSST_PORT}" - fi - else - if nc -h 2>&1 | grep -q ncat;then - # Ncat - tcmd="nc ${WSREP_SST_OPT_HOST_UNESCAPED} ${TSST_PORT}" - elif nc -h 2>&1 | grep -q -- '-d\>';then - # Debian netcat - tcmd="nc ${WSREP_SST_OPT_HOST_UNESCAPED} ${TSST_PORT}" - else - # traditional netcat - tcmd="nc -q0 ${WSREP_SST_OPT_HOST_UNESCAPED} ${TSST_PORT}" - fi - fi - else - tfmt='socat' - wsrep_log_info "Using socat as streamer" - wsrep_check_programs socat - - donor_extra="" - joiner_extra="" - if [[ $encrypt -eq 2 || $encrypt -eq 3 || $encrypt -eq 4 ]]; then - if ! socat -V | grep -q WITH_OPENSSL; then - wsrep_log_error "******** FATAL ERROR ****************** " - wsrep_log_error "* socat is not openssl enabled. " - wsrep_log_error "* Unable to encrypt SST communications. " - wsrep_log_error "*************************************** " - exit 2 - fi - - # Determine the socat version - SOCAT_VERSION=`socat -V 2>&1 | grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1` - if [[ -z "$SOCAT_VERSION" ]]; then - wsrep_log_error "******** FATAL ERROR **************** " - wsrep_log_error "* Cannot determine the socat version. " - wsrep_log_error "************************************* " - exit 2 - fi - - # socat versions < 1.7.3 will have 512-bit dhparams (too small) - # so create 2048-bit dhparams and send that as a parameter - # socat version >= 1.7.3, checks to see if the peername matches the hostname - # set commonname="" to disable the peername checks - # - if ! check_for_version "$SOCAT_VERSION" "1.7.3"; then - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]]; then - # dhparams check (will create ssl_dhparams if needed) - check_for_dhparams - joiner_extra=",dhparam=$ssl_dhparams" - fi - fi - if check_for_version "$SOCAT_VERSION" "1.7.3"; then - donor_extra=',commonname=""' - fi - fi - - if [[ $encrypt -eq 2 ]]; then - wsrep_log_warning "**** WARNING **** encrypt=2 is deprecated and will be removed in a future release" - wsrep_log_info "Using openssl based encryption with socat: with crt and ca" - - verify_file_exists "$tcert" "Both certificate and CA files are required." \ - "Please check the 'tcert' option. " - verify_file_exists "$tca" "Both certificate and CA files are required." \ - "Please check the 'tca' option. " - - stagemsg+="-OpenSSL-Encrypted-2" - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - wsrep_log_info "Decrypting with CERT: $tcert, CA: $tca" - tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${tcert},cafile=${tca}${joiner_extra}${sockopt} stdio" - else - wsrep_log_info "Encrypting with CERT: $tcert, CA: $tca" - tcmd="socat -u stdio openssl-connect:${WSREP_SST_OPT_HOST}:${TSST_PORT},cert=${tcert},cafile=${tca}${donor_extra}${sockopt}" - fi - elif [[ $encrypt -eq 3 ]];then - wsrep_log_warning "**** WARNING **** encrypt=3 is deprecated and will be removed in a future release" - wsrep_log_info "Using openssl based encryption with socat: with key and crt" - - verify_file_exists "$tcert" "Both certificate and key files are required." \ - "Please check the 'tcert' option. " - verify_file_exists "$tkey" "Both certificate and key files are required." \ - "Please check the 'tkey' option. " - - stagemsg+="-OpenSSL-Encrypted-3" - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - wsrep_log_info "Decrypting with CERT: $tcert, KEY: $tkey" - tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${tcert},key=${tkey},verify=0${joiner_extra}${sockopt} stdio" - else - wsrep_log_info "Encrypting with CERT: $tcert, KEY: $tkey" - tcmd="socat -u stdio openssl-connect:${WSREP_SST_OPT_HOST}:${TSST_PORT},cert=${tcert},key=${tkey},verify=0${sockopt}" - fi - elif [[ $encrypt -eq 4 ]]; then - wsrep_log_info "Using openssl based encryption with socat: with key, crt, and ca" - - verify_file_exists "$ssl_ca" "CA, certificate, and key files are required." \ - "Please check the 'ssl-ca' option. " - verify_file_exists "$ssl_cert" "CA, certificate, and key files are required." \ - "Please check the 'ssl-cert' option. " - verify_file_exists "$ssl_key" "CA, certificate, and key files are required." \ - "Please check the 'ssl-key' option. " - - # Check to see that the key matches the cert - verify_cert_matches_key $ssl_cert $ssl_key - - stagemsg+="-OpenSSL-Encrypted-4" - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]]; then - wsrep_log_info "Decrypting with CERT: $ssl_cert, KEY: $ssl_key, CA: $ssl_ca" - tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${ssl_cert},key=${ssl_key},cafile=${ssl_ca},verify=1${joiner_extra}${sockopt} stdio" - else - wsrep_log_info "Encrypting with CERT: $ssl_cert, KEY: $ssl_key, CA: $ssl_ca" - tcmd="socat -u stdio openssl-connect:${WSREP_SST_OPT_HOST}:${TSST_PORT},cert=${ssl_cert},key=${ssl_key},cafile=${ssl_ca},verify=1${donor_extra}${sockopt}" - fi - - else - if [[ $encrypt -eq 1 ]]; then - wsrep_log_warning "**** WARNING **** encrypt=1 is deprecated and will be removed in a future release" - fi - - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]]; then - tcmd="socat -u TCP-LISTEN:${TSST_PORT},reuseaddr${sockopt} stdio" - else - tcmd="socat -u stdio TCP:${WSREP_SST_OPT_HOST}:${TSST_PORT}${sockopt}" - fi - fi - fi -} - -get_footprint() -{ - pushd $WSREP_SST_OPT_DATA 1>/dev/null - payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | du --files0-from=- --block-size=1 -c -s | awk 'END { print $1 }') - if $MY_PRINT_DEFAULTS xtrabackup | grep -q -- "--compress";then - # QuickLZ has around 50% compression ratio - # When compression/compaction used, the progress is only an approximate. - payload=$(( payload*1/2 )) - fi - popd 1>/dev/null - pcmd+=" -s $payload" - adjust_progress -} - -adjust_progress() -{ - - if ! command -v pv >/dev/null;then - wsrep_log_error "pv not found in path: $PATH" - wsrep_log_error "Disabling all progress/rate-limiting" - pcmd="" - rlimit="" - progress="" - return - fi - - if [[ -n $progress && $progress != '1' ]];then - if [[ -e $progress ]];then - pcmd+=" 2>>$progress" - else - pcmd+=" 2>$progress" - fi - elif [[ -z $progress && -n $rlimit ]];then - # When rlimit is non-zero - pcmd="pv -q" - fi - - if [[ -n $rlimit && "$WSREP_SST_OPT_ROLE" == "donor" ]];then - wsrep_log_info "Rate-limiting SST to $rlimit" - pcmd+=" -L \$rlimit" - fi -} - -read_cnf() -{ - sfmt=$(parse_cnf sst streamfmt "xbstream") - tfmt=$(parse_cnf sst transferfmt "socat") - tca=$(parse_cnf sst tca "") - tcert=$(parse_cnf sst tcert "") - tkey=$(parse_cnf sst tkey "") - encrypt=$(parse_cnf sst encrypt 0) - sockopt=$(parse_cnf sst sockopt "") - progress=$(parse_cnf sst progress "") - rebuild=$(parse_cnf sst rebuild 0) - ttime=$(parse_cnf sst time 0) - if [ "${OS}" = "FreeBSD" ]; then - cpat=$(parse_cnf sst cpat '.*\.pem$|.*init\.ok$|.*galera\.cache$|.*sst_in_progress$|.*\.sst$|.*gvwstate\.dat$|.*grastate\.dat$|.*\.err$|.*\.log$|.*RPM_UPGRADE_MARKER$|.*RPM_UPGRADE_HISTORY$') - else - cpat=$(parse_cnf sst cpat '.*\.pem$\|.*init\.ok$\|.*galera\.cache$\|.*sst_in_progress$\|.*\.sst$\|.*gvwstate\.dat$\|.*grastate\.dat$\|.*\.err$\|.*\.log$\|.*RPM_UPGRADE_MARKER$\|.*RPM_UPGRADE_HISTORY$') - fi - ealgo=$(parse_cnf xtrabackup encrypt "") - ekey=$(parse_cnf xtrabackup encrypt-key "") - ekeyfile=$(parse_cnf xtrabackup encrypt-key-file "") - scomp=$(parse_cnf sst compressor "") - sdecomp=$(parse_cnf sst decompressor "") - - - # Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html - if [[ -z $ealgo ]];then - ealgo=$(parse_cnf sst encrypt-algo "") - ekey=$(parse_cnf sst encrypt-key "") - ekeyfile=$(parse_cnf sst encrypt-key-file "") - fi - - # Pull the parameters needed for encrypt=4 - ssl_ca=$(parse_cnf sst ssl-ca "") - if [[ -z "$ssl_ca" ]]; then - ssl_ca=$(parse_cnf --mysqld ssl-ca "") - fi - ssl_cert=$(parse_cnf sst ssl-cert "") - if [[ -z "$ssl_cert" ]]; then - ssl_cert=$(parse_cnf --mysqld ssl-cert "") - fi - ssl_key=$(parse_cnf sst ssl-key "") - if [[ -z "$ssl_key" ]]; then - ssl_key=$(parse_cnf --mysqld ssl-key "") - fi - - rlimit=$(parse_cnf sst rlimit "") - uextra=$(parse_cnf sst use-extra 0) - iopts=$(parse_cnf sst inno-backup-opts "") - iapts=$(parse_cnf sst inno-apply-opts "") - impts=$(parse_cnf sst inno-move-opts "") - stimeout=$(parse_cnf sst sst-initial-timeout 100) - ssyslog=$(parse_cnf sst sst-syslog 0) - ssystag=$(parse_cnf mysqld_safe syslog-tag "${SST_SYSLOG_TAG:-}") - ssystag+="-" - - if [[ $ssyslog -ne -1 ]];then - if $MY_PRINT_DEFAULTS mysqld_safe | grep -q -- "--syslog";then - ssyslog=1 - fi - fi -} - -get_stream() -{ - if [[ $sfmt == 'xbstream' ]];then - wsrep_log_info "Streaming with xbstream" - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - strmcmd="xbstream -x" - else - strmcmd="xbstream -c \${INFO_FILE}" - fi - else - sfmt="tar" - wsrep_log_info "Streaming with tar" - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - strmcmd="tar xfi - " - else - strmcmd="tar cf - \${INFO_FILE} " - fi - - fi -} - -get_proc() -{ - set +e - nproc=$(grep -c processor /proc/cpuinfo) - [[ -z $nproc || $nproc -eq 0 ]] && nproc=1 - set -e -} - -sig_joiner_cleanup() -{ - wsrep_log_error "Removing $MAGIC_FILE file due to signal" - rm -f "$MAGIC_FILE" -} - -cleanup_joiner() -{ - # Since this is invoked just after exit NNN - local estatus=$? - if [[ $estatus -ne 0 ]];then - wsrep_log_error "Cleanup after exit with status:$estatus" - elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then - wsrep_log_info "Removing the sst_in_progress file" - wsrep_cleanup_progress_file - fi - if [[ -n $progress && -p $progress ]];then - wsrep_log_info "Cleaning up fifo file $progress" - rm $progress - fi - if [[ -n ${STATDIR:-} ]];then - [[ -d $STATDIR ]] && rm -rf $STATDIR - fi - - # Final cleanup - pgid=$(ps -o pgid= $$ | grep -o '[0-9]*') - - # This means no setsid done in mysqld. - # We don't want to kill mysqld here otherwise. - if [[ $$ -eq $pgid ]];then - - # This means a signal was delivered to the process. - # So, more cleanup. - if [[ $estatus -ge 128 ]];then - kill -KILL -$$ || true - fi - - fi - - exit $estatus -} - -check_pid() -{ - local pid_file="$1" - [ -r "$pid_file" ] && ps -p $(cat "$pid_file") >/dev/null 2>&1 -} - -cleanup_donor() -{ - # Since this is invoked just after exit NNN - local estatus=$? - if [[ $estatus -ne 0 ]];then - wsrep_log_error "Cleanup after exit with status:$estatus" - fi - - if [[ -n ${XTRABACKUP_PID:-} ]];then - if check_pid $XTRABACKUP_PID - then - wsrep_log_error "xtrabackup process is still running. Killing... " - kill_xtrabackup - fi - - fi - rm -f ${DATA}/${IST_FILE} || true - - if [[ -n $progress && -p $progress ]];then - wsrep_log_info "Cleaning up fifo file $progress" - rm -f $progress || true - fi - - wsrep_log_info "Cleaning up temporary directories" - - if [[ -n $xtmpdir ]];then - [[ -d $xtmpdir ]] && rm -rf $xtmpdir || true - fi - - if [[ -n $itmpdir ]];then - [[ -d $itmpdir ]] && rm -rf $itmpdir || true - fi - - # Final cleanup - pgid=$(ps -o pgid= $$ | grep -o '[0-9]*') - - # This means no setsid done in mysqld. - # We don't want to kill mysqld here otherwise. - if [[ $$ -eq $pgid ]];then - - # This means a signal was delivered to the process. - # So, more cleanup. - if [[ $estatus -ge 128 ]];then - kill -KILL -$$ || true - fi - - fi - - exit $estatus - -} - -kill_xtrabackup() -{ - local PID=$(cat $XTRABACKUP_PID) - [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || : - wsrep_log_info "Removing xtrabackup pid file $XTRABACKUP_PID" - rm -f "$XTRABACKUP_PID" || true -} - -# waits ~1 minute for nc/socat to open the port and then reports ready -# (regardless of timeout) -wait_for_listen() -{ - local HOST=$1 - local PORT=$2 - local MODULE=$3 - local LSOF_OUT - - for i in {1..300} - do - LSOF_OUT=$(lsof -sTCP:LISTEN -i TCP:${PORT} -a -c nc -c socat -F c 2> /dev/null || :) - [ -n "${LSOF_OUT}" ] && break - sleep 0.2 - done - - echo "ready ${HOST}:${PORT}/${MODULE}//${WSREP_SST_OPT_SST_VER:-1}" -} - -check_extra() -{ - local use_socket=1 - if [[ $uextra -eq 1 ]];then - if [ $(parse_cnf --mysqld thread-handling) = 'pool-of-threads'];then - local eport=$(parse_cnf --mysqld extra-port) - if [[ -n $eport ]];then - # Xtrabackup works only locally. - # Hence, setting host to 127.0.0.1 unconditionally. - wsrep_log_info "SST through extra_port $eport" - INNOEXTRA+=" --host=127.0.0.1 --port=$eport " - use_socket=0 - else - wsrep_log_error "Extra port $eport null, failing" - exit 1 - fi - else - wsrep_log_info "Thread pool not set, ignore the option use_extra" - fi - fi - if [[ $use_socket -eq 1 ]] && [[ -n "${WSREP_SST_OPT_SOCKET}" ]];then - INNOEXTRA+=" --socket=${WSREP_SST_OPT_SOCKET}" - fi -} - -recv_joiner() -{ - local dir=$1 - local msg=$2 - local tmt=$3 - local checkf=$4 - local ltcmd - - if [[ ! -d ${dir} ]];then - # This indicates that IST is in progress - return - fi - - pushd ${dir} 1>/dev/null - set +e - - if [[ $tmt -gt 0 ]] && command -v timeout >/dev/null;then - if timeout --help | grep -q -- '-k';then - ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" - else - ltcmd="timeout -s9 $tmt $tcmd" - fi - timeit "$msg" "$ltcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" - else - timeit "$msg" "$tcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" - fi - - set -e - popd 1>/dev/null - - if [[ ${RC[0]} -eq 124 ]];then - wsrep_log_error "Possible timeout in receving first data from donor in gtid stage" - exit 32 - fi - - for ecode in "${RC[@]}";do - if [[ $ecode -ne 0 ]];then - wsrep_log_error "Error while getting data from donor node: " \ - "exit codes: ${RC[@]}" - exit 32 - fi - done - - if [[ $checkf -eq 1 && ! -r "${MAGIC_FILE}" ]];then - # this message should cause joiner to abort - wsrep_log_error "xtrabackup process ended without creating '${MAGIC_FILE}'" - wsrep_log_info "Contents of datadir" - wsrep_log_info "$(ls -l ${dir}/*)" - exit 32 - fi -} - - -send_donor() -{ - local dir=$1 - local msg=$2 - - pushd ${dir} 1>/dev/null - set +e - timeit "$msg" "$strmcmd | $tcmd; RC=( "\${PIPESTATUS[@]}" )" - set -e - popd 1>/dev/null - - - for ecode in "${RC[@]}";do - if [[ $ecode -ne 0 ]];then - wsrep_log_error "Error while getting data from donor node: " \ - "exit codes: ${RC[@]}" - exit 32 - fi - done - -} - -# Returns the version string in a standardized format -# Input "1.2.3" => echoes "010203" -# Wrongly formatted values => echoes "000000" -normalize_version() -{ - local major=0 - local minor=0 - local patch=0 - - # Only parses purely numeric version numbers, 1.2.3 - # Everything after the first three values are ignored - if [[ $1 =~ ^([0-9]+)\.([0-9]+)\.?([0-9]*)([\.0-9])*$ ]]; then - major=${BASH_REMATCH[1]} - minor=${BASH_REMATCH[2]} - patch=${BASH_REMATCH[3]} - fi - - printf %02d%02d%02d $major $minor $patch -} - -# Compares two version strings -# The first parameter is the version to be checked -# The second parameter is the minimum version required -# Returns 1 (failure) if $1 >= $2, 0 (success) otherwise -check_for_version() -{ - local local_version_str="$( normalize_version $1 )" - local required_version_str="$( normalize_version $2 )" - - if [[ "$local_version_str" < "$required_version_str" ]]; then - return 1 - else - return 0 - fi -} - -monitor_process() -{ - local sst_stream_pid=$1 - - while true ; do - - if ! ps --pid "${WSREP_SST_OPT_PARENT}" &>/dev/null; then - wsrep_log_error "Parent mysqld process (PID:${WSREP_SST_OPT_PARENT}) terminated unexpectedly." - kill -- -"${WSREP_SST_OPT_PARENT}" - exit 32 - fi - - if ! ps --pid "${sst_stream_pid}" &>/dev/null; then - break - fi - - sleep 0.1 - - done -} - - -wsrep_check_programs "$INNOBACKUPEX_BIN" - -# check the version, we require XB-2.4 to ensure that we can pass the -# datadir via the command-line option -XB_REQUIRED_VERSION="2.3.5" - -XB_VERSION=`$INNOBACKUPEX_BIN --version 2>&1 | grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1` -if [[ -z $XB_VERSION ]]; then - wsrep_log_error "FATAL: Cannot determine the $INNOBACKUPEX_BIN version. Needs xtrabackup-$XB_REQUIRED_VERSION or higher to perform SST" - exit 2 -fi - -if ! check_for_version $XB_VERSION $XB_REQUIRED_VERSION; then - wsrep_log_error "FATAL: The $INNOBACKUPEX_BIN version is $XB_VERSION. Needs xtrabackup-$XB_REQUIRED_VERSION or higher to perform SST" - exit 2 -fi - - -rm -f "${MAGIC_FILE}" - -if [[ ! ${WSREP_SST_OPT_ROLE} == 'joiner' && ! ${WSREP_SST_OPT_ROLE} == 'donor' ]];then - wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}" - exit 22 -fi - -read_cnf - -if ${INNOBACKUPEX_BIN} /tmp --help 2>/dev/null | grep -q -- '--version-check'; then - disver="--no-version-check" -fi - -if [[ ${FORCE_FTWRL:-0} -eq 1 ]];then - wsrep_log_info "Forcing FTWRL due to environment variable FORCE_FTWRL equal to $FORCE_FTWRL" - iopts+=" --no-backup-locks " -fi - - -INNOEXTRA="" - -if [[ $ssyslog -eq 1 ]];then - - if ! command -v logger >/dev/null;then - wsrep_log_error "logger not in path: $PATH. Ignoring" - else - - wsrep_log_info "Logging all stderr of SST/Innobackupex to syslog" - - exec 2> >(logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE) - - wsrep_log_error() - { - logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" - } - - wsrep_log_info() - { - logger -p daemon.info -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" - } - - INNOAPPLY="2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-apply " - INNOMOVE="2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move " - INNOBACKUP="2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)" - fi - -else - INNOAPPLY="&>\${DATA}/innobackup.prepare.log" - INNOMOVE="&>\${DATA}/innobackup.move.log" - INNOBACKUP="2>\${DATA}/innobackup.backup.log" -fi - -get_stream -get_transfer - -INNOAPPLY="${INNOBACKUPEX_BIN} $disver $iapts --apply-log \$rebuildcmd \${DATA} ${INNOAPPLY}" -INNOMOVE="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_CONF} $disver $impts --move-back --force-non-empty-directories \${DATA} ${INNOMOVE}" -INNOBACKUP="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir ${INNOBACKUP}" - -if [ "$WSREP_SST_OPT_ROLE" = "donor" ] -then - trap cleanup_donor EXIT - - if [ $WSREP_SST_OPT_BYPASS -eq 0 ] - then - usrst=0 - if [[ -z $WSREP_SST_OPT_SST_VER ]];then - wsrep_log_error "Upgrade joiner to 5.6.21 or higher for backup locks support" - wsrep_log_error "The joiner is not supported for this version of donor" - exit 93 - fi - - if [[ -z $(parse_cnf --mysqld tmpdir "") && -z $(parse_cnf xtrabackup tmpdir "") ]];then - xtmpdir=$(mktemp -d) - tmpopts=" --tmpdir=$xtmpdir " - wsrep_log_info "Using $xtmpdir as xtrabackup temporary directory" - fi - - itmpdir=$(mktemp -d) - wsrep_log_info "Using $itmpdir as innobackupex temporary directory" - - if [[ -n "${WSREP_SST_OPT_USER:-}" && "$WSREP_SST_OPT_USER" != "(null)" ]]; then - INNOEXTRA+=" --user=$WSREP_SST_OPT_USER" - usrst=1 - fi - - if [ -n "${WSREP_SST_OPT_PSWD:-}" ]; then - INNOEXTRA+=" --password=$WSREP_SST_OPT_PSWD" - elif [[ $usrst -eq 1 ]];then - # Empty password, used for testing, debugging etc. - INNOEXTRA+=" --password=" - fi - - get_keys - check_extra - - wsrep_log_info "Streaming GTID file before SST" - - # Store donor's wsrep GTID (state ID) and wsrep_gtid_domain_id - # (separated by a space). - echo "${WSREP_SST_OPT_GTID} ${WSREP_SST_OPT_GTID_DOMAIN_ID}" > "${MAGIC_FILE}" - - ttcmd="$tcmd" - - if [[ $encrypt -eq 1 ]];then - if [[ -n $scomp ]];then - tcmd=" \$ecmd | $scomp | $tcmd " - else - tcmd=" \$ecmd | $tcmd " - fi - elif [[ -n $scomp ]];then - tcmd=" $scomp | $tcmd " - fi - - send_donor $DATA "${stagemsg}-gtid" - - # Restore the transport commmand to its original state - tcmd="$ttcmd" - if [[ -n $progress ]];then - get_footprint - tcmd="$pcmd | $tcmd" - elif [[ -n $rlimit ]];then - adjust_progress - tcmd="$pcmd | $tcmd" - fi - - wsrep_log_info "Sleeping before data transfer for SST" - sleep 10 - - wsrep_log_info "Streaming the backup to joiner at ${WSREP_SST_OPT_HOST} ${WSREP_SST_OPT_PORT:-4444}" - - # Add compression to the head of the stream (if specified) - if [[ -n $scomp ]]; then - tcmd="$scomp | $tcmd" - fi - - # Add encryption to the head of the stream (if specified) - if [[ $encrypt -eq 1 ]]; then - tcmd=" \$ecmd | $tcmd " - fi - - set +e - timeit "${stagemsg}-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )" - set -e - - if [ ${RC[0]} -ne 0 ]; then - wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \ - "Check ${DATA}/innobackup.backup.log" - exit 22 - elif [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then - wsrep_log_error "$tcmd finished with error: ${RC[1]}" - exit 22 - fi - - # innobackupex implicitly writes PID to fixed location in $xtmpdir - XTRABACKUP_PID="$xtmpdir/xtrabackup_pid" - - - else # BYPASS FOR IST - - wsrep_log_info "Bypassing the SST for IST" - echo "continue" # now server can resume updating data - - # Store donor's wsrep GTID (state ID) and wsrep_gtid_domain_id - # (separated by a space). - echo "${WSREP_SST_OPT_GTID} ${WSREP_SST_OPT_GTID_DOMAIN_ID}" > "${MAGIC_FILE}" - echo "1" > "${DATA}/${IST_FILE}" - get_keys - if [[ $encrypt -eq 1 ]];then - if [[ -n $scomp ]];then - tcmd=" \$ecmd | $scomp | $tcmd " - else - tcmd=" \$ecmd | $tcmd " - fi - elif [[ -n $scomp ]];then - tcmd=" $scomp | $tcmd " - fi - strmcmd+=" \${IST_FILE}" - - send_donor $DATA "${stagemsg}-IST" - - fi - - echo "done ${WSREP_SST_OPT_GTID}" - wsrep_log_info "Total time on donor: $totime seconds" - -elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ] -then - [[ -e $SST_PROGRESS_FILE ]] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" - [[ -n $SST_PROGRESS_FILE ]] && touch $SST_PROGRESS_FILE - - ib_home_dir=$(parse_cnf --mysqld innodb-data-home-dir "") - ib_log_dir=$(parse_cnf --mysqld innodb-log-group-home-dir "") - ib_undo_dir=$(parse_cnf --mysqld innodb-undo-directory "") - - stagemsg="Joiner-Recv" - - sencrypted=1 - nthreads=1 - - MODULE="xtrabackup_sst" - - rm -f "${DATA}/${IST_FILE}" - - # May need xtrabackup_checkpoints later on - rm -f ${DATA}/xtrabackup_binary ${DATA}/xtrabackup_galera_info ${DATA}/xtrabackup_logfile - - wait_for_listen ${WSREP_SST_OPT_HOST} ${WSREP_SST_OPT_PORT:-4444} ${MODULE} & - - trap sig_joiner_cleanup HUP PIPE INT TERM - trap cleanup_joiner EXIT - - if [[ -n $progress ]];then - adjust_progress - tcmd+=" | $pcmd" - fi - - get_keys - if [[ $encrypt -eq 1 && $sencrypted -eq 1 ]];then - if [[ -n $sdecomp ]];then - strmcmd=" $sdecomp | \$ecmd | $strmcmd" - else - strmcmd=" \$ecmd | $strmcmd" - fi - elif [[ -n $sdecomp ]];then - strmcmd=" $sdecomp | $strmcmd" - fi - - STATDIR=$(mktemp -d) - MAGIC_FILE="${STATDIR}/${INFO_FILE}" - recv_joiner $STATDIR "${stagemsg}-gtid" $stimeout 1 - - - if ! ps -p ${WSREP_SST_OPT_PARENT} &>/dev/null - then - wsrep_log_error "Parent mysqld process (PID:${WSREP_SST_OPT_PARENT}) terminated unexpectedly." - exit 32 - fi - - if [ ! -r "${STATDIR}/${IST_FILE}" ] - then - - if [[ -d ${DATA}/.sst ]];then - wsrep_log_info "WARNING: Stale temporary SST directory: ${DATA}/.sst from previous state transfer. Removing" - rm -rf ${DATA}/.sst - fi - mkdir -p ${DATA}/.sst - (recv_joiner $DATA/.sst "${stagemsg}-SST" 0 0) & - jpid=$! - wsrep_log_info "Proceeding with SST" - - - wsrep_log_info "Cleaning the existing datadir and innodb-data/log directories" - if [ "${OS}" = "FreeBSD" ]; then - find -E $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -prune -regex $cpat -o -exec rm -rfv {} 1>&2 \+ - else - find $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -prune -regex $cpat -o -exec rm -rfv {} 1>&2 \+ - fi - - tempdir=$(parse_cnf --mysqld log-bin "") - if [[ -n ${tempdir:-} ]];then - binlog_dir=$(dirname $tempdir) - binlog_file=$(basename $tempdir) - if [[ -n ${binlog_dir:-} && $binlog_dir != '.' && $binlog_dir != $DATA ]];then - pattern="$binlog_dir/$binlog_file\.[0-9]+$" - wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" - find $binlog_dir -maxdepth 1 -type f -regex $pattern -exec rm -fv {} 1>&2 \+ || true - rm $binlog_dir/*.index || true - fi - fi - - - - TDATA=${DATA} - DATA="${DATA}/.sst" - - - MAGIC_FILE="${DATA}/${INFO_FILE}" - wsrep_log_info "Waiting for SST streaming to complete!" - monitor_process $jpid - - get_proc - - if [[ ! -s ${DATA}/xtrabackup_checkpoints ]];then - wsrep_log_error "xtrabackup_checkpoints missing, failed innobackupex/SST on donor" - exit 2 - fi - - # Rebuild indexes for compact backups - if grep -q 'compact = 1' ${DATA}/xtrabackup_checkpoints;then - wsrep_log_info "Index compaction detected" - rebuild=1 - fi - - if [[ $rebuild -eq 1 ]];then - nthreads=$(parse_cnf xtrabackup rebuild-threads $nproc) - wsrep_log_info "Rebuilding during prepare with $nthreads threads" - rebuildcmd="--rebuild-indexes --rebuild-threads=$nthreads" - fi - - if test -n "$(find ${DATA} -maxdepth 1 -type f -name '*.qp' -print -quit)";then - - wsrep_log_info "Compressed qpress files found" - - if ! command -v qpress >/dev/null;then - wsrep_log_error "qpress not found in path: $PATH" - exit 22 - fi - - if [[ -n $progress ]] && pv --help | grep -q 'line-mode';then - count=$(find ${DATA} -type f -name '*.qp' | wc -l) - count=$(( count*2 )) - if pv --help | grep -q FORMAT;then - pvopts="-f -s $count -l -N Decompression -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" - else - pvopts="-f -s $count -l -N Decompression" - fi - pcmd="pv $pvopts" - adjust_progress - dcmd="$pcmd | xargs -n 2 qpress -T${nproc}d" - else - dcmd="xargs -n 2 qpress -T${nproc}d" - fi - - - # Decompress the qpress files - wsrep_log_info "Decompression with $nproc threads" - timeit "Joiner-Decompression" "find ${DATA} -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" - extcode=$? - - if [[ $extcode -eq 0 ]];then - wsrep_log_info "Removing qpress files after decompression" - find ${DATA} -type f -name '*.qp' -delete - if [[ $? -ne 0 ]];then - wsrep_log_error "Something went wrong with deletion of qpress files. Investigate" - fi - else - wsrep_log_error "Decompression failed. Exit code: $extcode" - exit 22 - fi - fi - - - if [[ ! -z $WSREP_SST_OPT_BINLOG ]];then - - BINLOG_DIRNAME=$(dirname $WSREP_SST_OPT_BINLOG) - BINLOG_FILENAME=$(basename $WSREP_SST_OPT_BINLOG) - - # To avoid comparing data directory and BINLOG_DIRNAME - mv $DATA/${BINLOG_FILENAME}.* $BINLOG_DIRNAME/ 2>/dev/null || true - - pushd $BINLOG_DIRNAME &>/dev/null - for bfiles in $(ls -1 ${BINLOG_FILENAME}.[0-9]*);do - echo ${BINLOG_DIRNAME}/${bfiles} >> ${BINLOG_FILENAME}.index - done - popd &> /dev/null - - fi - - wsrep_log_info "Preparing the backup at ${DATA}" - timeit "Xtrabackup prepare stage" "$INNOAPPLY" - - if [ $? -ne 0 ]; - then - wsrep_log_error "${INNOBACKUPEX_BIN} apply finished with errors. Check ${DATA}/innobackup.prepare.log" - exit 22 - fi - - MAGIC_FILE="${TDATA}/${INFO_FILE}" - set +e - rm $TDATA/innobackup.prepare.log $TDATA/innobackup.move.log - set -e - wsrep_log_info "Moving the backup to ${TDATA}" - timeit "Xtrabackup move stage" "$INNOMOVE" - if [[ $? -eq 0 ]];then - wsrep_log_info "Move successful, removing ${DATA}" - rm -rf $DATA - DATA=${TDATA} - else - wsrep_log_error "Move failed, keeping ${DATA} for further diagnosis" - wsrep_log_error "Check ${DATA}/innobackup.move.log for details" - exit 22 - fi - - - else - wsrep_log_info "${IST_FILE} received from donor: Running IST" - fi - - if [[ ! -r ${MAGIC_FILE} ]];then - wsrep_log_error "SST magic file ${MAGIC_FILE} not found/readable" - exit 2 - fi - wsrep_log_info "Galera co-ords from recovery: $(cat ${MAGIC_FILE})" - cat "${MAGIC_FILE}" # Output : UUID:seqno wsrep_gtid_domain_id - wsrep_log_info "Total time on joiner: $totime seconds" -fi - -exit 0 diff --git a/scripts/wsrep_sst_xtrabackup.sh b/scripts/wsrep_sst_xtrabackup.sh deleted file mode 100644 index 41ed4485de5..00000000000 --- a/scripts/wsrep_sst_xtrabackup.sh +++ /dev/null @@ -1,692 +0,0 @@ -#!/bin/bash -ue -# Copyright (C) 2013 Percona Inc -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; see the file COPYING. If not, write to the -# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston -# MA 02110-1301 USA. - -# Optional dependencies and options documented here: http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html -# Make sure to read that before proceeding! - - - - -. $(dirname $0)/wsrep_sst_common - -ealgo="" -ekey="" -ekeyfile="" -encrypt=0 -nproc=1 -ecode=0 -XTRABACKUP_PID="" -tcert="" -tpem="" -sockopt="" -progress="" -ttime=0 -totime=0 -lsn="${WSREP_SST_OPT_LSN}" -incremental=0 -ecmd="" -rlimit="" - -sfmt="tar" -strmcmd="" -tfmt="" -tcmd="" -rebuild=0 -rebuildcmd="" -payload=0 -pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p' " -pvopts="-f -i 10 -N $WSREP_SST_OPT_ROLE " -uextra=0 - -if pv --help 2>/dev/null | grep -q FORMAT;then - pvopts+=$pvformat -fi -pcmd="pv $pvopts" -declare -a RC - -INNOBACKUPEX_BIN=innobackupex -DATA="${WSREP_SST_OPT_DATA}" -INFO_FILE="xtrabackup_galera_info" -IST_FILE="xtrabackup_ist" -MAGIC_FILE="${DATA}/${INFO_FILE}" - -# Setting the path for ss and ip -export PATH="/usr/sbin:/sbin:$PATH" - -timeit(){ - local stage=$1 - shift - local cmd="$@" - local x1 x2 took extcode - - if [[ $ttime -eq 1 ]];then - x1=$(date +%s) - wsrep_log_info "Evaluating $cmd" - eval "$cmd" - extcode=$? - x2=$(date +%s) - took=$(( x2-x1 )) - wsrep_log_info "NOTE: $stage took $took seconds" - totime=$(( totime+took )) - else - wsrep_log_info "Evaluating $cmd" - eval "$cmd" - extcode=$? - fi - return $extcode -} - -get_keys() -{ - if [[ $encrypt -eq 2 ]];then - return - fi - - if [[ $encrypt -eq 0 ]];then - if $MY_PRINT_DEFAULTS xtrabackup | grep -q encrypt;then - wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html " - fi - return - fi - - if [[ $sfmt == 'tar' ]];then - wsrep_log_info "NOTE: Xtrabackup-based encryption - encrypt=1 - cannot be enabled with tar format" - encrypt=0 - return - fi - - wsrep_log_info "Xtrabackup based encryption enabled in my.cnf - Supported only from Xtrabackup 2.1.4" - - if [[ -z $ealgo ]];then - wsrep_log_error "FATAL: Encryption algorithm empty from my.cnf, bailing out" - exit 3 - fi - - if [[ -z $ekey && ! -r $ekeyfile ]];then - wsrep_log_error "FATAL: Either key or keyfile must be readable" - exit 3 - fi - - if [[ -z $ekey ]];then - ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key-file=$ekeyfile" - else - ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key=$ekey" - fi - - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - ecmd+=" -d" - fi -} - -get_transfer() -{ - TSST_PORT=${WSREP_SST_OPT_PORT:-4444} - - if [[ $tfmt == 'nc' ]];then - wsrep_check_programs nc - wsrep_log_info "Using netcat as streamer" - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - if nc -h 2>&1 | grep -q ncat;then - # Ncat - tcmd="nc -l ${TSST_PORT}" - elif nc -h 2>&1 | grep -q -- '-d\>';then - # Debian netcat - tcmd="nc -dl ${TSST_PORT}" - else - # traditional netcat - tcmd="nc -l -p ${TSST_PORT}" - fi - else - if nc -h 2>&1 | grep -q ncat;then - # Ncat - tcmd="nc ${REMOTEIP} ${TSST_PORT}" - elif nc -h 2>&1 | grep -q -- '-d\>';then - # Debian netcat - tcmd="nc ${REMOTEIP} ${TSST_PORT}" - else - # traditional netcat - tcmd="nc -q0 ${REMOTEIP} ${TSST_PORT}" - fi - fi - else - tfmt='socat' - wsrep_check_programs socat - wsrep_log_info "Using socat as streamer" - - if [[ $encrypt -eq 2 ]] && ! socat -V | grep -q OPENSSL;then - wsrep_log_info "NOTE: socat is not openssl enabled, falling back to plain transfer" - encrypt=0 - fi - - if [[ $encrypt -eq 2 ]];then - wsrep_log_info "Using openssl based encryption with socat" - if [[ -z $tpem || -z $tcert ]];then - wsrep_log_error "Both PEM and CRT files required" - exit 22 - fi - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - wsrep_log_info "Decrypting with PEM $tpem, CA: $tcert" - tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=$tpem,cafile=${tcert}${sockopt} stdio" - else - wsrep_log_info "Encrypting with PEM $tpem, CA: $tcert" - tcmd="socat -u stdio openssl-connect:${WSREP_SST_OPT_HOST}:${TSST_PORT},cert=$tpem,cafile=${tcert}${sockopt}" - fi - else - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - tcmd="socat -u TCP-LISTEN:${TSST_PORT},reuseaddr${sockopt} stdio" - else - tcmd="socat -u stdio TCP:${WSREP_SST_OPT_HOST}:${TSST_PORT}${sockopt}" - fi - fi - fi - -} - -get_footprint() -{ - pushd $WSREP_SST_OPT_DATA 1>/dev/null - payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | du --files0-from=- --block-size=1 -c -s | awk 'END { print $1 }') - if $MY_PRINT_DEFAULTS xtrabackup | grep -q -- "--compress";then - # QuickLZ has around 50% compression ratio - # When compression/compaction used, the progress is only an approximate. - payload=$(( payload*1/2 )) - fi - popd 1>/dev/null - pcmd+=" -s $payload" - adjust_progress -} - -adjust_progress() -{ - if [[ -n $progress && $progress != '1' ]];then - if [[ -e $progress ]];then - pcmd+=" 2>>$progress" - else - pcmd+=" 2>$progress" - fi - elif [[ -z $progress && -n $rlimit ]];then - # When rlimit is non-zero - pcmd="pv -q" - fi - - if [[ -n $rlimit && "$WSREP_SST_OPT_ROLE" == "donor" ]];then - wsrep_log_info "Rate-limiting SST to $rlimit" - pcmd+=" -L \$rlimit" - fi -} - -read_cnf() -{ - sfmt=$(parse_cnf sst streamfmt "tar") - tfmt=$(parse_cnf sst transferfmt "socat") - tcert=$(parse_cnf sst tca "") - tpem=$(parse_cnf sst tcert "") - encrypt=$(parse_cnf sst encrypt 0) - sockopt=$(parse_cnf sst sockopt "") - progress=$(parse_cnf sst progress "") - rebuild=$(parse_cnf sst rebuild 0) - ttime=$(parse_cnf sst time 0) - incremental=$(parse_cnf sst incremental 0) - ealgo=$(parse_cnf xtrabackup encrypt "") - ekey=$(parse_cnf xtrabackup encrypt-key "") - ekeyfile=$(parse_cnf xtrabackup encrypt-key-file "") - - # Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html - if [[ -z $ealgo ]];then - ealgo=$(parse_cnf sst encrypt-algo "") - ekey=$(parse_cnf sst encrypt-key "") - ekeyfile=$(parse_cnf sst encrypt-key-file "") - fi - rlimit=$(parse_cnf sst rlimit "") - uextra=$(parse_cnf sst use_extra 0) -} - -get_stream() -{ - if [[ $sfmt == 'xbstream' ]];then - wsrep_log_info "Streaming with xbstream" - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - strmcmd="xbstream -x" - else - strmcmd="xbstream -c \${INFO_FILE} \${IST_FILE}" - fi - else - sfmt="tar" - wsrep_log_info "Streaming with tar" - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - strmcmd="tar xfi - --recursive-unlink -h" - else - strmcmd="tar cf - \${INFO_FILE} \${IST_FILE}" - fi - - fi -} - -get_proc() -{ - set +e - nproc=$(grep -c processor /proc/cpuinfo) - [[ -z $nproc || $nproc -eq 0 ]] && nproc=1 - set -e -} - -sig_joiner_cleanup() -{ - wsrep_log_error "Removing $MAGIC_FILE file due to signal" - rm -f "$MAGIC_FILE" -} - -cleanup_joiner() -{ - # Since this is invoked just after exit NNN - local estatus=$? - if [[ $estatus -ne 0 ]];then - wsrep_log_error "Cleanup after exit with status:$estatus" - fi - if [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then - wsrep_log_info "Removing the sst_in_progress file" - wsrep_cleanup_progress_file - fi - if [[ -n $progress && -p $progress ]];then - wsrep_log_info "Cleaning up fifo file $progress" - rm $progress - fi -} - -check_pid() -{ - local pid_file="$1" - [ -r "$pid_file" ] && ps -p $(cat "$pid_file") >/dev/null 2>&1 -} - -cleanup_donor() -{ - # Since this is invoked just after exit NNN - local estatus=$? - if [[ $estatus -ne 0 ]];then - wsrep_log_error "Cleanup after exit with status:$estatus" - fi - - if [[ -n $XTRABACKUP_PID ]];then - if check_pid $XTRABACKUP_PID - then - wsrep_log_error "xtrabackup process is still running. Killing... " - kill_xtrabackup - fi - - rm -f $XTRABACKUP_PID - fi - rm -f ${DATA}/${IST_FILE} - - if [[ -n $progress && -p $progress ]];then - wsrep_log_info "Cleaning up fifo file $progress" - rm $progress - fi -} - -kill_xtrabackup() -{ - local PID=$(cat $XTRABACKUP_PID) - [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || : - rm -f "$XTRABACKUP_PID" -} - -# waits ~10 seconds for nc to open the port and then reports ready -# (regardless of timeout) -wait_for_listen() -{ - local PORT=$1 - local ADDR=$2 - local MODULE=$3 - for i in {1..50} - do - ss -p state listening "( sport = :$PORT )" | grep -qE 'socat|nc' && break - sleep 0.2 - done - if [[ $incremental -eq 1 ]];then - echo "ready ${ADDR}/${MODULE}/$lsn" - else - echo "ready ${ADDR}/${MODULE}" - fi -} - -check_extra() -{ - local use_socket=1 - if [[ $uextra -eq 1 ]];then - if [ $(parse_cnf --mysqld thread-handling) = 'pool-of-threads'];then - local eport=$(parse_cnf --mysqld extra-port) - if [[ -n $eport ]];then - # Xtrabackup works only locally. - # Hence, setting host to 127.0.0.1 unconditionally. - wsrep_log_info "SST through extra_port $eport" - INNOEXTRA+=" --host=127.0.0.1 --port=$eport " - use_socket=0 - else - wsrep_log_error "Extra port $eport null, failing" - exit 1 - fi - else - wsrep_log_info "Thread pool not set, ignore the option use_extra" - fi - fi - if [[ $use_socket -eq 1 ]] && [[ -n "${WSREP_SST_OPT_SOCKET}" ]];then - INNOEXTRA+=" --socket=${WSREP_SST_OPT_SOCKET}" - fi -} - -wsrep_check_programs "innobackupex" - -rm -f "${MAGIC_FILE}" - -if [[ ! ${WSREP_SST_OPT_ROLE} == 'joiner' && ! ${WSREP_SST_OPT_ROLE} == 'donor' ]];then - wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}" - exit 22 -fi - -read_cnf -get_stream -get_transfer - -INNOEXTRA="" -INNOAPPLY="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_CONF} --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log" -INNOBACKUP="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_CONF} \$INNOEXTRA --galera-info --stream=\$sfmt \${TMPDIR} 2>\${DATA}/innobackup.backup.log" - -if [ "$WSREP_SST_OPT_ROLE" = "donor" ] -then - trap cleanup_donor EXIT - - if [ $WSREP_SST_OPT_BYPASS -eq 0 ] - then - usrst=0 - TMPDIR="${TMPDIR:-/tmp}" - - if [[ -n "${WSREP_SST_OPT_USER:-}" && "$WSREP_SST_OPT_USER" != "(null)" ]]; then - INNOEXTRA+=" --user=$WSREP_SST_OPT_USER" - usrst=1 - fi - - if [ -n "${WSREP_SST_OPT_PSWD:-}" ]; then - INNOEXTRA+=" --password=$WSREP_SST_OPT_PSWD" - elif [[ $usrst -eq 1 ]];then - # Empty password, used for testing, debugging etc. - INNOEXTRA+=" --password=" - fi - - get_keys - if [[ $encrypt -eq 1 ]];then - if [[ -n $ekey ]];then - INNOEXTRA+=" --encrypt=$ealgo --encrypt-key=$ekey " - else - INNOEXTRA+=" --encrypt=$ealgo --encrypt-key-file=$ekeyfile " - fi - fi - - if [[ -n $lsn ]];then - INNOEXTRA+=" --incremental --incremental-lsn=$lsn " - fi - - check_extra - - wsrep_log_info "Streaming the backup to joiner at ${WSREP_SST_OPT_HOST} ${WSREP_SST_OPT_PORT}" - - if [[ -n $progress ]];then - get_footprint - tcmd="$pcmd | $tcmd" - elif [[ -n $rlimit ]];then - adjust_progress - tcmd="$pcmd | $tcmd" - fi - - set +e - timeit "Donor-Transfer" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )" - set -e - - if [ ${RC[0]} -ne 0 ]; then - wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \ - "Check ${DATA}/innobackup.backup.log" - exit 22 - elif [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then - wsrep_log_error "$tcmd finished with error: ${RC[1]}" - exit 22 - fi - - # innobackupex implicitly writes PID to fixed location in ${TMPDIR} - XTRABACKUP_PID="${TMPDIR}/xtrabackup_pid" - - else # BYPASS FOR IST - - wsrep_log_info "Bypassing the SST for IST" - echo "continue" # now server can resume updating data - - # Store donor's wsrep GTID (state ID) and wsrep_gtid_domain_id - # (separated by a space) - echo "${WSREP_SST_OPT_GTID} ${WSREP_SST_OPT_GTID_DOMAIN_ID}" > "${MAGIC_FILE}" - echo "1" > "${DATA}/${IST_FILE}" - get_keys - pushd ${DATA} 1>/dev/null - set +e - if [[ $encrypt -eq 1 ]];then - tcmd=" $ecmd | $tcmd" - fi - timeit "Donor-IST-Unencrypted-transfer" "$strmcmd | $tcmd; RC=( "\${PIPESTATUS[@]}" )" - set -e - popd 1>/dev/null - - for ecode in "${RC[@]}";do - if [[ $ecode -ne 0 ]];then - wsrep_log_error "Error while streaming data to joiner node: " \ - "exit codes: ${RC[@]}" - exit 1 - fi - done - fi - - echo "done ${WSREP_SST_OPT_GTID}" - wsrep_log_info "Total time on donor: $totime seconds" - -elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ] -then - [[ -e $SST_PROGRESS_FILE ]] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" - touch $SST_PROGRESS_FILE - - if [[ ! -e ${DATA}/ibdata1 ]];then - incremental=0 - fi - - if [[ $incremental -eq 1 ]];then - wsrep_log_info "Incremental SST enabled" - #lsn=$(/pxc/bin/mysqld $WSREP_SST_OPT_CONF --basedir=/pxc --wsrep-recover 2>&1 | grep -o 'log sequence number .*' | cut -d " " -f 4 | head -1) - lsn=$(grep to_lsn xtrabackup_checkpoints | cut -d= -f2 | tr -d ' ') - wsrep_log_info "Recovered LSN: $lsn" - fi - - sencrypted=1 - nthreads=1 - - MODULE="xtrabackup_sst" - - # May need xtrabackup_checkpoints later on - rm -f ${DATA}/xtrabackup_binary ${DATA}/xtrabackup_galera_info ${DATA}/xtrabackup_logfile - - ADDR="${WSREP_SST_OPT_HOST}:${WSREP_SST_OPT_PORT:-4444}" - - wait_for_listen ${WSREP_SST_OPT_PORT:-4444} ${ADDR} ${MODULE} & - - trap sig_joiner_cleanup HUP PIPE INT TERM - trap cleanup_joiner EXIT - - if [[ -n $progress ]];then - adjust_progress - tcmd+=" | $pcmd" - fi - - if [[ $incremental -eq 1 ]];then - BDATA=$DATA - DATA=$(mktemp -d) - MAGIC_FILE="${DATA}/${INFO_FILE}" - fi - - get_keys - set +e - if [[ $encrypt -eq 1 && $sencrypted -eq 1 ]];then - strmcmd=" $ecmd | $strmcmd" - fi - - pushd ${DATA} 1>/dev/null - timeit "Joiner-Recv-Unencrypted" "$tcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" - popd 1>/dev/null - - set -e - - if [[ $sfmt == 'xbstream' ]];then - # Special handling till lp:1193240 is fixed" - if [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then - wsrep_log_error "Xbstream failed" - wsrep_log_error "Data directory ${DATA} may not be empty: lp:1193240" \ - "Manual intervention required in that case" - exit 32 - fi - fi - - wait %% # join for wait_for_listen thread - - for ecode in "${RC[@]}";do - if [[ $ecode -ne 0 ]];then - wsrep_log_error "Error while getting data from donor node: " \ - "exit codes: ${RC[@]}" - exit 32 - fi - done - - if [ ! -r "${MAGIC_FILE}" ] - then - # this message should cause joiner to abort - wsrep_log_error "xtrabackup process ended without creating '${MAGIC_FILE}'" - wsrep_log_info "Contents of datadir" - wsrep_log_info "$(ls -l ${DATA}/**/*)" - exit 32 - fi - - if ! ps -p ${WSREP_SST_OPT_PARENT} &>/dev/null - then - wsrep_log_error "Parent mysqld process (PID:${WSREP_SST_OPT_PARENT}) terminated unexpectedly." - exit 32 - fi - - if [ ! -r "${DATA}/${IST_FILE}" ] - then - wsrep_log_info "Proceeding with SST" - wsrep_log_info "Removing existing ib_logfile files" - if [[ $incremental -ne 1 ]];then - rm -f ${DATA}/ib_logfile* - else - rm -f ${BDATA}/ib_logfile* - fi - - get_proc - - # Rebuild indexes for compact backups - if grep -q 'compact = 1' ${DATA}/xtrabackup_checkpoints;then - wsrep_log_info "Index compaction detected" - rebuild=1 - fi - - if [[ $rebuild -eq 1 ]];then - nthreads=$(parse_cnf xtrabackup rebuild-threads $nproc) - wsrep_log_info "Rebuilding during prepare with $nthreads threads" - rebuildcmd="--rebuild-indexes --rebuild-threads=$nthreads" - fi - - if test -n "$(find ${DATA} -maxdepth 1 -type f -name '*.qp' -print -quit)";then - - wsrep_log_info "Compressed qpress files found" - - if ! command -v qpress >/dev/null;then - wsrep_log_error "qpress not found in path: $PATH" - exit 22 - fi - - if [[ -n $progress ]] && pv --help | grep -q 'line-mode';then - count=$(find ${DATA} -type f -name '*.qp' | wc -l) - count=$(( count*2 )) - if pv --help | grep -q FORMAT;then - pvopts="-f -s $count -l -N Decompression -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" - else - pvopts="-f -s $count -l -N Decompression" - fi - pcmd="pv $pvopts" - adjust_progress - dcmd="$pcmd | xargs -n 2 qpress -T${nproc}d" - else - dcmd="xargs -n 2 qpress -T${nproc}d" - fi - - wsrep_log_info "Removing existing ibdata1 file" - rm -f ${DATA}/ibdata1 - - # Decompress the qpress files - wsrep_log_info "Decompression with $nproc threads" - timeit "Decompression" "find ${DATA} -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" - extcode=$? - - if [[ $extcode -eq 0 ]];then - wsrep_log_info "Removing qpress files after decompression" - find ${DATA} -type f -name '*.qp' -delete - if [[ $? -ne 0 ]];then - wsrep_log_error "Something went wrong with deletion of qpress files. Investigate" - fi - else - wsrep_log_error "Decompression failed. Exit code: $extcode" - exit 22 - fi - fi - - if [[ $incremental -eq 1 ]];then - # Added --ibbackup=xtrabackup_55 because it fails otherwise citing connection issues. - INNOAPPLY="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_CONF} \ - --ibbackup=xtrabackup_55 --apply-log $rebuildcmd --redo-only $BDATA --incremental-dir=${DATA} &>>${BDATA}/innobackup.prepare.log" - fi - - wsrep_log_info "Preparing the backup at ${DATA}" - timeit "Xtrabackup prepare stage" "$INNOAPPLY" - - if [[ $incremental -eq 1 ]];then - wsrep_log_info "Cleaning up ${DATA} after incremental SST" - [[ -d ${DATA} ]] && rm -rf ${DATA} - DATA=$BDATA - fi - - if [ $? -ne 0 ]; - then - wsrep_log_error "${INNOBACKUPEX_BIN} finished with errors. Check ${DATA}/innobackup.prepare.log" - exit 22 - fi - else - wsrep_log_info "${IST_FILE} received from donor: Running IST" - fi - - if [[ ! -r ${MAGIC_FILE} ]];then - wsrep_log_error "SST magic file ${MAGIC_FILE} not found/readable" - exit 2 - fi - - cat "${MAGIC_FILE}" # Output : UUID:seqno wsrep_gtid_domain_id - wsrep_log_info "Total time on joiner: $totime seconds" -fi - -exit 0 |