From 97509e270c4a4a8fc7acedbb37c74a3c924740b1 Mon Sep 17 00:00:00 2001 From: Brandon Nesterenko Date: Wed, 11 Aug 2021 11:29:37 -0600 Subject: MDEV-4989: Support for GTID in mysqlbinlog New Feature: =========== This commit extends the mariadb-binlog capabilities to allow events to be filtered by GTID ranges. More specifically, the following capabilities are addressed: 1) GTIDs can be used to filter results on local binlog files 2) GTIDs can be used to filter results from remote servers 3) For a given GTID range, its start-position is exclusive and its stop-position is inclusive. This allows users to receive events strictly after those that they already have, and is useful in cases such as: 1) events were received out of order and should be re-sent, or 2) specifying the gtid state of a slave to get events newer than their current state. domain id are reset to their former values. If a seq_no is 0 for start-position, it means to include the entirety of the domain. If a seq_no is 0 for stop-position, it means to exclude all events from that domain. 4) Implemented --gtid-strict-mode that ensures the GTID event stream in each domain is monotonically increasing 5) Added new level of verbosity in mysqlbinlog -vvv to print additional diagnostic information about invalid GTID states To facilitate these features, the --start-position and --stop-position arguments have been extended to accept values formatted as a list of GTID positions, e.g. --start-position=0-1-0,1-2-55 A few additional notes: 1) this commit squashes together the commits: f4319661120e-78a9d49907ba 2) Changed rpl.rpl_blackhole_row_annotate test because it has out of order GTIDs in its binlog, so I added --skip-gtid-strict-mode 3) After all binlog events have been written, the session server id and domain id are reset to their values in the global state. Reviewed By: =========== Andrei Elkin: --- client/mysqlbinlog.cc | 381 ++++++++- man/mysqlbinlog.1 | 38 +- .../include/mysqlbinlog_gtid_strict_mode.inc | 421 ++++++++++ .../include/mysqlbinlog_gtid_window_test_cases.inc | 606 ++++++++++++++ .../r/binlog_mysqlbinlog_gtid_strict_mode.result | 925 +++++++++++++++++++++ .../binlog/r/binlog_mysqlbinlog_gtid_window.result | 468 +++++++++++ .../t/binlog_mysqlbinlog_gtid_strict_mode.test | 105 +++ .../binlog/t/binlog_mysqlbinlog_gtid_window.test | 151 ++++ .../suite/rpl/t/rpl_blackhole_row_annotate.test | 2 +- sql/log_event.h | 48 ++ sql/log_event_client.cc | 2 + sql/rpl_gtid.cc | 846 ++++++++++++++++++- sql/rpl_gtid.h | 448 ++++++++++ 13 files changed, 4418 insertions(+), 23 deletions(-) create mode 100644 mysql-test/suite/binlog/include/mysqlbinlog_gtid_strict_mode.inc create mode 100644 mysql-test/suite/binlog/include/mysqlbinlog_gtid_window_test_cases.inc create mode 100644 mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_strict_mode.result create mode 100644 mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_window.result create mode 100644 mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_strict_mode.test create mode 100644 mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_window.test diff --git a/client/mysqlbinlog.cc b/client/mysqlbinlog.cc index 3d9b9712ec8..3ef174fefcb 100644 --- a/client/mysqlbinlog.cc +++ b/client/mysqlbinlog.cc @@ -128,6 +128,7 @@ static my_bool print_row_count_used= 0, print_row_event_positions_used= 0; static my_bool debug_info_flag, debug_check_flag; static my_bool force_if_open_opt= 1; static my_bool opt_raw_mode= 0, opt_stop_never= 0; +my_bool opt_gtid_strict_mode= true; static ulong opt_stop_never_slave_server_id= 0; static my_bool opt_verify_binlog_checksum= 1; static ulonglong offset = 0; @@ -143,10 +144,15 @@ static char *charset= 0; static uint verbose= 0; -static ulonglong start_position, stop_position; +static char *start_pos_str, *stop_pos_str; +static ulonglong start_position= BIN_LOG_HEADER_SIZE, + stop_position= (longlong)(~(my_off_t)0) ; #define start_position_mot ((my_off_t)start_position) #define stop_position_mot ((my_off_t)stop_position) +static Binlog_gtid_state_validator *gtid_state_validator= NULL; +static Domain_gtid_event_filter *domain_gtid_filter= NULL; + static char *start_datetime_str, *stop_datetime_str; static my_time_t start_datetime= 0, stop_datetime= MY_TIME_T_MAX; static ulonglong rec_count= 0; @@ -981,6 +987,10 @@ static bool print_row_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, return result; } +static inline my_bool is_gtid_filtering_enabled() +{ + return domain_gtid_filter != NULL; +} /** Print the given event, and either delete it or delegate the deletion @@ -1008,16 +1018,143 @@ Exit_status process_event(PRINT_EVENT_INFO *print_event_info, Log_event *ev, char ll_buff[21]; Log_event_type ev_type= ev->get_type_code(); my_bool destroy_evt= TRUE; + my_bool gtid_err= FALSE; DBUG_ENTER("process_event"); Exit_status retval= OK_CONTINUE; IO_CACHE *const head= &print_event_info->head_cache; + /* + We use Gtid_list_log_event information to determine if there is missing + data between where a user expects events to start/stop (i.e. the GTIDs + provided by --start-position and --stop-position), and the true start of + the specified binary logs. The first GLLE provides the initial state of the + binary logs. + + If --start-position is provided as a file offset, we want to skip initial + GTID state verification + */ + static my_bool was_first_glle_processed= start_position > BIN_LOG_HEADER_SIZE; + /* Bypass flashback settings to event */ ev->is_flashback= opt_flashback; #ifdef WHEN_FLASHBACK_REVIEW_READY ev->need_flashback_review= opt_flashback_review; #endif + /* + Run time estimation of the output window configuration. + + Do not validate GLLE information is start position is provided as a file + offset. + */ + if (ev_type == GTID_LIST_EVENT && ev->when) + { + Gtid_list_log_event *glev= (Gtid_list_log_event *)ev; + + /* + If this is the first Gtid_list_log_event, initialize the state of the + GTID stream auditor to be consistent with the binary logs provided + */ + if (gtid_state_validator && !was_first_glle_processed && glev->count) + { + if (gtid_state_validator->initialize_gtid_state(stderr, glev->list, + glev->count)) + goto err; + + if (domain_gtid_filter && !domain_gtid_filter->get_num_start_gtids()) + { + /* + We need to validate the GTID list from --stop-position because we + couldn't prove it intrinsically (i.e. using stop > start) + */ + rpl_gtid *stop_gtids= domain_gtid_filter->get_stop_gtids(); + size_t n_stop_gtids= domain_gtid_filter->get_num_stop_gtids(); + if (gtid_state_validator->verify_stop_state(stderr, stop_gtids, + n_stop_gtids)) + { + my_free(stop_gtids); + goto err; + } + my_free(stop_gtids); + } + } + + /* + Verify that we are able to process events from this binlog. For example, + if our current GTID state is behind the state of the GLLE in the new log, + a user may have accidentally left out a log file to process. + */ + if (gtid_state_validator && verbose >= 3) + for (size_t k= 0; k < glev->count; k++) + gtid_state_validator->verify_gtid_state(stderr, &(glev->list[k])); + + was_first_glle_processed= TRUE; + } + + if (ev_type == GTID_EVENT) + { + rpl_gtid ev_gtid; + Gtid_log_event *gle= (Gtid_log_event*) ev; + ev_gtid= {gle->domain_id, gle->server_id, gle->seq_no}; + + /* + If the binlog output should be filtered using GTIDs, test the new event + group to see if its events should be ignored. + */ + if (domain_gtid_filter) + { + if (domain_gtid_filter->has_finished()) + { + retval= OK_STOP; + goto end; + } + + if (!domain_gtid_filter->exclude(&ev_gtid)) + print_event_info->activate_current_event_group(); + else + print_event_info->deactivate_current_event_group(); + } + + /* + Where we always ensure the initial binlog state is valid, we only + continually monitor the GTID stream for validity if we are in GTID + strict mode (for errors) or if three levels of verbosity is provided + (for warnings). + + If we don't care about ensuring GTID validity, just delete the auditor + object to disable it for future checks. + */ + if (gtid_state_validator) + { + if (!(opt_gtid_strict_mode || verbose >= 3)) + { + delete gtid_state_validator; + + /* + Explicitly reset to NULL to simplify checks on if auditing is enabled + i.e. if it is defined, assume we want to use it + */ + gtid_state_validator= NULL; + } + else + { + gtid_err= gtid_state_validator->record(&ev_gtid); + if (gtid_err && opt_gtid_strict_mode) + { + gtid_state_validator->report(stderr, opt_gtid_strict_mode); + goto err; + } + } + } + } + + /* + If the GTID is ignored, it shouldn't count towards offset (rec_count should + not be incremented) + */ + if (!print_event_info->is_event_group_active()) + goto end_skip_count; + /* Format events are not concerned by --offset and such, we always need to read them to be able to process the wanted events. @@ -1458,6 +1595,7 @@ err: retval= ERROR_STOP; end: rec_count++; +end_skip_count: DBUG_PRINT("info", ("end event processing")); /* @@ -1658,15 +1796,22 @@ static struct my_option my_options[] = &start_datetime_str, &start_datetime_str, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"start-position", 'j', - "Start reading the binlog at position N. Applies to the first binlog " - "passed on the command line.", - &start_position, &start_position, 0, GET_ULL, - REQUIRED_ARG, BIN_LOG_HEADER_SIZE, BIN_LOG_HEADER_SIZE, - /* - COM_BINLOG_DUMP accepts only 4 bytes for the position - so remote log reading has lower limit. - */ - (ulonglong)(0xffffffffffffffffULL), 0, 0, 0}, + "Start reading the binlog at this position. Type can either be a positive " + "integer or a GTID list. When using a positive integer, the value only " + "applies to the first binlog passed on the command line. In GTID mode, " + "multiple GTIDs can be passed as a comma separated list, where each must " + "have a unique domain id. The list represents the gtid binlog state that " + "the client (another \"replica\" server) is aware of. Therefore, each GTID " + "is exclusive; only events after a given sequence number will be printed to " + "allow users to receive events after their current state.", + &start_pos_str, &start_pos_str, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, + 0, 0, 0}, + {"gtid-strict-mode", 0, "Process binlog according to gtid-strict-mode " + "specification. The start, stop positions are verified to satisfy " + "start < stop comparison condition. Sequence numbers of any gtid domain " + "must comprise monotically growing sequence", + &opt_gtid_strict_mode, &opt_gtid_strict_mode, 0, + GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0}, {"stop-datetime", OPT_STOP_DATETIME, "Stop reading the binlog at first event having a datetime equal or " "posterior to the argument; the argument must be a date and time " @@ -1684,11 +1829,14 @@ static struct my_option my_options[] = &opt_stop_never_slave_server_id, &opt_stop_never_slave_server_id, 0, GET_ULONG, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"stop-position", OPT_STOP_POSITION, - "Stop reading the binlog at position N. Applies to the last binlog " - "passed on the command line.", - &stop_position, &stop_position, 0, GET_ULL, - REQUIRED_ARG, (longlong)(~(my_off_t)0), BIN_LOG_HEADER_SIZE, - (ulonglong)(~(my_off_t)0), 0, 0, 0}, + "Stop reading the binlog at this position. Type can either be a positive " + "integer or a GTID list. When using a positive integer, the value only " + "applies to the last binlog passed on the command line. In GTID mode, " + "multiple GTIDs can be passed as a comma separated list, where each must " + "have a unique domain id. Each GTID is inclusive; only events up to the " + "given sequence numbers are printed.", + &stop_pos_str, &stop_pos_str, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, + 0, 0}, {"table", 'T', "List entries for just this table (local log only).", &table, &table, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, @@ -1702,7 +1850,9 @@ that may lead to an endless loop.", &user, &user, 0, GET_STR_ALLOC, REQUIRED_ARG, 0, 0, 0, 0, 0, 0}, {"verbose", 'v', "Reconstruct SQL statements out of row events. " - "-v -v adds comments on column data types.", + "-v -v adds comments on column data types. " + "-v -v -v adds diagnostic warnings about event " + "integrity before program exit.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, {"version", 'V', "Print version and exit.", 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}, @@ -1824,8 +1974,14 @@ static void cleanup() my_free(const_cast(dirname_for_local_load)); my_free(start_datetime_str); my_free(stop_datetime_str); + my_free(start_pos_str); + my_free(stop_pos_str); free_root(&glob_root, MYF(0)); + delete domain_gtid_filter; + if (gtid_state_validator) + delete gtid_state_validator; + delete binlog_filter; delete glob_description_event; if (mysql) @@ -2075,6 +2231,110 @@ get_one_option(const struct my_option *opt, const char *argument, const char *fi print_version(); opt_version= 1; break; + case OPT_STOP_POSITION: + { + /* Stop position was already specified, so reset it and use the new list */ + if (domain_gtid_filter && domain_gtid_filter->get_num_stop_gtids() > 0) + domain_gtid_filter->clear_stop_gtids(); + + uint32 n_stop_gtid_ranges= 0; + rpl_gtid *stop_gtids= gtid_parse_string_to_list( + stop_pos_str, strlen(stop_pos_str), &n_stop_gtid_ranges); + if (stop_gtids == NULL) + { + int err= 0; + char *end_ptr= NULL; + /* + No GTIDs specified in OPT_STOP_POSITION specification. Treat the value + as a singular index. + */ + stop_position= my_strtoll10(stop_pos_str, &end_ptr, &err); + + if (err || *end_ptr) + { + // Can't parse the position from the user + sql_print_error("Stop position argument value is invalid. Should be " + "either a positive integer or GTID."); + return 1; + } + } + else if (n_stop_gtid_ranges > 0) + { + uint32 gtid_idx; + + if (domain_gtid_filter == NULL) + domain_gtid_filter= new Domain_gtid_event_filter(); + + for (gtid_idx = 0; gtid_idx < n_stop_gtid_ranges; gtid_idx++) + { + rpl_gtid *stop_gtid= &stop_gtids[gtid_idx]; + if (domain_gtid_filter->add_stop_gtid(stop_gtid)) + { + my_free(stop_gtids); + return 1; + } + } + my_free(stop_gtids); + } + else + { + DBUG_ASSERT(0); + } + break; + } + case 'j': + { + /* Start position was already specified, so reset it and use the new list */ + if (domain_gtid_filter && domain_gtid_filter->get_num_start_gtids() > 0) + domain_gtid_filter->clear_start_gtids(); + + uint32 n_start_gtid_ranges= 0; + rpl_gtid *start_gtids= gtid_parse_string_to_list( + start_pos_str, strlen(start_pos_str), &n_start_gtid_ranges); + + if (start_gtids == NULL) + { + int err= 0; + char *end_ptr= NULL; + /* + No GTIDs specified in OPT_START_POSITION specification. Treat the value + as a singular index. + */ + start_position= my_strtoll10(start_pos_str, &end_ptr, &err); + + if (err || *end_ptr) + { + // Can't parse the position from the user + sql_print_error("Start position argument value is invalid. Should be " + "either a positive integer or GTID."); + return 1; + } + } + else if (n_start_gtid_ranges > 0) + { + uint32 gtid_idx; + + if (domain_gtid_filter == NULL) + domain_gtid_filter= new Domain_gtid_event_filter(); + + for (gtid_idx = 0; gtid_idx < n_start_gtid_ranges; gtid_idx++) + { + rpl_gtid *start_gtid= &start_gtids[gtid_idx]; + if (start_gtid->seq_no && + domain_gtid_filter->add_start_gtid(start_gtid)) + { + my_free(start_gtids); + return 1; + } + } + my_free(start_gtids); + } + else + { + DBUG_ASSERT(0); + } + break; + } case '?': usage(); opt_version= 1; @@ -2107,6 +2367,37 @@ static int parse_args(int *argc, char*** argv) start_position); start_position= UINT_MAX32; } + + /* + Always initialize the stream auditor initially because it is used to check + the initial state of the binary log is correct. If we don't want it later + (i.e. --skip-gtid-strict-mode or -vvv is not given), it is deleted when we + are certain the initial gtid state is set. + */ + gtid_state_validator= new Binlog_gtid_state_validator(); + + if (domain_gtid_filter) + { + if (opt_gtid_strict_mode && domain_gtid_filter->validate_window_filters()) + { + /* + In strict mode, if any --start/stop-position GTID ranges are invalid, + quit in error. Note that any specific error messages will have + already been written. + */ + die(); + } + + /* + GTIDs before a start position shouldn't be validated, so we initialize + the stream auditor to only monitor GTIDs after these positions. + */ + size_t n_start_gtids= domain_gtid_filter->get_num_start_gtids(); + rpl_gtid *start_gtids= domain_gtid_filter->get_start_gtids(); + gtid_state_validator->initialize_start_gtids(start_gtids, n_start_gtids); + my_free(start_gtids); + } + return 0; } @@ -2187,6 +2478,9 @@ static Exit_status dump_log_entries(const char* logname) if (!print_event_info.init_ok()) return ERROR_STOP; + + if (domain_gtid_filter) + print_event_info.enable_event_group_filtering(); /* Set safe delimiter, to dump things like CREATE PROCEDURE safely @@ -2288,6 +2582,40 @@ static Exit_status check_master_version() goto err; } + if (domain_gtid_filter && domain_gtid_filter->get_num_start_gtids() > 0) + { + char str_buf[256]; + String query_str(str_buf, sizeof(str_buf), system_charset_info); + query_str.length(0); + query_str.append(STRING_WITH_LEN("SET @slave_connect_state='"), + system_charset_info); + + size_t n_start_gtids= domain_gtid_filter->get_num_start_gtids(); + rpl_gtid *start_gtids= domain_gtid_filter->get_start_gtids(); + + for (size_t gtid_idx = 0; gtid_idx < n_start_gtids; gtid_idx++) + { + char buf[256]; + rpl_gtid *start_gtid= &start_gtids[gtid_idx]; + + sprintf(buf, "%u-%u-%llu", + start_gtid->domain_id, start_gtid->server_id, + start_gtid->seq_no); + query_str.append(buf, strlen(buf)); + if (gtid_idx < n_start_gtids - 1) + query_str.append(','); + } + my_free(start_gtids); + + query_str.append(STRING_WITH_LEN("'"), system_charset_info); + if (unlikely(mysql_real_query(mysql, query_str.ptr(), query_str.length()))) + { + error("Setting @slave_connect_state failed with error: %s", + mysql_error(mysql)); + goto err; + } + } + delete glob_description_event; glob_description_event= NULL; @@ -3210,12 +3538,32 @@ int main(int argc, char** argv) "/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */;\n"); fprintf(result_file, "/*!50530 SET @@SESSION.PSEUDO_SLAVE_MODE=0*/;\n"); + + if (is_gtid_filtering_enabled()) + { + fprintf(result_file, + "/*!100001 SET @@SESSION.SERVER_ID=@@GLOBAL.SERVER_ID */;\n" + "/*!100001 SET @@SESSION.GTID_DOMAIN_ID=@@GLOBAL.GTID_DOMAIN_ID " + "*/;\n"); + } } if (tmpdir.list) free_tmpdir(&tmpdir); if (result_file && result_file != stdout) my_fclose(result_file, MYF(0)); + + /* + Ensure the GTID state is correct. If not, end in error. + + Note that in gtid strict mode, we will not report here if any invalid GTIDs + are processed because it immediately errors (i.e. retval will be + ERROR_STOP) + */ + if (retval != ERROR_STOP && gtid_state_validator && + gtid_state_validator->report(stderr, opt_gtid_strict_mode)) + retval= ERROR_STOP; + cleanup(); /* We cannot free DBUG, it is used in global destructors after exit(). */ my_end(my_end_arg | MY_DONT_FREE_DBUG); @@ -3271,3 +3619,4 @@ struct encryption_service_st encryption_handler= #include "sql_list.cc" #include "rpl_filter.cc" #include "compat56.cc" +#include "rpl_gtid.cc" diff --git a/man/mysqlbinlog.1 b/man/mysqlbinlog.1 index 633300bb7c5..7ad8c714490 100644 --- a/man/mysqlbinlog.1 +++ b/man/mysqlbinlog.1 @@ -992,8 +992,15 @@ This option is useful for point\-in\-time recovery\&. \fB\-\-start\-position=\fR\fB\fIN\fR\fR, \fB\-j \fR\fB\fIN\fR\fR .sp -Start reading the binary log at the first event having a position equal to or greater than -\fIN\fR\&. This option applies to the first log file named on the command line\&. +Start reading the binary log at \fIN\fR\&. Type can either be a positive +integer or a GTID\& list\&. When using a positive integer, the value only +applies to the first binlog passed on the command line, and the first event +that has a position equal to or greater than \fIN\fR is printed\&. In GTID mode, +multiple GTIDs can be passed as a comma separated list, where each must have a +unique domain id\&. The list represents the gtid binlog state that the client +(another "replica" server) is aware of\&. Therefore, each GTID is exclusive; only +events after a given sequence number will be printed to allow users to receive +events after their current state\&. .sp This option is useful for point\-in\-time recovery\&. .RE @@ -1006,6 +1013,23 @@ This option is useful for point\-in\-time recovery\&. .sp -1 .IP \(bu 2.3 .\} +.\" mysqlbinlog: gtid-strict-mode +.\" gtid-strict-mode option: mysqlbinlog +\fB\-\-gtid\-strict\-mode +.sp +Process binlog according to gtid-strict-mode specification\&. The start, stop +positions are verified to satisfy start < stop comparison condition\&. Sequence +numbers of any gtid domain must comprise monotically growing sequence\&. +.RE +.sp +.RS 4 +.ie n \{\ +\h'-04'\(bu\h'+03'\c +.\} +.el \{\ +.sp -1 +.IP \(bu 2.3 +.\} .\" mysqlbinlog: stop-datetime option .\" stop-datetime option: mysqlbinlog \fB\-\-stop\-datetime=\fR\fB\fIdatetime\fR\fR @@ -1063,8 +1087,13 @@ The slave server_id used for \fB--read-from-remote-server --stop-never\fR\&. .\" stop-position option: mysqlbinlog \fB\-\-stop\-position=\fR\fB\fIN\fR\fR .sp -Stop reading the binary log at the first event having a position equal to or greater than -\fIN\fR\&. This option applies to the last log file named on the command line\&. +Stop reading the binary log at the first event having a position equal to or +greater than \fIN\fR\&. Type can either be a positive integer or a GTID +list\&. When using a positive integer, the value only applies to the last log +file named on the command line\&. When in GTID mode, multiple GTIDs can be +passed as a comma separated list, where each must have a unique domain id\&. +Each GTID is inclusive; only events up to the given sequence numbers are +printed. .sp This option is useful for point\-in\-time recovery\&. .RE @@ -1133,6 +1162,7 @@ The MariaDB username to use when connecting to a remote server\&. \fB\-v\fR .sp Reconstruct row events and display them as commented SQL statements\&. If this option is given twice, the output includes comments to indicate column data types and some metadata\&. +If this option is given three times, the output includes diagnostic warnings about event integrity before program exit\&. .sp For examples that show the effect of \fB\-\-base64\-output\fR diff --git a/mysql-test/suite/binlog/include/mysqlbinlog_gtid_strict_mode.inc b/mysql-test/suite/binlog/include/mysqlbinlog_gtid_strict_mode.inc new file mode 100644 index 00000000000..5150e9060b5 --- /dev/null +++ b/mysql-test/suite/binlog/include/mysqlbinlog_gtid_strict_mode.inc @@ -0,0 +1,421 @@ +# +# This file runs test cases for using --gtid-strict-mode with mariadb-binlog to +# ensure warnings are properly displayed +# +# param $is_strict_mode boolean (0 for false, 1 for true) to enable or +# disable strict mode for GTID processing +# + +--let MYSQLD_DATADIR=`select @@datadir` +--let OUT_FILE=$MYSQLTEST_VARDIR/tmp/binlog.out + +if ($is_strict_mode == 0) +{ + --let BINLOG_STRICT_MODE_PARAM=--skip-gtid-strict-mode +} +if ($is_strict_mode == 1) +{ + --let BINLOG_STRICT_MODE_PARAM=--gtid-strict-mode +} +if ($is_verbose == 1) +{ + --let BINLOG_STRICT_MODE_PARAM=$BINLOG_STRICT_MODE_PARAM -vvv +} + +--let $log_error_ = $MYSQLTEST_VARDIR/tmp/out.err +--let SEARCH_FILE=$log_error_ + +--echo # +--echo # Test Case 1: +--echo # Sequential sequence numbers results in no errors or warnings +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +INSERT INTO t1 values (3); +FLUSH LOGS; +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 $BINLOG_STRICT_MODE_PARAM 2> log_error_ > OUT_FILE +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 $BINLOG_STRICT_MODE_PARAM 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=$DEFAULT_ERROR_PREFIX +--source include/search_pattern_in_file.inc +--remove_file $log_error_ +DROP TABLE t1; +RESET MASTER; + +--echo # +--echo # Test Case 2: +--echo # A skipped sequence number results in no errors or warnings if all +--echo # numbers are monotonic (i.e. gaps in sequence number are allowed +--echo # provided they never decrease) +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 5; +INSERT INTO t1 values (3); +FLUSH LOGS; +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 $BINLOG_STRICT_MODE_PARAM 2> log_error_ > OUT_FILE +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 $BINLOG_STRICT_MODE_PARAM 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=$DEFAULT_ERROR_PREFIX +--source include/search_pattern_in_file.inc +--remove_file $log_error_ +DROP TABLE t1; +RESET MASTER; + +--echo # +--echo # Test Case 3: +--echo # A sequence number lower than the last processed value results in a +--echo # warning or error +CREATE TABLE t1 (a int); +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 1; +INSERT INTO t1 values (1); +FLUSH LOGS; +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 $BINLOG_STRICT_MODE_PARAM 2> log_error_ > OUT_FILE +--error $is_strict_mode +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 $BINLOG_STRICT_MODE_PARAM 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=$DEFAULT_ERROR_PREFIX: Found out of order GTID +--source include/search_pattern_in_file.inc +--remove_file $log_error_ +DROP TABLE t1; +RESET MASTER; + +--echo # +--echo # Test Case 4: +--echo # Skipping a GTID and later receiving it results in a warning or error +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (3); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (4); +INSERT INTO t1 values (5); +FLUSH LOGS; +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 $BINLOG_STRICT_MODE_PARAM 2> log_error_ > OUT_FILE +--error $is_strict_mode +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 $BINLOG_STRICT_MODE_PARAM 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=$DEFAULT_ERROR_PREFIX: Found out of order GTID +--source include/search_pattern_in_file.inc +--remove_file $log_error_ +DROP TABLE t1; +RESET MASTER; + +--echo # +--echo # Test Case 5: +--echo # Repeat sequence numbers produce a warning +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (3); +INSERT INTO t1 values (4); +FLUSH LOGS; +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 $BINLOG_STRICT_MODE_PARAM 2> log_error_ > OUT_FILE +--error $is_strict_mode +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 $BINLOG_STRICT_MODE_PARAM 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=$DEFAULT_ERROR_PREFIX: Found out of order GTID +--source include/search_pattern_in_file.inc +--remove_file $log_error_ +DROP TABLE t1; +RESET MASTER; + +--echo # +--echo # Test Case 6: +--echo # Warnings from different domains are all displayed +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (3); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (4); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t2 values (2); +SET @@session.gtid_seq_no= 2; +INSERT INTO t2 values (3); +SET @@session.gtid_seq_no= 4; +INSERT INTO t2 values (4); +FLUSH LOGS; +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 $BINLOG_STRICT_MODE_PARAM 2> log_error_ > OUT_FILE +--error $is_strict_mode +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 $BINLOG_STRICT_MODE_PARAM 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=$DEFAULT_ERROR_PREFIX: Found out of order GTID +--source include/search_pattern_in_file.inc +--remove_file $log_error_ +DROP TABLE t1; +DROP TABLE t2; +RESET MASTER; + +--echo # +--echo # Test Case 7: +--echo # A decreasing seq_no before a start-position is ignored +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +SET @@session.gtid_seq_no= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (1); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (2); +--let $start_binlog_pos= query_get_value(SHOW MASTER STATUS,Position, 1) +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (4); +INSERT INTO t1 values (3); +INSERT INTO t1 values (5); +FLUSH LOGS; + +--echo # GTID-based start-position +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-4 $BINLOG_STRICT_MODE_PARAM > log_error_ > OUT_FILE +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-4 $BINLOG_STRICT_MODE_PARAM 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=$DEFAULT_ERROR_PREFIX +--source include/search_pattern_in_file.inc + +--echo # Position-based start-position +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=start_binlog_pos $BINLOG_STRICT_MODE_PARAM 2> log_error_ > OUT_FILE +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=$start_binlog_pos $BINLOG_STRICT_MODE_PARAM 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=$DEFAULT_ERROR_PREFIX +--source include/search_pattern_in_file.inc + +--remove_file $log_error_ +DROP TABLE t1; +RESET MASTER; + +--echo # +--echo # Test Case 8: +--echo # A decreasing seq_no inside of a --start/--stop position window is +--echo # displayed +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +--let $start_binlog_pos= query_get_value(SHOW MASTER STATUS,Position, 1) +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 5; +INSERT INTO t1 values (4); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (3); +SET @@session.gtid_seq_no= 6; +INSERT INTO t1 values (5); +--let $stop_binlog_pos= query_get_value(SHOW MASTER STATUS,Position, 1) +FLUSH LOGS; + +--echo # GTID-based window +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 --stop-position=0-1-6 $BINLOG_STRICT_MODE_PARAM > log_error_ > OUT_FILE +--error $is_strict_mode +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 --stop-position=0-1-6 $BINLOG_STRICT_MODE_PARAM 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=$DEFAULT_ERROR_PREFIX: Found out of order GTID +--source include/search_pattern_in_file.inc + +--echo # Position-based window +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=start_binlog_pos --stop-position=stop_binlog_pos $BINLOG_STRICT_MODE_PARAM 2> log_error_ > OUT_FILE +--error $is_strict_mode +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=$start_binlog_pos --stop-position=$stop_binlog_pos $BINLOG_STRICT_MODE_PARAM 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=$DEFAULT_ERROR_PREFIX: Found out of order GTID +--source include/search_pattern_in_file.inc + +--remove_file $log_error_ +DROP TABLE t1; +RESET MASTER; + +--echo # +--echo # Test Case 9: +--echo # Error if --stop-position is not greater than or equal to +--echo # --start-position +--echo # +--echo # Note: Error is only displayed in strict mode, -vvv has no effect here +FLUSH LOGS; +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-2 --stop-position=0-1-1 $BINLOG_STRICT_MODE_PARAM 2> log_error_ > OUT_FILE +--error $is_strict_mode +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-2 --stop-position=0-1-1 $BINLOG_STRICT_MODE_PARAM 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=ERROR: Queried GTID range is invalid in strict mode +--source include/search_pattern_in_file.inc + +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-2,1-2-1 --stop-position=0-1-1,1-2-2 $BINLOG_STRICT_MODE_PARAM 2> log_error_ > OUT_FILE +--error $is_strict_mode +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-2,1-2-1 --stop-position=0-1-1,1-2-2 $BINLOG_STRICT_MODE_PARAM 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=ERROR: Queried GTID range is invalid in strict mode +--source include/search_pattern_in_file.inc +RESET MASTER; + +--echo # +--echo # Test Case 10: +--echo # Strict mode warnings should be independent of --offset option +--echo # specification +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +SET @@session.gtid_seq_no= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (1); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (3); +INSERT INTO t1 values (4); +FLUSH LOGS; +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 --offset=8 $BINLOG_STRICT_MODE_PARAM 2> log_error_ > OUT_FILE +--error $is_strict_mode +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 --offset=8 $BINLOG_STRICT_MODE_PARAM 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=$DEFAULT_ERROR_PREFIX: Found out of order GTID +--source include/search_pattern_in_file.inc +--remove_file $log_error_ +DROP TABLE t1; +RESET MASTER; + +--echo # +--echo # Test Case 11: +--echo # Strict mode warnings should be independent of --start-timestamp +--echo # option specification +set @a=UNIX_TIMESTAMP("1970-01-21 15:32:22"); +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +SET @@session.gtid_seq_no= 1; +SET timestamp=@a; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +SET timestamp=@a+1; +INSERT INTO t1 values (1); +SET @@session.gtid_seq_no= 2; +SET timestamp=@a+2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 4; +SET timestamp=@a+3; +INSERT INTO t1 values (3); +INSERT INTO t1 values (4); +FLUSH LOGS; +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 --start-datetime="1970-01-21 15:32:24" $BINLOG_STRICT_MODE_PARAM 2> log_error_ > OUT_FILE +--error $is_strict_mode +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 --start-datetime="1970-01-21 15:32:24" $BINLOG_STRICT_MODE_PARAM 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=$DEFAULT_ERROR_PREFIX: Found out of order GTID +--source include/search_pattern_in_file.inc +--remove_file $log_error_ +DROP TABLE t1; +RESET MASTER; + +--echo # +--echo # Test Case 12: +--echo # Specifying multiple binary logs with a log-position start should +--echo # skip GTID state verification +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +--let $b2_start_pos= query_get_value(SHOW MASTER STATUS,Position, 1) +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +INSERT INTO t2 values (1); +FLUSH LOGS; +INSERT INTO t2 values (2); +FLUSH LOGS; +--let BINLOG_FILE1= query_get_value(SHOW BINARY LOGS, Log_name, 1) +# Skip file 2 as input +--let BINLOG_FILE2= query_get_value(SHOW BINARY LOGS, Log_name, 2) +--echo # MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE1 MYSQLD_DATADIR/BINLOG_FILE2 --start-position=b2_start_pos $BINLOG_STRICT_MODE_PARAM 2> log_error_ > OUT_FILE +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/$BINLOG_FILE1 $MYSQLD_DATADIR/$BINLOG_FILE2 --start-position=$b2_start_pos $BINLOG_STRICT_MODE_PARAM 2> $log_error_ > $OUT_FILE +DROP TABLE t1; +DROP TABLE t2; + + +--echo # +--echo # Test Case 13: +--echo # If multiple binary logs should be specified but a middle log is +--echo # missing, we should detect that and warn when using -vvv + +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +FLUSH LOGS; +INSERT INTO t1 values (1); +--let $b2_start_pos= query_get_value(SHOW MASTER STATUS,Position, 1) +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +FLUSH LOGS; +SET @@session.gtid_domain_id= 2; +SET @@session.server_id= 3; +CREATE TABLE t3 (a int); +FLUSH LOGS; + +--echo # +--echo # GLLE from each log for state reference +--let $binlog_file= query_get_value(SHOW BINARY LOGS, Log_name, 1) +--source include/show_gtid_list.inc +--let $binlog_file= +--let $binlog_file= query_get_value(SHOW BINARY LOGS, Log_name, 2) +--source include/show_gtid_list.inc +--let $binlog_file= +--let $binlog_file= query_get_value(SHOW BINARY LOGS, Log_name, 3) +--source include/show_gtid_list.inc +--let $binlog_file= + +--let BINLOG_FILE1= query_get_value(SHOW BINARY LOGS, Log_name, 1) +# Skip file 2 as input +--let BINLOG_FILE3= query_get_value(SHOW BINARY LOGS, Log_name, 3) +--echo # MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE1 MYSQLD_DATADIR/BINLOG_FILE3 $BINLOG_STRICT_MODE_PARAM 2> log_error_ > OUT_FILE +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/$BINLOG_FILE1 $MYSQLD_DATADIR/$BINLOG_FILE3 $BINLOG_STRICT_MODE_PARAM 2> $log_error_ > $OUT_FILE +--echo # We should have two warnings about missing data from domains 0 and 1 if +--echo # -vvv is specified +--let SEARCH_FILE=$log_error_ +--let SEARCH_PATTERN=WARNING: Binary logs are missing data for domain 0[^\n]+the last seen event was +--source include/search_pattern_in_file.inc +--let SEARCH_PATTERN=WARNING: Binary logs are missing data for domain 1[^\n]+neither the starting GTID position list nor any processed events +--source include/search_pattern_in_file.inc +--remove_file $log_error_ +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +RESET MASTER; + +--echo # +--echo # Test Case 14: +--echo # If a --stop-position GTID occurs before the first specified binlog's +--echo # GLLE, error +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +FLUSH LOGS; +INSERT INTO t1 values (3); +FLUSH LOGS; + +--echo # +--echo # GLLE from each log for state reference +--let $binlog_file= query_get_value(SHOW BINARY LOGS, Log_name, 1) +--source include/show_gtid_list.inc +--let $binlog_file= +--let $binlog_file= query_get_value(SHOW BINARY LOGS, Log_name, 2) +--source include/show_gtid_list.inc +--let $binlog_file= +--echo # MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE2 $BINLOG_STRICT_MODE_PARAM --stop-position=0-1-2 2> log_error_ > OUT_FILE +--error 1 +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/$BINLOG_FILE2 $BINLOG_STRICT_MODE_PARAM --stop-position=0-1-2 2> $log_error_ > $OUT_FILE +--let SEARCH_PATTERN=ERROR: --stop-position GTID +--source include/search_pattern_in_file.inc +--remove_file $log_error_ +DROP TABLE t1; + +--remove_file $OUT_FILE diff --git a/mysql-test/suite/binlog/include/mysqlbinlog_gtid_window_test_cases.inc b/mysql-test/suite/binlog/include/mysqlbinlog_gtid_window_test_cases.inc new file mode 100644 index 00000000000..2830f3f665d --- /dev/null +++ b/mysql-test/suite/binlog/include/mysqlbinlog_gtid_window_test_cases.inc @@ -0,0 +1,606 @@ +# +# This file runs test cases for providing GTIDs to --start-position and +# --stop-position arguments in mariadb-binlog +# +# param $is_remote boolean (0 for false, 1 for true) to perform a local file +# or remote host analysis +# + +--let MYSQLD_DATADIR=`select @@datadir` +--let data_inconsistent_err= "table data is inconsistent after replaying binlog using GTID start/stop positions"; +--let $tmp_out_ = $MYSQLTEST_VARDIR/tmp/null.log + +## Initialize test data +# +set @a=UNIX_TIMESTAMP("1970-01-21 15:32:22"); +SET timestamp=@a; +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +--let $empty_binlog_pos= query_get_value(SHOW MASTER STATUS,Position, 1) +CREATE TABLE t1 (a int); +SET timestamp=@a+1; +INSERT INTO t1 values (1), (2); +--let test2_t1_mid_checksum= `CHECKSUM TABLE t1` + +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +SET timestamp=@a+2; +CREATE TABLE t2 (a int); +SET timestamp=@a+3; +INSERT INTO t2 values (1); +--let t2_mid_checksum= `CHECKSUM TABLE t2` + +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +SET timestamp=@a+4; +INSERT INTO t1 values (3), (4); +--let t1_final_checksum_pos= query_get_value(SHOW MASTER STATUS,Position, 1) +--let t1_final_checksum= `CHECKSUM TABLE t1` + +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +INSERT INTO t2 values (2); +--let test4_t2_good_checksum= `CHECKSUM TABLE t2` + +SET @@session.server_id= 3; +INSERT INTO t2 values (3); +--let test3_t2_good_checksum= `CHECKSUM TABLE t2` + +SET @@session.server_id= 2; +INSERT INTO t2 values (4); +--let t2_final_checksum= `CHECKSUM TABLE t2` + +FLUSH LOGS; + +# Multiple binlog file case, used by test 18 +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t3 (a int); +INSERT INTO t3 VALUES (1); +--let t3_final_checksum= `CHECKSUM TABLE t3` + +FLUSH LOGS; + +--echo # Gtid list event of the 2nd binlog file whose content is +--echo # matched against --start-position in the following tests: +--let $binlog_file= query_get_value(SHOW BINARY LOGS, Log_name, 2) +--source include/show_gtid_list.inc +--let $binlog_file= + +--let BINLOG_FILE= query_get_value(SHOW BINARY LOGS, Log_name, 1) +--let BINLOG_FILE2= query_get_value(SHOW BINARY LOGS, Log_name, 2) + +if ($is_remote == 0) +{ + --let BINLOG_FILE_PARAM=$MYSQLD_DATADIR/$BINLOG_FILE.orig + --let BINLOG_FILE_PARAM2=$MYSQLD_DATADIR/$BINLOG_FILE2.orig +} +if ($is_remote == 1) +{ + --let BINLOG_FILE_PARAM= --read-from-remote-server $BINLOG_FILE + --let BINLOG_FILE_PARAM2= --read-from-remote-server $BINLOG_FILE2 +} + +--copy_file $MYSQLD_DATADIR/$BINLOG_FILE $MYSQLD_DATADIR/$BINLOG_FILE.orig +--copy_file $MYSQLD_DATADIR/$BINLOG_FILE2 $MYSQLD_DATADIR/$BINLOG_FILE2.orig + +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; + + +--echo # +--echo # Test Case 1: +--echo # The end of the binlog file resets the server and domain id of the +--echo # session + +# As gtid_domain_id and server_id should not change after reading binlog in GTID +# mode, change variables to otherwise-unused values to ensure they remain +--let $reset_gtid_domain_id = `select @@session.gtid_domain_id` +--let $reset_server_id = `select @@session.server_id` +SET @@session.gtid_domain_id= 10; +SET @@session.server_id= 20; + +# Replay the binlog +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-2 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-2 | $MYSQL +--let $test_gtid_domain_id = `select @@session.gtid_domain_id` + +# Ensure variables haven't changed +--let $assert_text = session gtid_domain_id should not change when reading binlog in GTID mode +--let $assert_cond = @@session.gtid_domain_id = 10 +--source include/assert.inc +--let $assert_text = session server_id should not change when reading binlog in GTID mode +--let $assert_cond = @@session.server_id = 20 +--source include/assert.inc + +# Reset back to previous state +--eval SET @@session.gtid_domain_id= $reset_gtid_domain_id +--eval SET @@session.server_id= $reset_server_id +DROP TABLE t1; + + +--echo # +--echo # Test Case 2: +--echo # Single GTID range specified +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-2 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-2 | $MYSQL + +if ($test2_t1_mid_checksum != `CHECKSUM TABLE t1`) +{ + die $data_inconsistent_err; +} +DROP TABLE t1; + + +--echo # +--echo # Test Case 3: +--echo # Single GTID range with different server_ids +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=1-2-0 --stop-position=1-3-4 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --start-position=1-2-0 --stop-position=1-3-4 | $MYSQL + +if ($test3_t2_good_checksum != `CHECKSUM TABLE t2`) +{ + die $data_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't1'`) +{ + die "t1 should not exist as binlog replay should exclude domain 0 from stop position"; +} +DROP TABLE t2; + + +--echo # +--echo # Test Case 4: +--echo # Multiple GTID ranges specified +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0,1-2-0 --stop-position=0-1-3,1-2-3 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --start-position=0-1-0,1-2-0 --stop-position=0-1-3,1-2-3 | $MYSQL + +# Reuse checksum spot from test 4 +if ($t1_final_checksum != `CHECKSUM TABLE t1`) +{ + die $data_inconsistent_err; +} +if ($test4_t2_good_checksum != `CHECKSUM TABLE t2`) +{ + die $data_inconsistent_err; +} +DROP TABLE t1; +DROP TABLE t2; + + +--echo # +--echo # Test Case 5: +--echo # Multiple GTID ranges specified where the domain ids are listed in +--echo # different orders between start/stop position +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --stop-position=0-1-3,1-2-3 --start-position=1-2-0,0-1-0 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --stop-position=0-1-3,1-2-3 --start-position=1-2-0,0-1-0 | $MYSQL + +# Reuse checksum spot from test 4 +if ($t1_final_checksum != `CHECKSUM TABLE t1`) +{ + die $data_inconsistent_err; +} +if ($test4_t2_good_checksum != `CHECKSUM TABLE t2`) +{ + die $data_inconsistent_err; +} +DROP TABLE t1; +DROP TABLE t2; + + +--echo # +--echo # Test Case 6: +--echo # Only start position specified +CREATE TABLE t1 (a int); +INSERT INTO t1 values (3), (4); +--let test6_t1_mid_checksum= `CHECKSUM TABLE t1` +DROP TABLE t1; +CREATE TABLE t1 (a int); +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-2 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --start-position=0-1-2 | $MYSQL +if ($test6_t1_mid_checksum != `CHECKSUM TABLE t1`) +{ + die $data_inconsistent_err; +} +if ($t2_final_checksum != `CHECKSUM TABLE t2`) +{ + die $data_inconsistent_err; +} +DROP TABLE t1; +DROP TABLE t2; + + +--echo # +--echo # Test Case 7: +--echo # Only stop position specified +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --stop-position=0-1-2 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --stop-position=0-1-2 | $MYSQL + +# Reuse checksum spot from test 2 +if ($test2_t1_mid_checksum != `CHECKSUM TABLE t1`) +{ + die $data_inconsistent_err; +} +DROP TABLE t1; + + +--echo # +--echo # Test Case 8: +--echo # Seq_no=0 in --start-position includes all events for a domain +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0,1-2-0 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --start-position=0-1-0,1-2-0 | $MYSQL +if ($t1_final_checksum != `CHECKSUM TABLE t1`) +{ + die "t1 data should be complete as binlog replay should include domain 0 entirely in results"; +} +if ($t2_final_checksum != `CHECKSUM TABLE t2`) +{ + die "t2 data should be complete as binlog replay should include domain 1 entirely in results"; +} +DROP TABLE t1; +DROP TABLE t2; + +--echo # +--echo # Test Case 9: +--echo # Seq_no=0 in --stop-position excludes all events for a domain +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --stop-position=0-1-0,1-2-0 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --stop-position=0-1-0,1-2-0 | $MYSQL +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't1'`) +{ + die "t1 should not exist as binlog replay should exclude domain 0 from results"; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't2'`) +{ + die "t2 should not exist as binlog replay should exclude domain 1 from results"; +} + +--echo # +--echo # Test Case 10: +--echo # Output stops for all domain ids when all --stop-position GTID values +--echo # have been hit. +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --stop-position=0-1-2 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --stop-position=0-1-2 | $MYSQL +if ($test2_t1_mid_checksum != `CHECKSUM TABLE t1`) +{ + die $data_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't2'`) +{ + die "t2 should not exist as binlog replay should exclude domain 1 from results"; +} +DROP TABLE t1; + +--echo # +--echo # Test Case 11: +--echo # All GTID events from other domains are printed until the +--echo # --stop-position values are hit +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --stop-position=1-3-4 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --stop-position=1-3-4 | $MYSQL +if ($test3_t2_good_checksum != `CHECKSUM TABLE t2`) +{ + die $data_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't1'`) +{ + die "t1 should not exist as binlog replay should exclude domain 0 from stop position"; +} +DROP TABLE t2; + +--echo # +--echo # Test Case 12: +--echo # Scalar and GTID values can be used together for stop or start +--echo # position +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=empty_binlog_pos --stop-position=0-1-2 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --start-position=$empty_binlog_pos --stop-position=0-1-2 | $MYSQL +if ($test2_t1_mid_checksum != `CHECKSUM TABLE t1`) +{ + die $data_inconsistent_err; +} +DROP TABLE t1; +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=t1_final_checksum_pos | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=$t1_final_checksum_pos | $MYSQL +if ($t1_final_checksum != `CHECKSUM TABLE t1`) +{ + die $data_inconsistent_err; +} +if ($t2_mid_checksum != `CHECKSUM TABLE t2`) +{ + die $data_inconsistent_err; +} +DROP TABLE t1; +DROP TABLE t2; + +--echo # +--echo # Test Case 13: +--echo # If the start position is delayed within the binlog, events occurring +--echo # before that position are ignored +CREATE TABLE t1 (a int); +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-1 --stop-position=0-1-2 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --start-position=0-1-1 --stop-position=0-1-2 | $MYSQL +if ($test2_t1_mid_checksum != `CHECKSUM TABLE t1`) +{ + die $data_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't2'`) +{ + die "t2 should not exist as binlog replay should exclude domain 1 from results"; +} +DROP TABLE t1; + +--echo # +--echo # Test Case 14: +--echo # If start position is repeated, the last specification overrides all +--echo # previous ones +CREATE TABLE t1 (a int); +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --start-position=0-1-1 --stop-position=0-1-2 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --start-position=0-1-0 --start-position=0-1-1 --stop-position=0-1-2 | $MYSQL +if ($test2_t1_mid_checksum != `CHECKSUM TABLE t1`) +{ + die $data_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't2'`) +{ + die "t2 should not exist as binlog replay should exclude domain 1 from results"; +} +DROP TABLE t1; + +--echo # +--echo # Test Case 15: +--echo # If stop position is repeated, the last specification overrides all +--echo # previous ones +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-1 --stop-position=0-1-2 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-1 --stop-position=0-1-2 | $MYSQL +if ($test2_t1_mid_checksum != `CHECKSUM TABLE t1`) +{ + die $data_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't2'`) +{ + die "t2 should not exist as binlog replay should exclude domain 1 from results"; +} +DROP TABLE t1; + +--echo # +--echo # Test Case 16: +--echo # Start position with --offset= skips n events after the first +--echo # GTID is found + +# t1 needs to be specified because its creation should be skipped from +# --offset specification +CREATE TABLE t1 (a int); + +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-2 --offset=5 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-2 --offset=5 | $MYSQL +if ($test2_t1_mid_checksum != `CHECKSUM TABLE t1`) +{ + die $data_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't2'`) +{ + die "t2 should not exist as binlog replay should exclude events after GTID 0-1-2"; +} +DROP TABLE t1; + +--echo # +--echo # Test Case 17: +--echo # Start position with --start-datetime= where T occurs after the +--echo # specified GTID results in no events before T +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=1-2-2 --start-datetime="1970-01-21 15:32:24" | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=1-2-2 --start-datetime="1970-01-21 15:32:24" | $MYSQL +if ($t2_mid_checksum != `CHECKSUM TABLE t2`) +{ + die $data_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't1'`) +{ + die "t1 should not exist as binlog replay should exclude events before the given start-datetime"; +} +DROP TABLE t2; + +--echo # +--echo # Test Case 18: +--echo # If --stop-position is specified, domains which are not present +--echo # in its list should be excluded from output +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --stop-position=1-3-4 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --stop-position=1-3-4 | $MYSQL + +if ($test3_t2_good_checksum != `CHECKSUM TABLE t2`) +{ + die $data_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't1'`) +{ + die "t1 should not exist as binlog replay should exclude domain 0 from stop position"; +} +DROP TABLE t2; + +--echo # +--echo # Test Case 19: +--echo # If the start and stop GTIDs in any domain are equal, the domain +--echo # should not have any output +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-2 --stop-position=0-1-2 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --start-position=0-1-2 --stop-position=0-1-2 | $MYSQL + +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't1'`) +{ + die "t1 should not exist as binlog replay should exclude domain 0 from stop position"; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't2'`) +{ + die "t2 should not exist as binlog replay should exclude domain 1 from stop position"; +} + +--echo # +--echo # Test Case 20: +--echo # If --start-position and --stop-position have different domain ids, +--echo # only events from GTIDs in the --stop-position list are output +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=1-2-2 --stop-position=0-1-3 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM --start-position=1-2-2 --stop-position=0-1-3 | $MYSQL +if ($t1_final_checksum != `CHECKSUM TABLE t1`) +{ + die $data_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't2'`) +{ + die "t2 should not exist as binlog replay should exclude domain 1 from stop position"; +} +DROP TABLE t1; + +--echo # +--echo # Test Case 21: +--echo # Successive binary logs (e.g. logs with previous logs that have been +--echo # purged) will write events when the --start-position matches their +--echo # Gtid_list_log_event state +if ($is_remote == 1) +{ + --echo # + --echo # Reset server state + RESET MASTER; + --exec $MYSQL_BINLOG $MYSQLD_DATADIR/$BINLOG_FILE.orig | $MYSQL + FLUSH LOGS; + --exec $MYSQL_BINLOG $MYSQLD_DATADIR/$BINLOG_FILE2.orig | $MYSQL + FLUSH LOGS; + --eval PURGE BINARY LOGS TO "$BINLOG_FILE2" + DROP TABLE t1; + DROP TABLE t2; + DROP TABLE t3; +} +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM2 --start-position=0-1-3,1-2-5 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM2 --start-position=0-1-3,1-2-5 | $MYSQL +if ($t3_final_checksum != `CHECKSUM TABLE t3`) +{ + die $data_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't1'`) +{ + die "t1 should not exist as binlog replay should not have any events from unspecified binlog file"; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't2'`) +{ + die "t2 should not exist as binlog replay should not have any events from unspecified binlog file"; +} +DROP TABLE t3; + +--echo # +--echo # Test Case 22: +--echo # Successive binary logs can be called with --stop-position and +--echo # without --start-position +if ($is_remote == 1) +{ + --echo # + --echo # Reset server state + RESET MASTER; + --exec $MYSQL_BINLOG $MYSQLD_DATADIR/$BINLOG_FILE.orig | $MYSQL + FLUSH LOGS; + --exec $MYSQL_BINLOG $MYSQLD_DATADIR/$BINLOG_FILE2.orig | $MYSQL + FLUSH LOGS; + --eval PURGE BINARY LOGS TO "$BINLOG_FILE2" + DROP TABLE t1; + DROP TABLE t2; + DROP TABLE t3; +} +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM2 --stop-position=0-1-4 | MYSQL +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM2 --stop-position=0-1-4 | $MYSQL +if (`SELECT COUNT(*) FROM test.t3`) +{ + die $data_inconsistent_err; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't1'`) +{ + die "t1 should not exist as binlog replay should not have any events from unspecified binlog file"; +} +if (`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'test' AND table_name = 't2'`) +{ + die "t2 should not exist as binlog replay should not have any events from unspecified binlog file"; +} +DROP TABLE t3; + + +if ($is_remote == 1) +{ + --echo # + --echo # Remote-only setup for error cases + RESET MASTER; + --exec $MYSQL_BINLOG $MYSQLD_DATADIR/$BINLOG_FILE.orig | $MYSQL + FLUSH LOGS; + --exec $MYSQL_BINLOG $MYSQLD_DATADIR/$BINLOG_FILE2.orig | $MYSQL + FLUSH LOGS; + --eval PURGE BINARY LOGS TO "$BINLOG_FILE2" +} + +--echo # At the following error cases analysis +--echo # note incompatible --start-position with the value of +--echo # Gtid list event of the 2nd binlog file printed above. + +--let err_out_= $MYSQLTEST_VARDIR/tmp/err.out +--let SEARCH_FILE=$err_out_ + +--echo # +--echo # Error Case 1: +--echo # A GTID --start-position that does not mention all domains that make +--echo # up the binary log state should error +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM2 --start-position=0-1-3 > tmp_out_ 2> err_out_ +--error 1 +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM2 --start-position=0-1-3 > $tmp_out_ 2> $err_out_ +if ($is_remote == 1) +{ + --let SEARCH_PATTERN=ERROR: Got error reading packet from server +} +if ($is_remote == 0) +{ + --let SEARCH_PATTERN=ERROR: Starting GTID position list does not specify an initial value +} +--source include/search_pattern_in_file.inc +--remove_file $err_out_ + +--echo # +--echo # Error Case 2: +--echo # A GTID --start-position with any sequence numbers which occur before +--echo # the binary log state should result in error +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM2 --start-position=0-1-2,1-2-5 > tmp_out_ 2> err_out_ +--error 1 +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM2 --start-position=0-1-2,1-2-5 > $tmp_out_ 2> $err_out_ +if ($is_remote == 1) +{ + --let SEARCH_PATTERN=ERROR: Got error reading packet from server +} +if ($is_remote == 0) +{ + --let SEARCH_PATTERN=ERROR: Binary logs are missing data for domain 0 +} +--source include/search_pattern_in_file.inc +--remove_file $err_out_ + +--echo # +--echo # Error Case 3: +--echo # A GTID --start-position with any sequence numbers that are not +--echo # eventually processed results in error +--echo # MYSQL_BINLOG BINLOG_FILE_PARAM2 --start-position=0-1-8,1-2-6 > tmp_out_ 2> err_out_ +--error 1 +--exec $MYSQL_BINLOG $BINLOG_FILE_PARAM2 --start-position=0-1-8,1-2-6 > $tmp_out_ 2> $err_out_ +if ($is_remote == 1) +{ + --let SEARCH_PATTERN=ERROR: Got error reading packet from server +} +if ($is_remote == 0) +{ + --let SEARCH_PATTERN=ERROR: Binary logs never reached expected GTID state +} +--source include/search_pattern_in_file.inc +--remove_file $err_out_ + +if ($is_remote == 1) +{ + --echo # + --echo # Remote-only cleanup from error cases + DROP TABLE t1; + DROP TABLE t2; + DROP TABLE t3; + RESET MASTER; +} + +--remove_file $MYSQLD_DATADIR/$BINLOG_FILE.orig +--remove_file $MYSQLD_DATADIR/$BINLOG_FILE2.orig +--remove_file $tmp_out_ diff --git a/mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_strict_mode.result b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_strict_mode.result new file mode 100644 index 00000000000..668c070b5ea --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_strict_mode.result @@ -0,0 +1,925 @@ +############################### +# Test Setup +############################### +RESET MASTER; +#################################################### +# Test Case Group 1 +# +# Tests with --gtid-strict-mode should error and +# immediately quit with error on out of order GTIDs +#################################################### +# +# Test Case 1: +# Sequential sequence numbers results in no errors or warnings +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +INSERT INTO t1 values (3); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --gtid-strict-mode 2> log_error_ > OUT_FILE +NOT FOUND /ERROR/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 2: +# A skipped sequence number results in no errors or warnings if all +# numbers are monotonic (i.e. gaps in sequence number are allowed +# provided they never decrease) +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 5; +INSERT INTO t1 values (3); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --gtid-strict-mode 2> log_error_ > OUT_FILE +NOT FOUND /ERROR/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 3: +# A sequence number lower than the last processed value results in a +# warning or error +CREATE TABLE t1 (a int); +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 1; +INSERT INTO t1 values (1); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --gtid-strict-mode 2> log_error_ > OUT_FILE +FOUND 1 /ERROR: Found out of order GTID/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 4: +# Skipping a GTID and later receiving it results in a warning or error +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (3); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (4); +INSERT INTO t1 values (5); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --gtid-strict-mode 2> log_error_ > OUT_FILE +FOUND 1 /ERROR: Found out of order GTID/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 5: +# Repeat sequence numbers produce a warning +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (3); +INSERT INTO t1 values (4); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --gtid-strict-mode 2> log_error_ > OUT_FILE +FOUND 1 /ERROR: Found out of order GTID/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 6: +# Warnings from different domains are all displayed +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (3); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (4); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t2 values (2); +SET @@session.gtid_seq_no= 2; +INSERT INTO t2 values (3); +SET @@session.gtid_seq_no= 4; +INSERT INTO t2 values (4); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --gtid-strict-mode 2> log_error_ > OUT_FILE +FOUND 1 /ERROR: Found out of order GTID/ in out.err +DROP TABLE t1; +DROP TABLE t2; +RESET MASTER; +# +# Test Case 7: +# A decreasing seq_no before a start-position is ignored +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +SET @@session.gtid_seq_no= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (1); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (4); +INSERT INTO t1 values (3); +INSERT INTO t1 values (5); +FLUSH LOGS; +# GTID-based start-position +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-4 --gtid-strict-mode > log_error_ > OUT_FILE +NOT FOUND /ERROR/ in out.err +# Position-based start-position +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=start_binlog_pos --gtid-strict-mode 2> log_error_ > OUT_FILE +NOT FOUND /ERROR/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 8: +# A decreasing seq_no inside of a --start/--stop position window is +# displayed +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 5; +INSERT INTO t1 values (4); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (3); +SET @@session.gtid_seq_no= 6; +INSERT INTO t1 values (5); +FLUSH LOGS; +# GTID-based window +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 --stop-position=0-1-6 --gtid-strict-mode > log_error_ > OUT_FILE +FOUND 1 /ERROR: Found out of order GTID/ in out.err +# Position-based window +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=start_binlog_pos --stop-position=stop_binlog_pos --gtid-strict-mode 2> log_error_ > OUT_FILE +FOUND 1 /ERROR: Found out of order GTID/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 9: +# Error if --stop-position is not greater than or equal to +# --start-position +# +# Note: Error is only displayed in strict mode, -vvv has no effect here +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-2 --stop-position=0-1-1 --gtid-strict-mode 2> log_error_ > OUT_FILE +FOUND 1 /ERROR: Queried GTID range is invalid in strict mode/ in out.err +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-2,1-2-1 --stop-position=0-1-1,1-2-2 --gtid-strict-mode 2> log_error_ > OUT_FILE +FOUND 1 /ERROR: Queried GTID range is invalid in strict mode/ in out.err +RESET MASTER; +# +# Test Case 10: +# Strict mode warnings should be independent of --offset option +# specification +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +SET @@session.gtid_seq_no= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (1); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (3); +INSERT INTO t1 values (4); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 --offset=8 --gtid-strict-mode 2> log_error_ > OUT_FILE +FOUND 1 /ERROR: Found out of order GTID/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 11: +# Strict mode warnings should be independent of --start-timestamp +# option specification +set @a=UNIX_TIMESTAMP("1970-01-21 15:32:22"); +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +SET @@session.gtid_seq_no= 1; +SET timestamp=@a; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +SET timestamp=@a+1; +INSERT INTO t1 values (1); +SET @@session.gtid_seq_no= 2; +SET timestamp=@a+2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 4; +SET timestamp=@a+3; +INSERT INTO t1 values (3); +INSERT INTO t1 values (4); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 --start-datetime="1970-01-21 15:32:24" --gtid-strict-mode 2> log_error_ > OUT_FILE +FOUND 1 /ERROR: Found out of order GTID/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 12: +# Specifying multiple binary logs with a log-position start should +# skip GTID state verification +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +INSERT INTO t2 values (1); +FLUSH LOGS; +INSERT INTO t2 values (2); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE1 MYSQLD_DATADIR/BINLOG_FILE2 --start-position=b2_start_pos --gtid-strict-mode 2> log_error_ > OUT_FILE +DROP TABLE t1; +DROP TABLE t2; +# +# Test Case 13: +# If multiple binary logs should be specified but a middle log is +# missing, we should detect that and warn when using -vvv +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +FLUSH LOGS; +INSERT INTO t1 values (1); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +FLUSH LOGS; +SET @@session.gtid_domain_id= 2; +SET @@session.server_id= 3; +CREATE TABLE t3 (a int); +FLUSH LOGS; +# +# GLLE from each log for state reference +show binlog events in 'master-bin.000001' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Gtid_list 1 # [] +show binlog events in 'master-bin.000002' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000002 # Gtid_list 1 # [0-1-1] +show binlog events in 'master-bin.000003' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000003 # Gtid_list 1 # [1-2-1,0-1-2] +# MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE1 MYSQLD_DATADIR/BINLOG_FILE3 --gtid-strict-mode 2> log_error_ > OUT_FILE +# We should have two warnings about missing data from domains 0 and 1 if +# -vvv is specified +NOT FOUND /WARNING: Binary logs are missing data for domain 0[^\n]+the last seen event was/ in out.err +NOT FOUND /WARNING: Binary logs are missing data for domain 1[^\n]+neither the starting GTID position list nor any processed events/ in out.err +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +RESET MASTER; +# +# Test Case 14: +# If a --stop-position GTID occurs before the first specified binlog's +# GLLE, error +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +FLUSH LOGS; +INSERT INTO t1 values (3); +FLUSH LOGS; +# +# GLLE from each log for state reference +show binlog events in 'master-bin.000001' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Gtid_list 1 # [] +show binlog events in 'master-bin.000002' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000002 # Gtid_list 1 # [0-1-3] +# MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE2 --gtid-strict-mode --stop-position=0-1-2 2> log_error_ > OUT_FILE +FOUND 1 /ERROR: --stop-position GTID/ in out.err +DROP TABLE t1; +#################################################### +# Test Case Group 2 +# +# Test cases with --skip-gtid-strict-mode -vvv +# should not quit early or with error when +# encountering out of order GTIDs; however should +# produce warnings after binlog processing +#################################################### +# +# Test Case 1: +# Sequential sequence numbers results in no errors or warnings +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +INSERT INTO t1 values (3); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --skip-gtid-strict-mode -vvv 2> log_error_ > OUT_FILE +NOT FOUND /WARNING/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 2: +# A skipped sequence number results in no errors or warnings if all +# numbers are monotonic (i.e. gaps in sequence number are allowed +# provided they never decrease) +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 5; +INSERT INTO t1 values (3); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --skip-gtid-strict-mode -vvv 2> log_error_ > OUT_FILE +NOT FOUND /WARNING/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 3: +# A sequence number lower than the last processed value results in a +# warning or error +CREATE TABLE t1 (a int); +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 1; +INSERT INTO t1 values (1); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --skip-gtid-strict-mode -vvv 2> log_error_ > OUT_FILE +FOUND 1 /WARNING: Found out of order GTID/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 4: +# Skipping a GTID and later receiving it results in a warning or error +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (3); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (4); +INSERT INTO t1 values (5); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --skip-gtid-strict-mode -vvv 2> log_error_ > OUT_FILE +FOUND 1 /WARNING: Found out of order GTID/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 5: +# Repeat sequence numbers produce a warning +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (3); +INSERT INTO t1 values (4); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --skip-gtid-strict-mode -vvv 2> log_error_ > OUT_FILE +FOUND 1 /WARNING: Found out of order GTID/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 6: +# Warnings from different domains are all displayed +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (3); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (4); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t2 values (2); +SET @@session.gtid_seq_no= 2; +INSERT INTO t2 values (3); +SET @@session.gtid_seq_no= 4; +INSERT INTO t2 values (4); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --skip-gtid-strict-mode -vvv 2> log_error_ > OUT_FILE +FOUND 2 /WARNING: Found out of order GTID/ in out.err +DROP TABLE t1; +DROP TABLE t2; +RESET MASTER; +# +# Test Case 7: +# A decreasing seq_no before a start-position is ignored +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +SET @@session.gtid_seq_no= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (1); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (4); +INSERT INTO t1 values (3); +INSERT INTO t1 values (5); +FLUSH LOGS; +# GTID-based start-position +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-4 --skip-gtid-strict-mode -vvv > log_error_ > OUT_FILE +NOT FOUND /WARNING/ in out.err +# Position-based start-position +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=start_binlog_pos --skip-gtid-strict-mode -vvv 2> log_error_ > OUT_FILE +NOT FOUND /WARNING/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 8: +# A decreasing seq_no inside of a --start/--stop position window is +# displayed +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 5; +INSERT INTO t1 values (4); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (3); +SET @@session.gtid_seq_no= 6; +INSERT INTO t1 values (5); +FLUSH LOGS; +# GTID-based window +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 --stop-position=0-1-6 --skip-gtid-strict-mode -vvv > log_error_ > OUT_FILE +FOUND 1 /WARNING: Found out of order GTID/ in out.err +# Position-based window +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=start_binlog_pos --stop-position=stop_binlog_pos --skip-gtid-strict-mode -vvv 2> log_error_ > OUT_FILE +FOUND 1 /WARNING: Found out of order GTID/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 9: +# Error if --stop-position is not greater than or equal to +# --start-position +# +# Note: Error is only displayed in strict mode, -vvv has no effect here +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-2 --stop-position=0-1-1 --skip-gtid-strict-mode -vvv 2> log_error_ > OUT_FILE +NOT FOUND /ERROR: Queried GTID range is invalid in strict mode/ in out.err +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-2,1-2-1 --stop-position=0-1-1,1-2-2 --skip-gtid-strict-mode -vvv 2> log_error_ > OUT_FILE +NOT FOUND /ERROR: Queried GTID range is invalid in strict mode/ in out.err +RESET MASTER; +# +# Test Case 10: +# Strict mode warnings should be independent of --offset option +# specification +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +SET @@session.gtid_seq_no= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (1); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (3); +INSERT INTO t1 values (4); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 --offset=8 --skip-gtid-strict-mode -vvv 2> log_error_ > OUT_FILE +FOUND 1 /WARNING: Found out of order GTID/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 11: +# Strict mode warnings should be independent of --start-timestamp +# option specification +set @a=UNIX_TIMESTAMP("1970-01-21 15:32:22"); +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +SET @@session.gtid_seq_no= 1; +SET timestamp=@a; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +SET timestamp=@a+1; +INSERT INTO t1 values (1); +SET @@session.gtid_seq_no= 2; +SET timestamp=@a+2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 4; +SET timestamp=@a+3; +INSERT INTO t1 values (3); +INSERT INTO t1 values (4); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 --start-datetime="1970-01-21 15:32:24" --skip-gtid-strict-mode -vvv 2> log_error_ > OUT_FILE +FOUND 1 /WARNING: Found out of order GTID/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 12: +# Specifying multiple binary logs with a log-position start should +# skip GTID state verification +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +INSERT INTO t2 values (1); +FLUSH LOGS; +INSERT INTO t2 values (2); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE1 MYSQLD_DATADIR/BINLOG_FILE2 --start-position=b2_start_pos --skip-gtid-strict-mode -vvv 2> log_error_ > OUT_FILE +DROP TABLE t1; +DROP TABLE t2; +# +# Test Case 13: +# If multiple binary logs should be specified but a middle log is +# missing, we should detect that and warn when using -vvv +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +FLUSH LOGS; +INSERT INTO t1 values (1); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +FLUSH LOGS; +SET @@session.gtid_domain_id= 2; +SET @@session.server_id= 3; +CREATE TABLE t3 (a int); +FLUSH LOGS; +# +# GLLE from each log for state reference +show binlog events in 'master-bin.000001' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Gtid_list 1 # [] +show binlog events in 'master-bin.000002' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000002 # Gtid_list 1 # [0-1-1] +show binlog events in 'master-bin.000003' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000003 # Gtid_list 1 # [1-2-1,0-1-2] +# MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE1 MYSQLD_DATADIR/BINLOG_FILE3 --skip-gtid-strict-mode -vvv 2> log_error_ > OUT_FILE +# We should have two warnings about missing data from domains 0 and 1 if +# -vvv is specified +FOUND 1 /WARNING: Binary logs are missing data for domain 0[^\n]+the last seen event was/ in out.err +FOUND 1 /WARNING: Binary logs are missing data for domain 1[^\n]+neither the starting GTID position list nor any processed events/ in out.err +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +RESET MASTER; +# +# Test Case 14: +# If a --stop-position GTID occurs before the first specified binlog's +# GLLE, error +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +FLUSH LOGS; +INSERT INTO t1 values (3); +FLUSH LOGS; +# +# GLLE from each log for state reference +show binlog events in 'master-bin.000001' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Gtid_list 1 # [] +show binlog events in 'master-bin.000002' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000002 # Gtid_list 1 # [0-1-3] +# MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE2 --skip-gtid-strict-mode -vvv --stop-position=0-1-2 2> log_error_ > OUT_FILE +FOUND 1 /ERROR: --stop-position GTID/ in out.err +DROP TABLE t1; +#################################################### +# Test Case Group 3 +# +# Run test cases with --skip-gtid-strict-mode should +# neither produce errors nor warnings when +# encountering out of order GTIDs +#################################################### +# +# Test Case 1: +# Sequential sequence numbers results in no errors or warnings +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +INSERT INTO t1 values (3); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --skip-gtid-strict-mode 2> log_error_ > OUT_FILE +NOT FOUND /(ERROR|WARNING)/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 2: +# A skipped sequence number results in no errors or warnings if all +# numbers are monotonic (i.e. gaps in sequence number are allowed +# provided they never decrease) +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 5; +INSERT INTO t1 values (3); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --skip-gtid-strict-mode 2> log_error_ > OUT_FILE +NOT FOUND /(ERROR|WARNING)/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 3: +# A sequence number lower than the last processed value results in a +# warning or error +CREATE TABLE t1 (a int); +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 1; +INSERT INTO t1 values (1); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --skip-gtid-strict-mode 2> log_error_ > OUT_FILE +NOT FOUND /(ERROR|WARNING): Found out of order GTID/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 4: +# Skipping a GTID and later receiving it results in a warning or error +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (3); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (4); +INSERT INTO t1 values (5); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --skip-gtid-strict-mode 2> log_error_ > OUT_FILE +NOT FOUND /(ERROR|WARNING): Found out of order GTID/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 5: +# Repeat sequence numbers produce a warning +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (3); +INSERT INTO t1 values (4); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --skip-gtid-strict-mode 2> log_error_ > OUT_FILE +NOT FOUND /(ERROR|WARNING): Found out of order GTID/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 6: +# Warnings from different domains are all displayed +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (3); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (4); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t2 values (2); +SET @@session.gtid_seq_no= 2; +INSERT INTO t2 values (3); +SET @@session.gtid_seq_no= 4; +INSERT INTO t2 values (4); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --skip-gtid-strict-mode 2> log_error_ > OUT_FILE +NOT FOUND /(ERROR|WARNING): Found out of order GTID/ in out.err +DROP TABLE t1; +DROP TABLE t2; +RESET MASTER; +# +# Test Case 7: +# A decreasing seq_no before a start-position is ignored +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +SET @@session.gtid_seq_no= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (1); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (4); +INSERT INTO t1 values (3); +INSERT INTO t1 values (5); +FLUSH LOGS; +# GTID-based start-position +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-4 --skip-gtid-strict-mode > log_error_ > OUT_FILE +NOT FOUND /(ERROR|WARNING)/ in out.err +# Position-based start-position +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=start_binlog_pos --skip-gtid-strict-mode 2> log_error_ > OUT_FILE +NOT FOUND /(ERROR|WARNING)/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 8: +# A decreasing seq_no inside of a --start/--stop position window is +# displayed +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 5; +INSERT INTO t1 values (4); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (3); +SET @@session.gtid_seq_no= 6; +INSERT INTO t1 values (5); +FLUSH LOGS; +# GTID-based window +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 --stop-position=0-1-6 --skip-gtid-strict-mode > log_error_ > OUT_FILE +NOT FOUND /(ERROR|WARNING): Found out of order GTID/ in out.err +# Position-based window +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=start_binlog_pos --stop-position=stop_binlog_pos --skip-gtid-strict-mode 2> log_error_ > OUT_FILE +NOT FOUND /(ERROR|WARNING): Found out of order GTID/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 9: +# Error if --stop-position is not greater than or equal to +# --start-position +# +# Note: Error is only displayed in strict mode, -vvv has no effect here +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-2 --stop-position=0-1-1 --skip-gtid-strict-mode 2> log_error_ > OUT_FILE +NOT FOUND /ERROR: Queried GTID range is invalid in strict mode/ in out.err +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-2,1-2-1 --stop-position=0-1-1,1-2-2 --skip-gtid-strict-mode 2> log_error_ > OUT_FILE +NOT FOUND /ERROR: Queried GTID range is invalid in strict mode/ in out.err +RESET MASTER; +# +# Test Case 10: +# Strict mode warnings should be independent of --offset option +# specification +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +SET @@session.gtid_seq_no= 1; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +INSERT INTO t1 values (1); +SET @@session.gtid_seq_no= 2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 4; +INSERT INTO t1 values (3); +INSERT INTO t1 values (4); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 --offset=8 --skip-gtid-strict-mode 2> log_error_ > OUT_FILE +NOT FOUND /(ERROR|WARNING): Found out of order GTID/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 11: +# Strict mode warnings should be independent of --start-timestamp +# option specification +set @a=UNIX_TIMESTAMP("1970-01-21 15:32:22"); +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +SET @@session.gtid_seq_no= 1; +SET timestamp=@a; +CREATE TABLE t1 (a int); +SET @@session.gtid_seq_no= 3; +SET timestamp=@a+1; +INSERT INTO t1 values (1); +SET @@session.gtid_seq_no= 2; +SET timestamp=@a+2; +INSERT INTO t1 values (2); +SET @@session.gtid_seq_no= 4; +SET timestamp=@a+3; +INSERT INTO t1 values (3); +INSERT INTO t1 values (4); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1 --start-datetime="1970-01-21 15:32:24" --skip-gtid-strict-mode 2> log_error_ > OUT_FILE +NOT FOUND /(ERROR|WARNING): Found out of order GTID/ in out.err +DROP TABLE t1; +RESET MASTER; +# +# Test Case 12: +# Specifying multiple binary logs with a log-position start should +# skip GTID state verification +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +INSERT INTO t2 values (1); +FLUSH LOGS; +INSERT INTO t2 values (2); +FLUSH LOGS; +# MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE1 MYSQLD_DATADIR/BINLOG_FILE2 --start-position=b2_start_pos --skip-gtid-strict-mode 2> log_error_ > OUT_FILE +DROP TABLE t1; +DROP TABLE t2; +# +# Test Case 13: +# If multiple binary logs should be specified but a middle log is +# missing, we should detect that and warn when using -vvv +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +FLUSH LOGS; +INSERT INTO t1 values (1); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +CREATE TABLE t2 (a int); +FLUSH LOGS; +SET @@session.gtid_domain_id= 2; +SET @@session.server_id= 3; +CREATE TABLE t3 (a int); +FLUSH LOGS; +# +# GLLE from each log for state reference +show binlog events in 'master-bin.000001' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Gtid_list 1 # [] +show binlog events in 'master-bin.000002' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000002 # Gtid_list 1 # [0-1-1] +show binlog events in 'master-bin.000003' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000003 # Gtid_list 1 # [1-2-1,0-1-2] +# MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE1 MYSQLD_DATADIR/BINLOG_FILE3 --skip-gtid-strict-mode 2> log_error_ > OUT_FILE +# We should have two warnings about missing data from domains 0 and 1 if +# -vvv is specified +NOT FOUND /WARNING: Binary logs are missing data for domain 0[^\n]+the last seen event was/ in out.err +NOT FOUND /WARNING: Binary logs are missing data for domain 1[^\n]+neither the starting GTID position list nor any processed events/ in out.err +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +RESET MASTER; +# +# Test Case 14: +# If a --stop-position GTID occurs before the first specified binlog's +# GLLE, error +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +INSERT INTO t1 values (1); +INSERT INTO t1 values (2); +FLUSH LOGS; +INSERT INTO t1 values (3); +FLUSH LOGS; +# +# GLLE from each log for state reference +show binlog events in 'master-bin.000001' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000001 # Gtid_list 1 # [] +show binlog events in 'master-bin.000002' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000002 # Gtid_list 1 # [0-1-3] +# MYSQL_BINLOG MYSQLD_DATADIR/BINLOG_FILE2 --skip-gtid-strict-mode --stop-position=0-1-2 2> log_error_ > OUT_FILE +FOUND 1 /ERROR: --stop-position GTID/ in out.err +DROP TABLE t1; +############################## +# Cleanup +############################## +SET @@global.gtid_domain_id= 0; +SET @@global.server_id= 1; +End of the tests diff --git a/mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_window.result b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_window.result new file mode 100644 index 00000000000..63af0d20cc6 --- /dev/null +++ b/mysql-test/suite/binlog/r/binlog_mysqlbinlog_gtid_window.result @@ -0,0 +1,468 @@ +############################### +# Test Setup +############################### +RESET MASTER; +###################################### +# Test Group 1 +# Run test cases on local log file +###################################### +set @a=UNIX_TIMESTAMP("1970-01-21 15:32:22"); +SET timestamp=@a; +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +SET timestamp=@a+1; +INSERT INTO t1 values (1), (2); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +SET timestamp=@a+2; +CREATE TABLE t2 (a int); +SET timestamp=@a+3; +INSERT INTO t2 values (1); +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +SET timestamp=@a+4; +INSERT INTO t1 values (3), (4); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +INSERT INTO t2 values (2); +SET @@session.server_id= 3; +INSERT INTO t2 values (3); +SET @@session.server_id= 2; +INSERT INTO t2 values (4); +FLUSH LOGS; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t3 (a int); +INSERT INTO t3 VALUES (1); +FLUSH LOGS; +# Gtid list event of the 2nd binlog file whose content is +# matched against --start-position in the following tests: +show binlog events in 'master-bin.000002' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000002 # Gtid_list 1 # [1-3-4,1-2-5,0-1-3] +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +# +# Test Case 1: +# The end of the binlog file resets the server and domain id of the +# session +SET @@session.gtid_domain_id= 10; +SET @@session.server_id= 20; +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-2 | MYSQL +include/assert.inc [session gtid_domain_id should not change when reading binlog in GTID mode] +include/assert.inc [session server_id should not change when reading binlog in GTID mode] +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +DROP TABLE t1; +# +# Test Case 2: +# Single GTID range specified +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-2 | MYSQL +DROP TABLE t1; +# +# Test Case 3: +# Single GTID range with different server_ids +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=1-2-0 --stop-position=1-3-4 | MYSQL +DROP TABLE t2; +# +# Test Case 4: +# Multiple GTID ranges specified +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0,1-2-0 --stop-position=0-1-3,1-2-3 | MYSQL +DROP TABLE t1; +DROP TABLE t2; +# +# Test Case 5: +# Multiple GTID ranges specified where the domain ids are listed in +# different orders between start/stop position +# MYSQL_BINLOG BINLOG_FILE_PARAM --stop-position=0-1-3,1-2-3 --start-position=1-2-0,0-1-0 | MYSQL +DROP TABLE t1; +DROP TABLE t2; +# +# Test Case 6: +# Only start position specified +CREATE TABLE t1 (a int); +INSERT INTO t1 values (3), (4); +DROP TABLE t1; +CREATE TABLE t1 (a int); +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-2 | MYSQL +DROP TABLE t1; +DROP TABLE t2; +# +# Test Case 7: +# Only stop position specified +# MYSQL_BINLOG BINLOG_FILE_PARAM --stop-position=0-1-2 | MYSQL +DROP TABLE t1; +# +# Test Case 8: +# Seq_no=0 in --start-position includes all events for a domain +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0,1-2-0 | MYSQL +DROP TABLE t1; +DROP TABLE t2; +# +# Test Case 9: +# Seq_no=0 in --stop-position excludes all events for a domain +# MYSQL_BINLOG BINLOG_FILE_PARAM --stop-position=0-1-0,1-2-0 | MYSQL +# +# Test Case 10: +# Output stops for all domain ids when all --stop-position GTID values +# have been hit. +# MYSQL_BINLOG BINLOG_FILE_PARAM --stop-position=0-1-2 | MYSQL +DROP TABLE t1; +# +# Test Case 11: +# All GTID events from other domains are printed until the +# --stop-position values are hit +# MYSQL_BINLOG BINLOG_FILE_PARAM --stop-position=1-3-4 | MYSQL +DROP TABLE t2; +# +# Test Case 12: +# Scalar and GTID values can be used together for stop or start +# position +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=empty_binlog_pos --stop-position=0-1-2 | MYSQL +DROP TABLE t1; +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=t1_final_checksum_pos | MYSQL +DROP TABLE t1; +DROP TABLE t2; +# +# Test Case 13: +# If the start position is delayed within the binlog, events occurring +# before that position are ignored +CREATE TABLE t1 (a int); +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-1 --stop-position=0-1-2 | MYSQL +DROP TABLE t1; +# +# Test Case 14: +# If start position is repeated, the last specification overrides all +# previous ones +CREATE TABLE t1 (a int); +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --start-position=0-1-1 --stop-position=0-1-2 | MYSQL +DROP TABLE t1; +# +# Test Case 15: +# If stop position is repeated, the last specification overrides all +# previous ones +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-1 --stop-position=0-1-2 | MYSQL +DROP TABLE t1; +# +# Test Case 16: +# Start position with --offset= skips n events after the first +# GTID is found +CREATE TABLE t1 (a int); +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-2 --offset=5 | MYSQL +DROP TABLE t1; +# +# Test Case 17: +# Start position with --start-datetime= where T occurs after the +# specified GTID results in no events before T +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=1-2-2 --start-datetime="1970-01-21 15:32:24" | MYSQL +DROP TABLE t2; +# +# Test Case 18: +# If --stop-position is specified, domains which are not present +# in its list should be excluded from output +# MYSQL_BINLOG BINLOG_FILE_PARAM --stop-position=1-3-4 | MYSQL +DROP TABLE t2; +# +# Test Case 19: +# If the start and stop GTIDs in any domain are equal, the domain +# should not have any output +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-2 --stop-position=0-1-2 | MYSQL +# +# Test Case 20: +# If --start-position and --stop-position have different domain ids, +# only events from GTIDs in the --stop-position list are output +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=1-2-2 --stop-position=0-1-3 | MYSQL +DROP TABLE t1; +# +# Test Case 21: +# Successive binary logs (e.g. logs with previous logs that have been +# purged) will write events when the --start-position matches their +# Gtid_list_log_event state +# MYSQL_BINLOG BINLOG_FILE_PARAM2 --start-position=0-1-3,1-2-5 | MYSQL +DROP TABLE t3; +# +# Test Case 22: +# Successive binary logs can be called with --stop-position and +# without --start-position +# MYSQL_BINLOG BINLOG_FILE_PARAM2 --stop-position=0-1-4 | MYSQL +DROP TABLE t3; +# At the following error cases analysis +# note incompatible --start-position with the value of +# Gtid list event of the 2nd binlog file printed above. +# +# Error Case 1: +# A GTID --start-position that does not mention all domains that make +# up the binary log state should error +# MYSQL_BINLOG BINLOG_FILE_PARAM2 --start-position=0-1-3 > tmp_out_ 2> err_out_ +FOUND 2 /ERROR: Starting GTID position list does not specify an initial value/ in err.out +# +# Error Case 2: +# A GTID --start-position with any sequence numbers which occur before +# the binary log state should result in error +# MYSQL_BINLOG BINLOG_FILE_PARAM2 --start-position=0-1-2,1-2-5 > tmp_out_ 2> err_out_ +FOUND 1 /ERROR: Binary logs are missing data for domain 0/ in err.out +# +# Error Case 3: +# A GTID --start-position with any sequence numbers that are not +# eventually processed results in error +# MYSQL_BINLOG BINLOG_FILE_PARAM2 --start-position=0-1-8,1-2-6 > tmp_out_ 2> err_out_ +FOUND 2 /ERROR: Binary logs never reached expected GTID state/ in err.out +###################################### +# Test Group 2 +# Run test cases on remote host +###################################### +set @a=UNIX_TIMESTAMP("1970-01-21 15:32:22"); +SET timestamp=@a; +RESET MASTER; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t1 (a int); +SET timestamp=@a+1; +INSERT INTO t1 values (1), (2); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +SET timestamp=@a+2; +CREATE TABLE t2 (a int); +SET timestamp=@a+3; +INSERT INTO t2 values (1); +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +SET timestamp=@a+4; +INSERT INTO t1 values (3), (4); +SET @@session.gtid_domain_id= 1; +SET @@session.server_id= 2; +INSERT INTO t2 values (2); +SET @@session.server_id= 3; +INSERT INTO t2 values (3); +SET @@session.server_id= 2; +INSERT INTO t2 values (4); +FLUSH LOGS; +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +CREATE TABLE t3 (a int); +INSERT INTO t3 VALUES (1); +FLUSH LOGS; +# Gtid list event of the 2nd binlog file whose content is +# matched against --start-position in the following tests: +show binlog events in 'master-bin.000002' limit 1,1; +Log_name Pos Event_type Server_id End_log_pos Info +master-bin.000002 # Gtid_list 1 # [1-3-4,1-2-5,0-1-3] +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +# +# Test Case 1: +# The end of the binlog file resets the server and domain id of the +# session +SET @@session.gtid_domain_id= 10; +SET @@session.server_id= 20; +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-2 | MYSQL +include/assert.inc [session gtid_domain_id should not change when reading binlog in GTID mode] +include/assert.inc [session server_id should not change when reading binlog in GTID mode] +SET @@session.gtid_domain_id= 0; +SET @@session.server_id= 1; +DROP TABLE t1; +# +# Test Case 2: +# Single GTID range specified +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-2 | MYSQL +DROP TABLE t1; +# +# Test Case 3: +# Single GTID range with different server_ids +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=1-2-0 --stop-position=1-3-4 | MYSQL +DROP TABLE t2; +# +# Test Case 4: +# Multiple GTID ranges specified +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0,1-2-0 --stop-position=0-1-3,1-2-3 | MYSQL +DROP TABLE t1; +DROP TABLE t2; +# +# Test Case 5: +# Multiple GTID ranges specified where the domain ids are listed in +# different orders between start/stop position +# MYSQL_BINLOG BINLOG_FILE_PARAM --stop-position=0-1-3,1-2-3 --start-position=1-2-0,0-1-0 | MYSQL +DROP TABLE t1; +DROP TABLE t2; +# +# Test Case 6: +# Only start position specified +CREATE TABLE t1 (a int); +INSERT INTO t1 values (3), (4); +DROP TABLE t1; +CREATE TABLE t1 (a int); +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-2 | MYSQL +DROP TABLE t1; +DROP TABLE t2; +# +# Test Case 7: +# Only stop position specified +# MYSQL_BINLOG BINLOG_FILE_PARAM --stop-position=0-1-2 | MYSQL +DROP TABLE t1; +# +# Test Case 8: +# Seq_no=0 in --start-position includes all events for a domain +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0,1-2-0 | MYSQL +DROP TABLE t1; +DROP TABLE t2; +# +# Test Case 9: +# Seq_no=0 in --stop-position excludes all events for a domain +# MYSQL_BINLOG BINLOG_FILE_PARAM --stop-position=0-1-0,1-2-0 | MYSQL +# +# Test Case 10: +# Output stops for all domain ids when all --stop-position GTID values +# have been hit. +# MYSQL_BINLOG BINLOG_FILE_PARAM --stop-position=0-1-2 | MYSQL +DROP TABLE t1; +# +# Test Case 11: +# All GTID events from other domains are printed until the +# --stop-position values are hit +# MYSQL_BINLOG BINLOG_FILE_PARAM --stop-position=1-3-4 | MYSQL +DROP TABLE t2; +# +# Test Case 12: +# Scalar and GTID values can be used together for stop or start +# position +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=empty_binlog_pos --stop-position=0-1-2 | MYSQL +DROP TABLE t1; +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=t1_final_checksum_pos | MYSQL +DROP TABLE t1; +DROP TABLE t2; +# +# Test Case 13: +# If the start position is delayed within the binlog, events occurring +# before that position are ignored +CREATE TABLE t1 (a int); +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-1 --stop-position=0-1-2 | MYSQL +DROP TABLE t1; +# +# Test Case 14: +# If start position is repeated, the last specification overrides all +# previous ones +CREATE TABLE t1 (a int); +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --start-position=0-1-1 --stop-position=0-1-2 | MYSQL +DROP TABLE t1; +# +# Test Case 15: +# If stop position is repeated, the last specification overrides all +# previous ones +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-1 --stop-position=0-1-2 | MYSQL +DROP TABLE t1; +# +# Test Case 16: +# Start position with --offset= skips n events after the first +# GTID is found +CREATE TABLE t1 (a int); +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=0-1-2 --offset=5 | MYSQL +DROP TABLE t1; +# +# Test Case 17: +# Start position with --start-datetime= where T occurs after the +# specified GTID results in no events before T +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-0 --stop-position=1-2-2 --start-datetime="1970-01-21 15:32:24" | MYSQL +DROP TABLE t2; +# +# Test Case 18: +# If --stop-position is specified, domains which are not present +# in its list should be excluded from output +# MYSQL_BINLOG BINLOG_FILE_PARAM --stop-position=1-3-4 | MYSQL +DROP TABLE t2; +# +# Test Case 19: +# If the start and stop GTIDs in any domain are equal, the domain +# should not have any output +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=0-1-2 --stop-position=0-1-2 | MYSQL +# +# Test Case 20: +# If --start-position and --stop-position have different domain ids, +# only events from GTIDs in the --stop-position list are output +# MYSQL_BINLOG BINLOG_FILE_PARAM --start-position=1-2-2 --stop-position=0-1-3 | MYSQL +DROP TABLE t1; +# +# Test Case 21: +# Successive binary logs (e.g. logs with previous logs that have been +# purged) will write events when the --start-position matches their +# Gtid_list_log_event state +# +# Reset server state +RESET MASTER; +FLUSH LOGS; +FLUSH LOGS; +PURGE BINARY LOGS TO "master-bin.000002"; +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +# MYSQL_BINLOG BINLOG_FILE_PARAM2 --start-position=0-1-3,1-2-5 | MYSQL +DROP TABLE t3; +# +# Test Case 22: +# Successive binary logs can be called with --stop-position and +# without --start-position +# +# Reset server state +RESET MASTER; +FLUSH LOGS; +FLUSH LOGS; +PURGE BINARY LOGS TO "master-bin.000002"; +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +# MYSQL_BINLOG BINLOG_FILE_PARAM2 --stop-position=0-1-4 | MYSQL +DROP TABLE t3; +# +# Remote-only setup for error cases +RESET MASTER; +FLUSH LOGS; +FLUSH LOGS; +PURGE BINARY LOGS TO "master-bin.000002"; +# At the following error cases analysis +# note incompatible --start-position with the value of +# Gtid list event of the 2nd binlog file printed above. +# +# Error Case 1: +# A GTID --start-position that does not mention all domains that make +# up the binary log state should error +# MYSQL_BINLOG BINLOG_FILE_PARAM2 --start-position=0-1-3 > tmp_out_ 2> err_out_ +FOUND 1 /ERROR: Got error reading packet from server/ in err.out +# +# Error Case 2: +# A GTID --start-position with any sequence numbers which occur before +# the binary log state should result in error +# MYSQL_BINLOG BINLOG_FILE_PARAM2 --start-position=0-1-2,1-2-5 > tmp_out_ 2> err_out_ +FOUND 1 /ERROR: Got error reading packet from server/ in err.out +# +# Error Case 3: +# A GTID --start-position with any sequence numbers that are not +# eventually processed results in error +# MYSQL_BINLOG BINLOG_FILE_PARAM2 --start-position=0-1-8,1-2-6 > tmp_out_ 2> err_out_ +FOUND 1 /ERROR: Got error reading packet from server/ in err.out +# +# Remote-only cleanup from error cases +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +RESET MASTER; +# +# Error Case 4: +# User provides invalid positions +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=z +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=1- +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=1-2 +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=1-2- +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=-1 +# +# Error Case 5: +# User provides GTID ranges with repeated domain ids +# MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1,0-1-8 --stop-position=0-1-4,0-1-12 +############################## +# Cleanup +############################## +SET @@global.gtid_domain_id= 0; +SET @@global.server_id= 1; +# End of the tests diff --git a/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_strict_mode.test b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_strict_mode.test new file mode 100644 index 00000000000..166826408d9 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_strict_mode.test @@ -0,0 +1,105 @@ +# +# Purpose: +# +# This test ensures that the mariadb-binlog CLI tool properly displays errors +# and warnings for out of order GTIDs. +# +# +# Methodology: +# +# We simulate invalid sequence numberings by manually changing gtid_seq_no to +# differ from its expected linear sequence. Specifically, we test the following +# cases: +# Test Case 1) Sequential sequence numbers results in no warnings +# Test Case 2) A skipped sequence number results in no warnings if all numbers +# are monotonic (i.e. gaps in sequence number are allowed +# provided they never decrease) +# Test Case 3) A sequence number lower than the last processed value results +# in a warning +# Test Case 4) Skipping a GTID and later receiving it results in a warning +# Test Case 5) Repeat sequence numbers result in a warning +# Test Case 6) Warnings from different domains are all displayed +# Test Case 7) A decreasing seq_no before a start-position is ignored +# Test Case 8) A decreasing seq_no inside of a --start/--stop position window +# is displayed +# Test Case 9) Error if --stop-position is not greater than or equal to +# --start-position +# Test Case 10) Strict mode warnings should be independent of --offset option +# specification +# Test Case 11) Strict mode warnings should be independent of +# --start-timestamp option specification +# Test Case 12) Specifying multiple binary logs with a log-position start +# should skip GTID state verification +# Test Case 13) If multiple binary logs should be specified but a middle log +# is missing, we should detect that and warn when using -vvv +# Test Case 14) If a --stop-position GTID occurs before the first specified +# binlog's GLLE, error +# +# Note that test cases are tested under three scenarios: +# 1) --gtid-strict-mode should error and immediately quit with error on out of +# order GTIDs +# 2) --skip-gtid-strict-mode -vvv should not quit early or with error when +# encountering out of order GTIDs; however should produce warnings after +# binlog processing +# 3) --skip-gtid-strict-mode should neither produce errors nor warnings when +# encountering out of order GTIDs +# +# References: +# MDEV-4989: Support for GTID in mysqlbinlog +# + +--source include/have_log_bin.inc + +--echo ############################### +--echo # Test Setup +--echo ############################### + +## Save old state +# +let orig_gtid_domain_id = `select @@session.gtid_domain_id`; +let orig_server_id = `select @@session.server_id`; +RESET MASTER; + +--echo #################################################### +--echo # Test Case Group 1 +--echo # +--echo # Tests with --gtid-strict-mode should error and +--echo # immediately quit with error on out of order GTIDs +--echo #################################################### +--let is_strict_mode= 1 +--let is_verbose= 0 +--let DEFAULT_ERROR_PREFIX=ERROR +--source include/mysqlbinlog_gtid_strict_mode.inc + +--echo #################################################### +--echo # Test Case Group 2 +--echo # +--echo # Test cases with --skip-gtid-strict-mode -vvv +--echo # should not quit early or with error when +--echo # encountering out of order GTIDs; however should +--echo # produce warnings after binlog processing +--echo #################################################### +--let is_strict_mode= 0 +--let is_verbose= 1 +--let DEFAULT_ERROR_PREFIX=WARNING +--source include/mysqlbinlog_gtid_strict_mode.inc + +--echo #################################################### +--echo # Test Case Group 3 +--echo # +--echo # Run test cases with --skip-gtid-strict-mode should +--echo # neither produce errors nor warnings when +--echo # encountering out of order GTIDs +--echo #################################################### +--let is_strict_mode= 0 +--let is_verbose= 0 +--let DEFAULT_ERROR_PREFIX=(ERROR|WARNING) +--source include/mysqlbinlog_gtid_strict_mode.inc + +--echo ############################## +--echo # Cleanup +--echo ############################## +--eval SET @@global.gtid_domain_id= $orig_gtid_domain_id +--eval SET @@global.server_id= $orig_server_id + +--echo End of the tests diff --git a/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_window.test b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_window.test new file mode 100644 index 00000000000..3efb8d025b9 --- /dev/null +++ b/mysql-test/suite/binlog/t/binlog_mysqlbinlog_gtid_window.test @@ -0,0 +1,151 @@ +# +# Purpose: +# +# This test ensures that the mariadb-binlog CLI tool can filter log events +# using GTID ranges. More specifically, this test ensures the following +# capabilities: +# 1) GTIDs can be used to filter results on local binlog files +# 2) GTIDs can be used to filter results from remote servers +# 3) For a given GTID range, its start-position is exclusive and its +# stop-position is inclusive. This allows users to receive events strictly +# after what they already have. +# 4) After the events have been written, the session server id and domain id +# are reset to their former values +# +# +# Methodology: +# +# This test validates the expected capabilities using the following test cases +# on both a local binlog file and remote server for all binlog formats. +# Test Case 1) The end of the binlog file resets the server and domain id of +# the session +# Test Case 2) Single GTID range specified +# Test Case 3) Single GTID range with different server_ids +# Test Case 4) Multiple GTID ranges specified +# Test Case 5) Multiple GTID ranges specified where the domain ids are +# listed in different orders between start/stop position +# Test Case 6) Only start position specified +# Test Case 7) Only stop position specified +# Test Case 8) Seq_no=0 in --start-position includes all events for a domain +# Test Case 9) Seq_no=0 in --stop-position excludes all events for a domain +# Test Case 10) Output stops for all domain ids when all --stop-position GTID +# values have been hit. +# Test Case 11) All GTID events from other domains are printed until +# the --stop-position values are hit +# Test Case 12) Scalar and GTID values can be used together for stop or start +# position +# Test Case 13) If the start position is delayed within the binlog, events +# occurring before that position are ignored +# Test Case 14) If start position is repeated, the last specification +# overrides all previous ones +# Test Case 15) If stop position is repeated, the last specification +# overrides all previous ones +# Test Case 16) Start position with --offset= skips n events after the +# first GTID is found +# Test Case 17) Start position with --start-datetime= where T occurs +# after the specified GTID results in no events before T +# Test Case 18) If --stop-position is specified, domains which are not present +# in its list should be excluded from output +# Test Case 19) If the start and stop GTIDs in any domain are equal, the +# domain should not have any output +# Test Case 20) If --start-position and --stop-position have different domain +# ids, only events from GTIDs in the --stop-position list are +# output +# Test Case 21) Successive binary logs (e.g. logs with previous logs that +# have been purged) will write events when the --start-position +# matches their Gtid_list_log_event state +# Test Case 22) Successive binary logs can be called with --stop-position and +# without --start-position +# +# To validate for data consistency, each test case compares a checksum of +# correct data against a variant created after replaying the binlog using +# --(start|stop)-position. If the checksums are identical, the test passes. +# If the checksums differ, data has been changed and the test fails. +# +# Additionally, this test validates the following error scenarios: +# Error Case 1) A GTID --start-position that does not mention all domains +# that make up the binary log state should error +# Error Case 2) A GTID --start-position with any sequence numbers which +# occur before the binary log state should result in error +# Error Case 3) A GTID --start-position with any sequence numbers that are not +# eventually processed results in error +# Error Case 4) User provides invalid positions +# Error Case 5) User provides GTID ranges with repeated domain ids +# +# References: +# MDEV-4989: Support for GTID in mysqlbinlog +# + +--source include/have_log_bin.inc + +--echo ############################### +--echo # Test Setup +--echo ############################### + +## Save old state +# +let orig_gtid_domain_id = `select @@session.gtid_domain_id`; +let orig_server_id = `select @@session.server_id`; +RESET MASTER; + + +--echo ###################################### +--echo # Test Group 1 +--echo # Run test cases on local log file +--echo ###################################### +--let is_remote= 0 +--source include/mysqlbinlog_gtid_window_test_cases.inc + +--echo ###################################### +--echo # Test Group 2 +--echo # Run test cases on remote host +--echo ###################################### +--let is_remote= 1 +--source include/mysqlbinlog_gtid_window_test_cases.inc + +# Note that error cases 1-3 are in mysqlbinlog_gtid_window_test_cases.inc +# because we validate for error consistency of GTID state between +# mariadb-binlog working on local files and receiving errors from a server + +--let err_out_= $MYSQLTEST_VARDIR/tmp/err.out +--let tmp_out_= $MYSQLTEST_VARDIR/tmp/std.out + +--let $MYSQLD_DATADIR=`select @@datadir` +--echo # +--echo # Error Case 4: +--echo # User provides invalid positions +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=z +--error 1 +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=z + +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=1- +--error 1 +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=1- + +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=1-2 +--error 1 +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=1-2 + +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=1-2- +--error 1 +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=1-2- + +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=-1 +--error 1 +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=-1 + +--echo # +--echo # Error Case 5: +--echo # User provides GTID ranges with repeated domain ids +--echo # MYSQL_BINLOG MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1,0-1-8 --stop-position=0-1-4,0-1-12 +--error 1 +--exec $MYSQL_BINLOG $MYSQLD_DATADIR/master-bin.000001 --start-position=0-1-1,0-1-8 --stop-position=0-1-4,0-1-12 + + +--echo ############################## +--echo # Cleanup +--echo ############################## +--eval SET @@global.gtid_domain_id= $orig_gtid_domain_id +--eval SET @@global.server_id= $orig_server_id + +--echo # End of the tests diff --git a/mysql-test/suite/rpl/t/rpl_blackhole_row_annotate.test b/mysql-test/suite/rpl/t/rpl_blackhole_row_annotate.test index 07fc1e558cb..f7aefd625a1 100644 --- a/mysql-test/suite/rpl/t/rpl_blackhole_row_annotate.test +++ b/mysql-test/suite/rpl/t/rpl_blackhole_row_annotate.test @@ -44,6 +44,6 @@ FLUSH LOGS; let $MYSQLD_DATADIR= `select @@datadir`; --replace_regex /server id [0-9]*/server id #/ /server v [^ ]*/server v #.##.##/ /exec_time=[0-9]*/exec_time=#/ /thread_id=[0-9]*/thread_id=#/ /table id [0-9]*/table id #/ /mapped to number [0-9]*/mapped to number #/ /end_log_pos [0-9]*/end_log_pos #/ /# at [0-9]*/# at #/ /CRC32 0x[0-9a-f]*/CRC32 XXX/ /xid=\d*/xid=/ ---exec $MYSQL_BINLOG --base64-output=decode-rows $MYSQLD_DATADIR/slave-bin.000001 +--exec $MYSQL_BINLOG --base64-output=decode-rows --skip-gtid-strict-mode $MYSQLD_DATADIR/slave-bin.000001 source include/rpl_end.inc; diff --git a/sql/log_event.h b/sql/log_event.h index 3adc7a26d93..db77be2d99b 100644 --- a/sql/log_event.h +++ b/sql/log_event.h @@ -920,6 +920,20 @@ typedef struct st_print_event_info IO_CACHE review_sql_cache; #endif FILE *file; + + + + /* + Used to include the events within a GTID start/stop boundary + */ + my_bool m_is_event_group_active; + + /* + Tracks whether or not output events must be explicitly activated in order + to be printed + */ + my_bool m_is_event_group_filtering_enabled; + st_print_event_info(); ~st_print_event_info() { @@ -942,6 +956,40 @@ typedef struct st_print_event_info copy_event_cache_to_file_and_reinit(&body_cache, file); fflush(file); } + + /* + Notify that all events part of the current group should be printed + */ + void activate_current_event_group() + { + m_is_event_group_active= TRUE; + } + void deactivate_current_event_group() + { + m_is_event_group_active= FALSE; + } + + /* + Used for displaying events part of an event group. + Returns TRUE when both event group filtering is enabled and the current + event group should be displayed, OR if event group filtering is + disabled. More specifically, if filtering is disabled, all events + should be shown. + Returns FALSE when event group filtering is enabled and the current event + group is filtered out. + */ + my_bool is_event_group_active() + { + return m_is_event_group_filtering_enabled ? m_is_event_group_active : TRUE; + } + + /* + Notify that events must be explicitly activated in order to be printed + */ + void enable_event_group_filtering() + { + m_is_event_group_filtering_enabled= TRUE; + } } PRINT_EVENT_INFO; #endif diff --git a/sql/log_event_client.cc b/sql/log_event_client.cc index 7e0bf7d8e4c..66f2753d96b 100644 --- a/sql/log_event_client.cc +++ b/sql/log_event_client.cc @@ -3791,6 +3791,8 @@ st_print_event_info::st_print_event_info() printed_fd_event=FALSE; file= 0; base64_output_mode=BASE64_OUTPUT_UNSPEC; + m_is_event_group_active= TRUE; + m_is_event_group_filtering_enabled= FALSE; open_cached_file(&head_cache, NULL, NULL, 0, flags); open_cached_file(&body_cache, NULL, NULL, 0, flags); open_cached_file(&tail_cache, NULL, NULL, 0, flags); diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index 306ae878060..c1031133530 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -17,6 +17,7 @@ /* Definitions for MariaDB global transaction ID (GTID). */ +#ifndef MYSQL_CLIENT #include "mariadb.h" #include "sql_priv.h" #include "unireg.h" @@ -24,7 +25,6 @@ #include "sql_base.h" #include "sql_parse.h" #include "key.h" -#include "rpl_gtid.h" #include "rpl_rli.h" #include "slave.h" #include "log_event.h" @@ -1273,6 +1273,7 @@ rpl_slave_state::domain_to_gtid(uint32 domain_id, rpl_gtid *out_gtid) return true; } +#endif /* Parse a GTID at the start of a string, and update the pointer to point @@ -1310,7 +1311,6 @@ gtid_parser_helper(const char **ptr, const char *end, rpl_gtid *out_gtid) return 0; } - rpl_gtid * gtid_parse_string_to_list(const char *str, size_t str_len, uint32 *out_len) { @@ -1349,6 +1349,7 @@ gtid_parse_string_to_list(const char *str, size_t str_len, uint32 *out_len) return list; } +#ifndef MYSQL_CLIENT /* Update the slave replication state with the GTID position obtained from @@ -2974,3 +2975,844 @@ gtid_waiting::remove_from_wait_queue(gtid_waiting::hash_element *he, queue_remove(&he->queue, elem->queue_idx); } + +#endif + +void free_domain_lookup_element(void *p) +{ + struct Binlog_gtid_state_validator::audit_elem *audit_elem= + (struct Binlog_gtid_state_validator::audit_elem *) p; + delete_dynamic(&audit_elem->late_gtids_previous); + delete_dynamic(&audit_elem->late_gtids_real); + my_free(audit_elem); +} + +Binlog_gtid_state_validator::Binlog_gtid_state_validator() +{ + my_hash_init(PSI_INSTRUMENT_ME, &m_audit_elem_domain_lookup, &my_charset_bin, 32, + offsetof(struct audit_elem, domain_id), sizeof(uint32), + NULL, free_domain_lookup_element, HASH_UNIQUE); +} + +Binlog_gtid_state_validator::~Binlog_gtid_state_validator() +{ + my_hash_free(&m_audit_elem_domain_lookup); +} + +void Binlog_gtid_state_validator::initialize_start_gtids(rpl_gtid *start_gtids, + size_t n_gtids) +{ + size_t i; + for(i= 0; i < n_gtids; i++) + { + rpl_gtid *domain_state_gtid= &start_gtids[i]; + + /* + If we are initializing from a GLLE, we can have repeat domain ids from + differing servers, so we want to ensure our start gtid matches the last + known position + */ + struct audit_elem *audit_elem= (struct audit_elem *) my_hash_search( + &m_audit_elem_domain_lookup, + (const uchar *) &(domain_state_gtid->domain_id), 0); + if (audit_elem) + { + /* + We have this domain already specified, so try to overwrite with the + more recent GTID + */ + if (domain_state_gtid->seq_no > audit_elem->start_gtid.seq_no) + audit_elem->start_gtid = *domain_state_gtid; + continue; + } + + /* Initialize a new domain */ + audit_elem= (struct audit_elem *) my_malloc( + PSI_NOT_INSTRUMENTED, sizeof(struct audit_elem), MYF(MY_WME)); + if (!audit_elem) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return; + } + + audit_elem->domain_id= start_gtids[i].domain_id; + audit_elem->start_gtid= start_gtids[i]; + audit_elem->last_gtid= {audit_elem->domain_id, 0, 0}; + + my_init_dynamic_array(PSI_INSTRUMENT_ME, &audit_elem->late_gtids_real, + sizeof(rpl_gtid), 8, 8, MYF(0)); + my_init_dynamic_array(PSI_INSTRUMENT_ME, &audit_elem->late_gtids_previous, + sizeof(rpl_gtid), 8, 8, MYF(0)); + + if (my_hash_insert(&m_audit_elem_domain_lookup, (uchar *) audit_elem)) + { + my_free(audit_elem); + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return; + } + } +} + +my_bool Binlog_gtid_state_validator::initialize_gtid_state(FILE *out, + rpl_gtid *gtids, + size_t n_gtids) +{ + size_t i; + my_bool err= FALSE; + + /* + We weren't initialized with starting positions explicitly, so assume the + starting positions of the current gtid state + */ + if (!m_audit_elem_domain_lookup.records) + initialize_start_gtids(gtids, n_gtids); + + for(i= 0; i < n_gtids; i++) + { + rpl_gtid *domain_state_gtid= >ids[i]; + + struct audit_elem *audit_elem= (struct audit_elem *) my_hash_search( + &m_audit_elem_domain_lookup, + (const uchar *) &(domain_state_gtid->domain_id), 0); + + if (!audit_elem) + { + Binlog_gtid_state_validator::error( + out, + "Starting GTID position list does not specify an initial value " + "for domain %u, whose events may be present in the requested binlog " + "file(s). The last known position for this domain was %u-%u-%llu.", + domain_state_gtid->domain_id, PARAM_GTID((*domain_state_gtid))); + err= TRUE; + continue; + } + + if (audit_elem->start_gtid.seq_no < domain_state_gtid->seq_no) + { + Binlog_gtid_state_validator::error( + out, + "Binary logs are missing data for domain %u. Expected data to " + "start from state %u-%u-%llu; however, the initial GTID state of " + "the logs was %u-%u-%llu.", + domain_state_gtid->domain_id, PARAM_GTID(audit_elem->start_gtid), + PARAM_GTID((*domain_state_gtid))); + err= TRUE; + continue; + } + + if (domain_state_gtid->seq_no > audit_elem->last_gtid.seq_no) + audit_elem->last_gtid= *domain_state_gtid; + } + return err; +} + +my_bool Binlog_gtid_state_validator::verify_stop_state(FILE *out, + rpl_gtid *stop_gtids, + size_t n_stop_gtids) +{ + size_t i; + for(i= 0; i < n_stop_gtids; i++) + { + rpl_gtid *stop_gtid= &stop_gtids[i]; + + struct audit_elem *audit_elem= (struct audit_elem *) my_hash_search( + &m_audit_elem_domain_lookup, + (const uchar *) &(stop_gtid->domain_id), 0); + + /* + It is okay if stop gtid doesn't exist in current state because it will be treated + as a new domain + */ + if (audit_elem && stop_gtid->seq_no <= audit_elem->start_gtid.seq_no) + { + Binlog_gtid_state_validator::error( + out, + "--stop-position GTID %u-%u-%llu does not exist in the " + "specified binlog files. The current GTID state of domain %u in the " + "specified binary logs is %u-%u-%llu", + PARAM_GTID((*stop_gtid)), stop_gtid->domain_id, + PARAM_GTID(audit_elem->start_gtid)); + return TRUE; + } + } + + /* No issues with any GTIDs */ + return FALSE; +} + +my_bool +Binlog_gtid_state_validator::verify_gtid_state(FILE *out, + rpl_gtid *domain_state_gtid) +{ + struct audit_elem *audit_elem= (struct audit_elem *) my_hash_search( + &m_audit_elem_domain_lookup, + (const uchar *) &(domain_state_gtid->domain_id), 0); + + if (!audit_elem) + { + Binlog_gtid_state_validator::warn( + out, + "Binary logs are missing data for domain %u. The current binary log " + "specified its " + "current state for this domain as %u-%u-%llu, but neither the " + "starting GTID position list nor any processed events have " + "mentioned " + "this domain.", + domain_state_gtid->domain_id, PARAM_GTID((*domain_state_gtid))); + return TRUE; + } + + if (audit_elem->last_gtid.seq_no < domain_state_gtid->seq_no) + { + Binlog_gtid_state_validator::warn( + out, + "Binary logs are missing data for domain %u. The current binary log " + "state is %u-%u-%llu, but the last seen event was %u-%u-%llu.", + domain_state_gtid->domain_id, PARAM_GTID((*domain_state_gtid)), + PARAM_GTID(audit_elem->last_gtid)); + return TRUE; + } + + return FALSE; +} + +my_bool Binlog_gtid_state_validator::record(rpl_gtid *gtid) +{ + struct audit_elem *audit_elem= (struct audit_elem *) my_hash_search( + &m_audit_elem_domain_lookup, (const uchar *) &(gtid->domain_id), 0); + + if (!audit_elem) + { + /* + We haven't seen any GTIDs in this domian yet. Perform initial set up for + this domain so we can monitor its events. + */ + audit_elem= (struct audit_elem *) my_malloc( + PSI_NOT_INSTRUMENTED, sizeof(struct audit_elem), MYF(MY_WME)); + if (!audit_elem) + { + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return TRUE; + } + + audit_elem->domain_id= gtid->domain_id; + audit_elem->last_gtid= *gtid; + audit_elem->start_gtid= {gtid->domain_id, 0, 0}; + + my_init_dynamic_array(PSI_INSTRUMENT_ME, &audit_elem->late_gtids_real, + sizeof(rpl_gtid), 8, 8, MYF(0)); + my_init_dynamic_array(PSI_INSTRUMENT_ME, &audit_elem->late_gtids_previous, + sizeof(rpl_gtid), 8, 8, MYF(0)); + + if (my_hash_insert(&m_audit_elem_domain_lookup, (uchar *) audit_elem)) + { + my_free(audit_elem); + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return TRUE; + } + } + else + { + /* Out of order check */ + if (gtid->seq_no <= audit_elem->last_gtid.seq_no && + gtid->seq_no >= audit_elem->start_gtid.seq_no) + { + /* GTID is out of order */ + insert_dynamic(&audit_elem->late_gtids_real, (const void *) gtid); + insert_dynamic(&audit_elem->late_gtids_previous, + (const void *) &(audit_elem->last_gtid)); + + return TRUE; + } + else + { + /* GTID is valid */ + audit_elem->last_gtid= *gtid; + } + } + + return FALSE; +} + +/* + Data structure used to help pass data into report_audit_findings because + my_hash_iterate only passes one parameter +*/ +struct gtid_report_ctx +{ + FILE *out_file; + my_bool is_strict_mode; + my_bool contains_err; +}; + +static my_bool report_audit_findings(void *entry, void *report_ctx_arg) +{ + struct Binlog_gtid_state_validator::audit_elem *audit_el= + (struct Binlog_gtid_state_validator::audit_elem *) entry; + + struct gtid_report_ctx *report_ctx= + (struct gtid_report_ctx *) report_ctx_arg; + FILE *out= report_ctx->out_file; + my_bool is_strict_mode= report_ctx->is_strict_mode; + size_t i; + void (*report_f)(FILE*, const char*, ...); + + if (is_strict_mode) + report_f= Binlog_gtid_state_validator::error; + else + report_f= Binlog_gtid_state_validator::warn; + + if (audit_el) + { + if (audit_el->last_gtid.seq_no < audit_el->start_gtid.seq_no) + { + report_f(out, + "Binary logs never reached expected GTID state of %u-%u-%llu", + PARAM_GTID(audit_el->start_gtid)); + report_ctx->contains_err= TRUE; + } + + /* Report any out of order GTIDs */ + for(i= 0; i < audit_el->late_gtids_real.elements; i++) + { + rpl_gtid *real_gtid= + (rpl_gtid *) dynamic_array_ptr(&(audit_el->late_gtids_real), i); + rpl_gtid *last_gtid= (rpl_gtid *) dynamic_array_ptr( + &(audit_el->late_gtids_previous), i); + DBUG_ASSERT(real_gtid && last_gtid); + + report_f(out, + "Found out of order GTID. Got %u-%u-%llu after %u-%u-%llu", + PARAM_GTID((*real_gtid)), PARAM_GTID((*last_gtid))); + report_ctx->contains_err= TRUE; + } + } + + return FALSE; +} + +my_bool Binlog_gtid_state_validator::report(FILE *out, my_bool is_strict_mode) +{ + struct gtid_report_ctx report_ctx; + report_ctx.out_file= out; + report_ctx.is_strict_mode= is_strict_mode; + report_ctx.contains_err= FALSE; + my_hash_iterate(&m_audit_elem_domain_lookup, report_audit_findings, &report_ctx); + fflush(out); + return is_strict_mode ? report_ctx.contains_err : FALSE; +} + +Window_gtid_event_filter::Window_gtid_event_filter() + : m_has_start(FALSE), m_has_stop(FALSE), m_is_active(FALSE), + m_has_passed(FALSE) +{ + // m_start and m_stop do not need initial values if unused +} + +int Window_gtid_event_filter::set_start_gtid(rpl_gtid *start) +{ + if (m_has_start) + { + sql_print_error( + "Start position cannot have repeated domain " + "ids (found %u-%u-%llu when %u-%u-%llu was previously specified)", + PARAM_GTID((*start)), PARAM_GTID(m_start)); + return 1; + } + + m_has_start= TRUE; + m_start= *start; + return 0; +} + +int Window_gtid_event_filter::set_stop_gtid(rpl_gtid *stop) +{ + if (m_has_stop) + { + sql_print_error( + "Stop position cannot have repeated domain " + "ids (found %u-%u-%llu when %u-%u-%llu was previously specified)", + PARAM_GTID((*stop)), PARAM_GTID(m_stop)); + return 1; + } + + m_has_stop= TRUE; + m_stop= *stop; + return 0; +} + +my_bool Window_gtid_event_filter::is_range_invalid() +{ + if (m_has_start && m_has_stop && m_start.seq_no > m_stop.seq_no) + { + sql_print_error( + "Queried GTID range is invalid in strict mode. Stop position " + "%u-%u-%llu is not greater than or equal to start %u-%u-%llu.", + PARAM_GTID(m_stop), PARAM_GTID(m_start)); + return TRUE; + } + return FALSE; +} + +static inline my_bool is_gtid_at_or_after(rpl_gtid *boundary, + rpl_gtid *test_gtid) +{ + return test_gtid->domain_id == boundary->domain_id && + test_gtid->seq_no >= boundary->seq_no; +} + +static inline my_bool is_gtid_at_or_before(rpl_gtid *boundary, + rpl_gtid *test_gtid) +{ + return test_gtid->domain_id == boundary->domain_id && + test_gtid->seq_no <= boundary->seq_no; +} + +my_bool Window_gtid_event_filter::exclude(rpl_gtid *gtid) +{ + /* Assume result should be excluded to start */ + my_bool should_exclude= TRUE; + + DBUG_ASSERT((m_has_start || m_has_stop) && + (gtid->domain_id == m_start.domain_id || + gtid->domain_id == m_stop.domain_id)); + + if (!m_is_active && !m_has_passed) + { + /* + This filter has not yet been activated. Check if the gtid is within the + bounds of this window. + */ + + if (!m_has_start && is_gtid_at_or_before(&m_stop, gtid)) + { + /* + Start GTID was not provided, so we want to include everything from here + up to m_stop + */ + m_is_active= TRUE; + should_exclude= FALSE; + } + else if ((m_has_start && is_gtid_at_or_after(&m_start, gtid)) && + (!m_has_stop || is_gtid_at_or_before(&m_stop, gtid))) + { + m_is_active= TRUE; + + DBUG_PRINT("gtid-event-filter", + ("Window: Begin (%d-%d-%llu, %d-%d-%llu]", + PARAM_GTID(m_start), PARAM_GTID(m_stop))); + + /* + As the start of the range is exclusive, if this gtid is the start of + the range, exclude it + */ + if (gtid->seq_no == m_start.seq_no) + should_exclude= TRUE; + else + should_exclude= FALSE; + + if (m_has_stop && gtid->seq_no == m_stop.seq_no) + { + m_has_passed= TRUE; + DBUG_PRINT("gtid-event-filter", + ("Window: End (%d-%d-%llu, %d-%d-%llu]", + PARAM_GTID(m_start), PARAM_GTID(m_stop))); + } + } + } /* if (!m_is_active && !m_has_passed) */ + else if (m_is_active && !m_has_passed) + { + /* + This window is currently active so we want the event group to be included + in the results. Additionally check if we are at the end of the window. + If no end of the window is provided, go indefinitely + */ + should_exclude= FALSE; + + if (m_has_stop && is_gtid_at_or_after(&m_stop, gtid)) + { + DBUG_PRINT("gtid-event-filter", + ("Window: End (%d-%d-%llu, %d-%d-%llu]", + PARAM_GTID(m_start), PARAM_GTID(m_stop))); + m_is_active= FALSE; + m_has_passed= TRUE; + + if (!is_gtid_at_or_before(&m_stop, gtid)) + { + /* + The GTID is after the finite stop of the window, don't let it pass + through + */ + should_exclude= TRUE; + } + } + } + + return should_exclude; +} + +my_bool Window_gtid_event_filter::has_finished() +{ + return m_has_stop ? m_has_passed : FALSE; +} + +void free_gtid_filter_element(void *p) +{ + gtid_filter_element *gfe = (gtid_filter_element *) p; + if (gfe->filter) + delete gfe->filter; + my_free(gfe); +} + +Id_delegating_gtid_event_filter::Id_delegating_gtid_event_filter() + : m_num_stateful_filters(0), m_num_completed_filters(0) +{ + my_hash_init(PSI_INSTRUMENT_ME, &m_filters_by_id_hash, &my_charset_bin, 32, + offsetof(gtid_filter_element, identifier), + sizeof(gtid_filter_identifier), NULL, free_gtid_filter_element, + HASH_UNIQUE); + + m_default_filter= new Accept_all_gtid_filter(); +} + +Id_delegating_gtid_event_filter::~Id_delegating_gtid_event_filter() +{ + my_hash_free(&m_filters_by_id_hash); + delete m_default_filter; +} + +void Id_delegating_gtid_event_filter::set_default_filter( + Gtid_event_filter *filter) +{ + if (m_default_filter) + delete m_default_filter; + + m_default_filter= filter; +} + +gtid_filter_element * +Id_delegating_gtid_event_filter::find_or_create_filter_element_for_id( + gtid_filter_identifier filter_id) +{ + gtid_filter_element *fe= (gtid_filter_element *) my_hash_search( + &m_filters_by_id_hash, (const uchar *) &filter_id, 0); + + if (!fe) + { + gtid_filter_element *new_fe= (gtid_filter_element *) my_malloc( + PSI_NOT_INSTRUMENTED, sizeof(gtid_filter_element), MYF(MY_WME)); + new_fe->filter= NULL; + new_fe->identifier= filter_id; + if (my_hash_insert(&m_filters_by_id_hash, (uchar*) new_fe)) + { + my_free(new_fe); + my_error(ER_OUT_OF_RESOURCES, MYF(0)); + return NULL; + } + fe= new_fe; + } + + return fe; +} + +my_bool Id_delegating_gtid_event_filter::has_finished() +{ + /* + If all user-defined filters have deactivated, we are effectively + deactivated + */ + return m_num_stateful_filters && + m_num_completed_filters == m_num_stateful_filters; +} + +my_bool Id_delegating_gtid_event_filter::exclude(rpl_gtid *gtid) +{ + gtid_filter_identifier filter_id= get_id_from_gtid(gtid); + gtid_filter_element *filter_element= (gtid_filter_element *) my_hash_search( + &m_filters_by_id_hash, (const uchar *) &filter_id, 0); + Gtid_event_filter *filter= + (filter_element ? filter_element->filter : m_default_filter); + my_bool ret= TRUE; + + if(!filter_element || !filter->has_finished()) + { + ret= filter->exclude(gtid); + + /* + If this is an explicitly defined filter, e.g. Window-based filter, check + if it has completed, and update the counter accordingly if so. + */ + if (filter_element && filter->has_finished()) + m_num_completed_filters++; + } + + return ret; +} + +Window_gtid_event_filter * +Domain_gtid_event_filter::find_or_create_window_filter_for_id( + uint32 domain_id) +{ + gtid_filter_element *filter_element= + find_or_create_filter_element_for_id(domain_id); + Window_gtid_event_filter *wgef= NULL; + + if (filter_element->filter == NULL) + { + /* New filter */ + wgef= new Window_gtid_event_filter(); + filter_element->filter= wgef; + } + else if (filter_element->filter->get_filter_type() == WINDOW_GTID_FILTER_TYPE) + { + /* We have an existing window filter here */ + wgef= (Window_gtid_event_filter *) filter_element->filter; + } + else + { + /* + We have an existing filter but it is not of window type so propogate NULL + filter + */ + sql_print_error("cannot subset domain id %d by position, another rule " + "exists on that domain", + domain_id); + } + + return wgef; +} + +static my_bool check_filter_entry_validity(void *entry, void *are_filters_invalid_arg) +{ + gtid_filter_element *fe= (gtid_filter_element*) entry; + + if (fe) + { + Gtid_event_filter *gef= fe->filter; + if (gef->get_filter_type() == Gtid_event_filter::WINDOW_GTID_FILTER_TYPE) + { + Window_gtid_event_filter *wgef= (Window_gtid_event_filter *) gef; + if (wgef->is_range_invalid()) + { + *((int *) are_filters_invalid_arg)= 1; + return TRUE; + } + } + } + return FALSE; +} + +int Domain_gtid_event_filter::validate_window_filters() +{ + int are_filters_invalid= 0; + my_hash_iterate(&m_filters_by_id_hash, check_filter_entry_validity, &are_filters_invalid); + return are_filters_invalid; +} + +int Domain_gtid_event_filter::add_start_gtid(rpl_gtid *gtid) +{ + int err= 0; + Window_gtid_event_filter *filter_to_update= + find_or_create_window_filter_for_id(gtid->domain_id); + + if (filter_to_update == NULL) + { + err= 1; + } + else if (!(err= filter_to_update->set_start_gtid(gtid))) + { + gtid_filter_element *fe= (gtid_filter_element *) my_hash_search( + &m_filters_by_id_hash, (const uchar *) &(gtid->domain_id), 0); + insert_dynamic(&m_start_filters, (const void *) &fe); + } + + return err; +} + +int Domain_gtid_event_filter::add_stop_gtid(rpl_gtid *gtid) +{ + int err= 0; + Window_gtid_event_filter *filter_to_update= + find_or_create_window_filter_for_id(gtid->domain_id); + + if (filter_to_update == NULL) + { + err= 1; + } + else if (!(err= filter_to_update->set_stop_gtid(gtid))) + { + gtid_filter_element *fe= (gtid_filter_element *) my_hash_search( + &m_filters_by_id_hash, (const uchar *) &(gtid->domain_id), 0); + insert_dynamic(&m_stop_filters, (const void *) &fe); + + /* + A window with a stop position can be disabled, and is therefore stateful. + */ + m_num_stateful_filters++; + + /* + Default filtering behavior changes with GTID stop positions, where we + exclude all domains not present in the stop list + */ + if (m_default_filter->get_filter_type() == ACCEPT_ALL_GTID_FILTER_TYPE) + { + delete m_default_filter; + m_default_filter= new Reject_all_gtid_filter(); + } + } + + return err; +} + +rpl_gtid *Domain_gtid_event_filter::get_start_gtids() +{ + rpl_gtid *gtid_list; + uint32 i; + size_t n_start_gtids= get_num_start_gtids(); + + gtid_list= (rpl_gtid *) my_malloc( + PSI_INSTRUMENT_ME, n_start_gtids * sizeof(rpl_gtid), MYF(MY_WME)); + + for (i = 0; i < n_start_gtids; i++) + { + gtid_filter_element *fe= + *(gtid_filter_element **) dynamic_array_ptr(&m_start_filters, i); + DBUG_ASSERT(fe->filter && + fe->filter->get_filter_type() == WINDOW_GTID_FILTER_TYPE); + Window_gtid_event_filter *wgef= + (Window_gtid_event_filter *) fe->filter; + + rpl_gtid win_start_gtid= wgef->get_start_gtid(); + gtid_list[i]= win_start_gtid; + } + + return gtid_list; +} + +rpl_gtid *Domain_gtid_event_filter::get_stop_gtids() +{ + rpl_gtid *gtid_list; + uint32 i; + size_t n_stop_gtids= get_num_stop_gtids(); + + gtid_list= (rpl_gtid *) my_malloc( + PSI_INSTRUMENT_ME, n_stop_gtids * sizeof(rpl_gtid), MYF(MY_WME)); + + for (i = 0; i < n_stop_gtids; i++) + { + gtid_filter_element *fe= + *(gtid_filter_element **) dynamic_array_ptr(&m_stop_filters, i); + DBUG_ASSERT(fe->filter && + fe->filter->get_filter_type() == WINDOW_GTID_FILTER_TYPE); + Window_gtid_event_filter *wgef= + (Window_gtid_event_filter *) fe->filter; + + rpl_gtid win_stop_gtid= wgef->get_stop_gtid(); + gtid_list[i]= win_stop_gtid; + } + + return gtid_list; +} + +void Domain_gtid_event_filter::clear_start_gtids() +{ + uint32 i; + for (i = 0; i < get_num_start_gtids(); i++) + { + gtid_filter_element *fe= + *(gtid_filter_element **) dynamic_array_ptr(&m_start_filters, i); + DBUG_ASSERT(fe->filter && + fe->filter->get_filter_type() == WINDOW_GTID_FILTER_TYPE); + Window_gtid_event_filter *wgef= + (Window_gtid_event_filter *) fe->filter; + + if (wgef->has_stop()) + { + /* + Don't delete the whole filter if it already has a stop position attached + */ + wgef->clear_start_pos(); + } + else + { + /* + This domain only has a stop, so delete the whole filter + */ + my_hash_delete(&m_filters_by_id_hash, (uchar *) fe); + } + } + + reset_dynamic(&m_start_filters); +} + +void Domain_gtid_event_filter::clear_stop_gtids() +{ + uint32 i; + + for (i = 0; i < get_num_stop_gtids(); i++) + { + gtid_filter_element *fe= + *(gtid_filter_element **) dynamic_array_ptr(&m_stop_filters, i); + DBUG_ASSERT(fe->filter && + fe->filter->get_filter_type() == WINDOW_GTID_FILTER_TYPE); + Window_gtid_event_filter *wgef= + (Window_gtid_event_filter *) fe->filter; + + if (wgef->has_start()) + { + /* + Don't delete the whole filter if it already has a start position + attached + */ + wgef->clear_stop_pos(); + } + else + { + /* + This domain only has a start, so delete the whole filter + */ + my_hash_delete(&m_filters_by_id_hash, (uchar *) fe); + } + m_num_stateful_filters--; + } + + /* + Stop positions were cleared and we want to be inclusive again of other + domains again + */ + if (m_default_filter->get_filter_type() == REJECT_ALL_GTID_FILTER_TYPE) + { + delete m_default_filter; + m_default_filter= new Accept_all_gtid_filter(); + } + + reset_dynamic(&m_stop_filters); +} + +my_bool Domain_gtid_event_filter::exclude(rpl_gtid *gtid) +{ + my_bool include_domain= TRUE; + /* + If GTID stop positions are provided, we limit the domains which are output + to only be those specified with stop positions + */ + if (get_num_stop_gtids()) + { + gtid_filter_identifier filter_id= get_id_from_gtid(gtid); + gtid_filter_element *filter_element= + (gtid_filter_element *) my_hash_search(&m_filters_by_id_hash, + (const uchar *) &filter_id, 0); + if (filter_element) + { + Gtid_event_filter *filter= filter_element->filter; + if (filter->get_filter_type() == WINDOW_GTID_FILTER_TYPE) + { + Window_gtid_event_filter *wgef= (Window_gtid_event_filter *) filter; + include_domain= wgef->has_stop(); + } + } + } + + return include_domain ? Id_delegating_gtid_event_filter::exclude(gtid) + : TRUE; +} diff --git a/sql/rpl_gtid.h b/sql/rpl_gtid.h index 11541c8000c..8b8c3bb9c92 100644 --- a/sql/rpl_gtid.h +++ b/sql/rpl_gtid.h @@ -28,6 +28,7 @@ extern const LEX_CSTRING rpl_gtid_slave_state_table_name; class String; #define GTID_MAX_STR_LENGTH (10+1+10+1+20) +#define PARAM_GTID(G) G.domain_id, G.server_id, G.seq_no struct rpl_gtid { @@ -36,6 +37,9 @@ struct rpl_gtid uint64 seq_no; }; +/* Data structure to help with quick lookup for filters. */ +typedef decltype(rpl_gtid::domain_id) gtid_filter_identifier; + inline bool operator==(const rpl_gtid& lhs, const rpl_gtid& rhs) { return @@ -380,5 +384,449 @@ extern bool rpl_slave_state_tostring_helper(String *dest, const rpl_gtid *gtid, extern int gtid_check_rpl_slave_state_table(TABLE *table); extern rpl_gtid *gtid_parse_string_to_list(const char *p, size_t len, uint32 *out_len); +extern rpl_gtid *gtid_unpack_string_to_list(const char *p, size_t len, + uint32 *out_len); + + + +/* + This class ensures that the GTID state of an event stream is consistent with + the set of provided binary log files. In particular, it has two concerns: + + 1) Ensuring that GTID events are monotonically increasing within each + domain + 2) Ensuring that the GTID state of the specified binary logs is consistent + both with the initial state that a user provides, and between + binary logs (if multiple are specified) +*/ +class Binlog_gtid_state_validator +{ +public: + + struct audit_elem + { + uint32 domain_id; + + + /* + Holds the largest GTID received, and is indexed by domain_id + */ + rpl_gtid last_gtid; + + /* + Holds the largest GTID received, and is indexed by domain_id + */ + rpl_gtid start_gtid; + + /* + List of the problematic GTIDs received which were out of order + */ + DYNAMIC_ARRAY late_gtids_real; + + /* + For each problematic GTID in late_gtids_real, this list contains the last + GTID of the domain at the time of receiving the out of order GTID. + */ + DYNAMIC_ARRAY late_gtids_previous; + }; + + Binlog_gtid_state_validator(); + ~Binlog_gtid_state_validator(); + + /* + Initialize where we should start monitoring for invalid GTID entries + in the event stream. Note that these start positions must occur at or after + a given binary logs GTID state (from Gtid_list_log_event) + */ + void initialize_start_gtids(rpl_gtid *start_gtids, size_t n_gtids); + + /* + Initialize our current state so we know where to expect GTIDs to start + increasing from. Error if the state exists after our expected start_gtid + positions, because we know we will be missing event data (possibly from + a purged log). + */ + my_bool initialize_gtid_state(FILE *out, rpl_gtid *gtids, size_t n_gtids); + + /* + Ensures that the expected stop GTID positions exist within the specified + binary logs. + */ + my_bool verify_stop_state(FILE *out, rpl_gtid *stop_gtids, size_t n_stop_gtids); + + /* + Ensure a GTID state (e.g., from a Gtid_list_log_event) is consistent with + the current state of our auditing. For example, if we see a GTID from a + Gtid_list_log_event that is ahead of our current state for that domain, we + have missed events (perhaps from a missing log). + */ + my_bool verify_gtid_state(FILE *out, rpl_gtid *gtid_state_cur); + + /* + Take note of a new GTID being processed. + + returns TRUE if the GTID is invalid, FALSE on success + */ + my_bool record(rpl_gtid *gtid); + + /* + Writes warnings/errors (if any) during GTID processing + + Returns TRUE if any findings were reported, FALSE otherwise + */ + my_bool report(FILE *out, my_bool is_strict_mode); + + static void report_details(FILE *out, const char *format, va_list args) + { + vfprintf(out, format, args); + fprintf(out, "\n"); + } + + static void warn(FILE *out, const char *format,...) + { + va_list args; + va_start(args, format); + fprintf(out, "WARNING: "); + report_details(out, format, args); + } + + static void error(FILE *out, const char *format,...) + { + va_list args; + va_start(args, format); + fprintf(out, "ERROR: "); + report_details(out, format, args); + } + +private: + + /* + Holds the records for each domain id we are monitoring. Elements are of type + `struct audit_elem` and indexed by domian_id. + */ + HASH m_audit_elem_domain_lookup; +}; + +/* + Interface to support different methods of filtering log events by GTID +*/ +class Gtid_event_filter +{ +public: + Gtid_event_filter() {}; + virtual ~Gtid_event_filter() {}; + + enum gtid_event_filter_type + { + DELEGATING_GTID_FILTER_TYPE = 1, + WINDOW_GTID_FILTER_TYPE = 2, + ACCEPT_ALL_GTID_FILTER_TYPE = 3, + REJECT_ALL_GTID_FILTER_TYPE = 4 + }; + + /* + Run the filter on an input gtid to test if the corresponding log events + should be excluded from a result + + Returns TRUE when the event group corresponding to the input GTID should be + excluded. + Returns FALSE when the event group should be included. + */ + virtual my_bool exclude(rpl_gtid *) = 0; + + /* + The gtid_event_filter_type that corresponds to the underlying filter + implementation + */ + virtual uint32 get_filter_type() = 0; + + /* + For filters that can maintain their own state, this tests if the filter + implementation has completed. + + Returns TRUE when completed, and FALSE when the filter has not finished. + */ + virtual my_bool has_finished() = 0; +}; + +/* + Filter implementation which will include any and all input GTIDs. This is + used to set default behavior for GTIDs that do not have explicit filters + set on their domain_id, e.g. when a Window_gtid_event_filter is used for + a specific domain, then all other domain_ids will be accepted using this + filter implementation. +*/ +class Accept_all_gtid_filter : public Gtid_event_filter +{ +public: + Accept_all_gtid_filter() {} + ~Accept_all_gtid_filter() {} + my_bool exclude(rpl_gtid *gtid) { return FALSE; } + uint32 get_filter_type() { return ACCEPT_ALL_GTID_FILTER_TYPE; } + my_bool has_finished() { return FALSE; } +}; + +/* + Filter implementation to exclude all tested GTIDs. +*/ +class Reject_all_gtid_filter : public Gtid_event_filter +{ +public: + Reject_all_gtid_filter() {} + ~Reject_all_gtid_filter() {} + my_bool exclude(rpl_gtid *gtid) { return TRUE; } + uint32 get_filter_type() { return REJECT_ALL_GTID_FILTER_TYPE; } + my_bool has_finished() { return FALSE; } +}; + +/* + A filter implementation that includes events that exist between two GTID + positions, m_start (exclusive) and m_stop (inclusive), within a domain. + + This filter is stateful, such that it expects GTIDs to be an increasing + stream, and internally, the window will activate and deactivate when the start + and stop positions of the event stream have passed through, respectively. +*/ +class Window_gtid_event_filter : public Gtid_event_filter +{ +public: + Window_gtid_event_filter(); + ~Window_gtid_event_filter() {} + + my_bool exclude(rpl_gtid*); + my_bool has_finished(); + + /* + Set the GTID that begins this window (exclusive) + + Returns 0 on ok, non-zero on error + */ + int set_start_gtid(rpl_gtid *start); + + /* + Set the GTID that ends this window (inclusive) + + Returns 0 on ok, non-zero on error + */ + int set_stop_gtid(rpl_gtid *stop); + + uint32 get_filter_type() { return WINDOW_GTID_FILTER_TYPE; } + + /* + Validates the underlying range is correct, and writes an error if not, i.e. + m_start >= m_stop. + + Returns FALSE on ok, TRUE if range is invalid + */ + my_bool is_range_invalid(); + + /* + Getter/setter methods + */ + my_bool has_start() { return m_has_start; } + my_bool has_stop() { return m_has_stop; } + rpl_gtid get_start_gtid() { return m_start; } + rpl_gtid get_stop_gtid() { return m_stop; } + + void clear_start_pos() + { + m_has_start= FALSE; + m_start= {0, 0, 0}; + } + + void clear_stop_pos() + { + m_has_stop= FALSE; + m_stop= {0, 0, 0}; + } + +protected: + + /* + When processing GTID streams, the order in which they are processed should + be sequential with no gaps between events. If a gap is found within a + window, warn the user. + */ + void verify_gtid_is_expected(rpl_gtid *gtid); + +private: + + enum warning_flags + { + WARN_GTID_SEQUENCE_NUMBER_OUT_OF_ORDER= 0x1 + }; + + /* + m_has_start : Indicates if a start to this window has been explicitly + provided. A window starts immediately if not provided. + */ + my_bool m_has_start; + + /* + m_has_stop : Indicates if a stop to this window has been explicitly + provided. A window continues indefinitely if not provided. + */ + my_bool m_has_stop; + + /* + m_is_active : Indicates whether or not the program is currently reading + events from within this window. When TRUE, events with + different server ids than those specified by m_start or + m_stop will be passed through. + */ + my_bool m_is_active; + + /* + m_has_passed : Indicates whether or not the program is currently reading + events from within this window. + */ + my_bool m_has_passed; + + /* m_start : marks the GTID that begins the window (exclusive). */ + rpl_gtid m_start; + + /* m_stop : marks the GTID that ends the range (inclusive). */ + rpl_gtid m_stop; +}; + +typedef struct _gtid_filter_element +{ + Gtid_event_filter *filter; + gtid_filter_identifier identifier; /* Used for HASH lookup */ +} gtid_filter_element; + +/* + Gtid_event_filter subclass which has no specific implementation, but rather + delegates the filtering to specific identifiable/mapped implementations. + + A default filter is used for GTIDs that are passed through which no explicit + filter can be identified. + + This class should be subclassed, where the get_id_from_gtid function + specifies how to extract the filter identifier from a GTID. +*/ +class Id_delegating_gtid_event_filter : public Gtid_event_filter +{ +public: + Id_delegating_gtid_event_filter(); + ~Id_delegating_gtid_event_filter(); + + my_bool exclude(rpl_gtid *gtid); + my_bool has_finished(); + void set_default_filter(Gtid_event_filter *default_filter); + + uint32 get_filter_type() { return DELEGATING_GTID_FILTER_TYPE; } + + virtual gtid_filter_identifier get_id_from_gtid(rpl_gtid *) = 0; + +protected: + + uint32 m_num_stateful_filters; + uint32 m_num_completed_filters; + Gtid_event_filter *m_default_filter; + + HASH m_filters_by_id_hash; + + gtid_filter_element *find_or_create_filter_element_for_id(gtid_filter_identifier); +}; + +/* + A subclass of Id_delegating_gtid_event_filter which identifies filters using the + domain id of a GTID. + + Additional helper functions include: + add_start_gtid(GTID) : adds a start GTID position to this filter, to be + identified by its domain id + add_stop_gtid(GTID) : adds a stop GTID position to this filter, to be + identified by its domain id + clear_start_gtids() : removes existing GTID start positions + clear_stop_gtids() : removes existing GTID stop positions + get_start_gtids() : gets all added GTID start positions + get_stop_gtids() : gets all added GTID stop positions + get_num_start_gtids() : gets the count of added GTID start positions + get_num_stop_gtids() : gets the count of added GTID stop positions +*/ +class Domain_gtid_event_filter : public Id_delegating_gtid_event_filter +{ +public: + Domain_gtid_event_filter() + { + my_init_dynamic_array(PSI_INSTRUMENT_ME, &m_start_filters, + sizeof(gtid_filter_element*), 8, 8, MYF(0)); + my_init_dynamic_array(PSI_INSTRUMENT_ME, &m_stop_filters, + sizeof(gtid_filter_element*), 8, 8, MYF(0)); + } + ~Domain_gtid_event_filter() + { + delete_dynamic(&m_start_filters); + delete_dynamic(&m_stop_filters); + } + + /* + Returns the domain id of from the input GTID + */ + gtid_filter_identifier get_id_from_gtid(rpl_gtid *gtid) + { + return gtid->domain_id; + } + + /* + Override Id_delegating_gtid_event_filter to extend with domain specific + filtering logic + */ + my_bool exclude(rpl_gtid*); + + /* + Validates that window filters with both a start and stop GTID satisfy + stop_gtid > start_gtid + + Returns 0 on ok, non-zero if any windows are invalid. + */ + int validate_window_filters(); + + /* + Helper function to start a GTID window filter at the given GTID + + Returns 0 on ok, non-zero on error + */ + int add_start_gtid(rpl_gtid *gtid); + + /* + Helper function to end a GTID window filter at the given GTID + + Returns 0 on ok, non-zero on error + */ + int add_stop_gtid(rpl_gtid *gtid); + + /* + If start or stop position is respecified, we remove all existing values + and start over with the new specification. + */ + void clear_start_gtids(); + void clear_stop_gtids(); + + /* + Return list of all GTIDs used as start position. + + Note that this list is allocated and it is up to the user to free it + */ + rpl_gtid *get_start_gtids(); + + /* + Return list of all GTIDs used as stop position. + + Note that this list is allocated and it is up to the user to free it + */ + rpl_gtid *get_stop_gtids(); + + size_t get_num_start_gtids() { return m_start_filters.elements; } + size_t get_num_stop_gtids() { return m_stop_filters.elements; } + +private: + DYNAMIC_ARRAY m_start_filters; + DYNAMIC_ARRAY m_stop_filters; + + Window_gtid_event_filter *find_or_create_window_filter_for_id(gtid_filter_identifier); +}; #endif /* RPL_GTID_H */ -- cgit v1.2.1