diff options
-rw-r--r-- | mysql-test/collections/README | 30 | ||||
-rw-r--r-- | mysql-test/collections/README.experimental | 25 | ||||
-rw-r--r-- | mysql-test/collections/default.daily | 1 | ||||
-rw-r--r-- | mysql-test/collections/default.experimental | 1 | ||||
-rw-r--r-- | mysql-test/collections/default.push | 5 | ||||
-rw-r--r-- | mysql-test/collections/default.weekly | 0 | ||||
-rw-r--r-- | mysql-test/lib/mtr_report.pm | 39 | ||||
-rwxr-xr-x | mysql-test/mysql-test-run.pl | 26 |
8 files changed, 118 insertions, 9 deletions
diff --git a/mysql-test/collections/README b/mysql-test/collections/README new file mode 100644 index 00000000000..9af84646a40 --- /dev/null +++ b/mysql-test/collections/README @@ -0,0 +1,30 @@ +This directory contains collections of test runs that we run during our +integration and release testing. Each file contains zero or more lines, +with one invocation of mysql-test-run.pl on each. These invocations are +written so that, with the assumption that perl is in your search path, +any collection can run as a shell script or a batch file, with the parent +mysql-test directory being the current working directory. + +During integration testing, we choose the collection to run by following +these steps: + +1) We choose the extension to look for, based on these rules: + - If we're running a per-push test, we choose ".push" as the extension. + - If we're running a daily test, we choose ".daily" as the extension. + - If we're running a weekly test, we choose ".weekly" as the extension. + +2) If there is a collection that has the same name as the branch we're + testing plus the extension as determined in step 1, we choose that + collection. + +3) If the branch is unknown or we have removed all characters from it + and still not found a matching collection, we choose the name "default" + plus the extension determined in step 1. If there is no such file, + we give up and don't test anything at all. + +4) If we haven't found a collection yet, we remove the last character from + the branch name and go back to step 2. + +5) The commands from the collection are run line by line via execv() or + similar system calls. They are not run as a shell script. Shell + expansions are not guaranteed to work and most likely won't. diff --git a/mysql-test/collections/README.experimental b/mysql-test/collections/README.experimental new file mode 100644 index 00000000000..d06e55f1246 --- /dev/null +++ b/mysql-test/collections/README.experimental @@ -0,0 +1,25 @@ +The .experimental files in this directory contain names of test cases that +are still in development and whose failures should be considered expected, +instead of regressions. + +These files are to be used with the --experimental option of +mysql-test-run.pl. Please look at its help screen for usage information. + +The syntax is as follows: + +1) One line per test case. + +2) Empty lines and lines starting with a hash (#) are ignored. + +3) If any other line contains a black followed by a hash (#), the hash + and any subsequent characters are ignored. + +4) The full test case name including the suite and execution mode + must be specified, for example: + main.alias 'row' # bug#00000 + +5) As an exception to item 4, the last character of the test case + specification may be an asterisk (*). In that case, all test cases that + start with the same characters up to the last letter before the asterisk + are considered experimental: + main.a* # get rid of main.alias, main.alibaba and main.agliolio diff --git a/mysql-test/collections/default.daily b/mysql-test/collections/default.daily new file mode 100644 index 00000000000..194cc2aad59 --- /dev/null +++ b/mysql-test/collections/default.daily @@ -0,0 +1 @@ +perl mysql-test-run.pl --timer --force --comment=rpl_ndb_row --suite=rpl_ndb,ndb --mysqld=--binlog-format=row --experimental=collections/default.experimental diff --git a/mysql-test/collections/default.experimental b/mysql-test/collections/default.experimental new file mode 100644 index 00000000000..103069f79cf --- /dev/null +++ b/mysql-test/collections/default.experimental @@ -0,0 +1 @@ +funcs_1.charset_collation_1 # depends on compile-time decisions diff --git a/mysql-test/collections/default.push b/mysql-test/collections/default.push new file mode 100644 index 00000000000..0879b6fde2c --- /dev/null +++ b/mysql-test/collections/default.push @@ -0,0 +1,5 @@ +perl mysql-test-run.pl --timer --force --comment=n_mix --mysqld=--binlog-format=mixed --experimental=collections/default.experimental +perl mysql-test-run.pl --timer --force --comment=ps_row --ps-protocol --mysqld=--binlog-format=row --experimental=collections/default.experimental +perl mysql-test-run.pl --timer --force --comment=embedded --embedded --experimental=collections/default.experimental +perl mysql-test-run.pl --timer --force --comment=rpl_binlog_row --suite=rpl,binlog --mysqld=--binlog-format=row --experimental=collections/default.experimental +perl mysql-test-run.pl --timer --force --comment=funcs_1 --suite=funcs_1 --experimental=collections/default.experimental diff --git a/mysql-test/collections/default.weekly b/mysql-test/collections/default.weekly new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/mysql-test/collections/default.weekly diff --git a/mysql-test/lib/mtr_report.pm b/mysql-test/lib/mtr_report.pm index ce3fba87385..d1cae6324cb 100644 --- a/mysql-test/lib/mtr_report.pm +++ b/mysql-test/lib/mtr_report.pm @@ -69,6 +69,8 @@ sub _mtr_report_test_name ($) { print _name(), _timestamp(); printf "%-40s ", $tname; + + return $tname; } @@ -105,20 +107,41 @@ sub mtr_report_test_passed ($) { sub mtr_report_test ($) { my ($tinfo)= @_; - _mtr_report_test_name($tinfo); + my $test_name = _mtr_report_test_name($tinfo); - my $comment= $tinfo->{'comment'}; - my $logfile= $tinfo->{'logfile'}; - my $warnings= $tinfo->{'warnings'}; - my $result= $tinfo->{'result'}; + my $comment= $tinfo->{'comment'}; + my $logfile= $tinfo->{'logfile'}; + my $warnings= $tinfo->{'warnings'}; + my $result= $tinfo->{'result'}; if ($result eq 'MTR_RES_FAILED'){ my $timest = format_time(); + my $fail = "fail"; + + if ( $::opt_experimental ) + { + # Find out if this test case is an experimental one, so we can treat + # the failure as an expected failure instead of a regression. + for my $exp ( @$::opt_experimental ) { + if ( $exp ne $test_name ) { + if ( $exp ne "" && substr($exp, -1, 1) eq "*" ) { + $exp = substr($exp, 0, length($exp) - 1); + if ( substr($test_name, 0, length($exp)) ne $exp ) { + next; + } + } else { + next; + } + } + $fail = "exp-fail"; + last; + } + } if ( $warnings ) { - mtr_report("[ fail ] Found warnings/errors in server log file!"); + mtr_report("[ $fail ] Found warnings/errors in server log file!"); mtr_report(" Test ended at $timest"); mtr_report($warnings); return; @@ -126,14 +149,14 @@ sub mtr_report_test ($) { my $timeout= $tinfo->{'timeout'}; if ( $timeout ) { - mtr_report("[ fail ] timeout after $timeout seconds"); + mtr_report("[ $fail ] timeout after $timeout seconds"); mtr_report(" Test ended at $timest"); mtr_report("\n$tinfo->{'comment'}"); return; } else { - mtr_report("[ fail ]\n Test ended at $timest"); + mtr_report("[ $fail ]\n Test ended at $timest"); } if ( $logfile ) diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 511cd4d07ba..8e772279aa3 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -169,6 +169,8 @@ our $opt_client_debugger; my $config; # The currently running config my $current_config_name; # The currently running config file template +our $opt_experimental; + my $baseport; my $opt_build_thread= $ENV{'MTR_BUILD_THREAD'} || "auto"; @@ -792,7 +794,7 @@ sub command_line_setup { 'big-test' => \$opt_big_test, 'combination=s' => \@opt_combinations, 'skip-combinations' => \&collect_option, - + 'experimental=s' => \$opt_experimental, 'skip-im' => \&ignore_option, # Specify ports @@ -959,6 +961,28 @@ sub command_line_setup { mtr_print_thick_line('#'); } + if ( $opt_experimental ) + { + if ( open(FILE, "<", $opt_experimental) ) { + mtr_report("Using experimental file: $opt_experimental"); + $opt_experimental = []; + while(<FILE>) { + chomp; + s/( +|^)#.*$//; + s/^ +//; + s/ +$//; + if ( $_ eq "" ) { + next; + } + print " - $_\n"; + push @$opt_experimental, $_; + } + close FILE; + } else { + mtr_error("Can't read experimental file: $opt_experimental"); + } + } + foreach my $arg ( @ARGV ) { if ( $arg =~ /^--skip-/ ) |