summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--mysql-test/collections/README30
-rw-r--r--mysql-test/collections/README.experimental25
-rw-r--r--mysql-test/collections/default.daily1
-rw-r--r--mysql-test/collections/default.experimental1
-rw-r--r--mysql-test/collections/default.push5
-rw-r--r--mysql-test/collections/default.weekly0
-rw-r--r--mysql-test/lib/mtr_report.pm39
-rwxr-xr-xmysql-test/mysql-test-run.pl26
8 files changed, 118 insertions, 9 deletions
diff --git a/mysql-test/collections/README b/mysql-test/collections/README
new file mode 100644
index 00000000000..9af84646a40
--- /dev/null
+++ b/mysql-test/collections/README
@@ -0,0 +1,30 @@
+This directory contains collections of test runs that we run during our
+integration and release testing. Each file contains zero or more lines,
+with one invocation of mysql-test-run.pl on each. These invocations are
+written so that, with the assumption that perl is in your search path,
+any collection can run as a shell script or a batch file, with the parent
+mysql-test directory being the current working directory.
+
+During integration testing, we choose the collection to run by following
+these steps:
+
+1) We choose the extension to look for, based on these rules:
+ - If we're running a per-push test, we choose ".push" as the extension.
+ - If we're running a daily test, we choose ".daily" as the extension.
+ - If we're running a weekly test, we choose ".weekly" as the extension.
+
+2) If there is a collection that has the same name as the branch we're
+ testing plus the extension as determined in step 1, we choose that
+ collection.
+
+3) If the branch is unknown or we have removed all characters from it
+ and still not found a matching collection, we choose the name "default"
+ plus the extension determined in step 1. If there is no such file,
+ we give up and don't test anything at all.
+
+4) If we haven't found a collection yet, we remove the last character from
+ the branch name and go back to step 2.
+
+5) The commands from the collection are run line by line via execv() or
+ similar system calls. They are not run as a shell script. Shell
+ expansions are not guaranteed to work and most likely won't.
diff --git a/mysql-test/collections/README.experimental b/mysql-test/collections/README.experimental
new file mode 100644
index 00000000000..d06e55f1246
--- /dev/null
+++ b/mysql-test/collections/README.experimental
@@ -0,0 +1,25 @@
+The .experimental files in this directory contain names of test cases that
+are still in development and whose failures should be considered expected,
+instead of regressions.
+
+These files are to be used with the --experimental option of
+mysql-test-run.pl. Please look at its help screen for usage information.
+
+The syntax is as follows:
+
+1) One line per test case.
+
+2) Empty lines and lines starting with a hash (#) are ignored.
+
+3) If any other line contains a black followed by a hash (#), the hash
+ and any subsequent characters are ignored.
+
+4) The full test case name including the suite and execution mode
+ must be specified, for example:
+ main.alias 'row' # bug#00000
+
+5) As an exception to item 4, the last character of the test case
+ specification may be an asterisk (*). In that case, all test cases that
+ start with the same characters up to the last letter before the asterisk
+ are considered experimental:
+ main.a* # get rid of main.alias, main.alibaba and main.agliolio
diff --git a/mysql-test/collections/default.daily b/mysql-test/collections/default.daily
new file mode 100644
index 00000000000..194cc2aad59
--- /dev/null
+++ b/mysql-test/collections/default.daily
@@ -0,0 +1 @@
+perl mysql-test-run.pl --timer --force --comment=rpl_ndb_row --suite=rpl_ndb,ndb --mysqld=--binlog-format=row --experimental=collections/default.experimental
diff --git a/mysql-test/collections/default.experimental b/mysql-test/collections/default.experimental
new file mode 100644
index 00000000000..103069f79cf
--- /dev/null
+++ b/mysql-test/collections/default.experimental
@@ -0,0 +1 @@
+funcs_1.charset_collation_1 # depends on compile-time decisions
diff --git a/mysql-test/collections/default.push b/mysql-test/collections/default.push
new file mode 100644
index 00000000000..0879b6fde2c
--- /dev/null
+++ b/mysql-test/collections/default.push
@@ -0,0 +1,5 @@
+perl mysql-test-run.pl --timer --force --comment=n_mix --mysqld=--binlog-format=mixed --experimental=collections/default.experimental
+perl mysql-test-run.pl --timer --force --comment=ps_row --ps-protocol --mysqld=--binlog-format=row --experimental=collections/default.experimental
+perl mysql-test-run.pl --timer --force --comment=embedded --embedded --experimental=collections/default.experimental
+perl mysql-test-run.pl --timer --force --comment=rpl_binlog_row --suite=rpl,binlog --mysqld=--binlog-format=row --experimental=collections/default.experimental
+perl mysql-test-run.pl --timer --force --comment=funcs_1 --suite=funcs_1 --experimental=collections/default.experimental
diff --git a/mysql-test/collections/default.weekly b/mysql-test/collections/default.weekly
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/mysql-test/collections/default.weekly
diff --git a/mysql-test/lib/mtr_report.pm b/mysql-test/lib/mtr_report.pm
index ce3fba87385..d1cae6324cb 100644
--- a/mysql-test/lib/mtr_report.pm
+++ b/mysql-test/lib/mtr_report.pm
@@ -69,6 +69,8 @@ sub _mtr_report_test_name ($) {
print _name(), _timestamp();
printf "%-40s ", $tname;
+
+ return $tname;
}
@@ -105,20 +107,41 @@ sub mtr_report_test_passed ($) {
sub mtr_report_test ($) {
my ($tinfo)= @_;
- _mtr_report_test_name($tinfo);
+ my $test_name = _mtr_report_test_name($tinfo);
- my $comment= $tinfo->{'comment'};
- my $logfile= $tinfo->{'logfile'};
- my $warnings= $tinfo->{'warnings'};
- my $result= $tinfo->{'result'};
+ my $comment= $tinfo->{'comment'};
+ my $logfile= $tinfo->{'logfile'};
+ my $warnings= $tinfo->{'warnings'};
+ my $result= $tinfo->{'result'};
if ($result eq 'MTR_RES_FAILED'){
my $timest = format_time();
+ my $fail = "fail";
+
+ if ( $::opt_experimental )
+ {
+ # Find out if this test case is an experimental one, so we can treat
+ # the failure as an expected failure instead of a regression.
+ for my $exp ( @$::opt_experimental ) {
+ if ( $exp ne $test_name ) {
+ if ( $exp ne "" && substr($exp, -1, 1) eq "*" ) {
+ $exp = substr($exp, 0, length($exp) - 1);
+ if ( substr($test_name, 0, length($exp)) ne $exp ) {
+ next;
+ }
+ } else {
+ next;
+ }
+ }
+ $fail = "exp-fail";
+ last;
+ }
+ }
if ( $warnings )
{
- mtr_report("[ fail ] Found warnings/errors in server log file!");
+ mtr_report("[ $fail ] Found warnings/errors in server log file!");
mtr_report(" Test ended at $timest");
mtr_report($warnings);
return;
@@ -126,14 +149,14 @@ sub mtr_report_test ($) {
my $timeout= $tinfo->{'timeout'};
if ( $timeout )
{
- mtr_report("[ fail ] timeout after $timeout seconds");
+ mtr_report("[ $fail ] timeout after $timeout seconds");
mtr_report(" Test ended at $timest");
mtr_report("\n$tinfo->{'comment'}");
return;
}
else
{
- mtr_report("[ fail ]\n Test ended at $timest");
+ mtr_report("[ $fail ]\n Test ended at $timest");
}
if ( $logfile )
diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl
index 511cd4d07ba..8e772279aa3 100755
--- a/mysql-test/mysql-test-run.pl
+++ b/mysql-test/mysql-test-run.pl
@@ -169,6 +169,8 @@ our $opt_client_debugger;
my $config; # The currently running config
my $current_config_name; # The currently running config file template
+our $opt_experimental;
+
my $baseport;
my $opt_build_thread= $ENV{'MTR_BUILD_THREAD'} || "auto";
@@ -792,7 +794,7 @@ sub command_line_setup {
'big-test' => \$opt_big_test,
'combination=s' => \@opt_combinations,
'skip-combinations' => \&collect_option,
-
+ 'experimental=s' => \$opt_experimental,
'skip-im' => \&ignore_option,
# Specify ports
@@ -959,6 +961,28 @@ sub command_line_setup {
mtr_print_thick_line('#');
}
+ if ( $opt_experimental )
+ {
+ if ( open(FILE, "<", $opt_experimental) ) {
+ mtr_report("Using experimental file: $opt_experimental");
+ $opt_experimental = [];
+ while(<FILE>) {
+ chomp;
+ s/( +|^)#.*$//;
+ s/^ +//;
+ s/ +$//;
+ if ( $_ eq "" ) {
+ next;
+ }
+ print " - $_\n";
+ push @$opt_experimental, $_;
+ }
+ close FILE;
+ } else {
+ mtr_error("Can't read experimental file: $opt_experimental");
+ }
+ }
+
foreach my $arg ( @ARGV )
{
if ( $arg =~ /^--skip-/ )