MDEV-10604 Create a list of unstable MTR tests to be disabled in distribution buildsmariadb-10.0.27

- mysql-test/unstable-tests list is created, it includes
  = tests identified as unstable by Debian;
  = tests which failed in buildbot on 10.0 over the last ~6 months and were not fixed;
  = tests which have been recently modified or newly added
- '*' wildcard is now supported in skip lists

2 files changed, 274 insertions, 3 deletions

diff --git a/mysql-test/lib/mtr_cases.pm b/mysql-test/lib/mtr_cases.pm
index b76b10d42ed..10e5fd5c337 100644
--- a/mysql-test/lib/mtr_cases.pm
+++ b/mysql-test/lib/mtr_cases.pm
@@ -313,6 +313,7 @@ sub combinations_from_file($$)
 }
 
 our %disabled;
+our %disabled_wildcards;
 sub parse_disabled {
   my ($filename, $suitename) = @_;
 
@@ -321,10 +322,18 @@ sub parse_disabled {
       chomp;
       next if /^\s*#/ or /^\s*$/;
       mtr_error("Syntax error in $filename line $.")
-        unless /^\s*(?:([-0-9A-Za-z_\/]+)\.)?([-0-9A-Za-z_]+)\s*:\s*(.*?)\s*$/;
-      mtr_error("Wrong suite name in $filename line $.")
+        unless /^\s*(?:([-0-9A-Za-z_\/]+)\.)?([-0-9A-Za-z_\*]+)\s*:\s*(.*?)\s*$/;
+      mtr_error("Wrong suite name in $filename line $.: suitename = $suitename but the file says $1")
         if defined $1 and defined $suitename and $1 ne $suitename;
-      $disabled{($1 || $suitename || '') . ".$2"} = $3;
+      my ($sname, $casename, $text)= (($1 || $suitename || ''), $2, $3);
+
+      if ($casename =~ /\*/) {
+        # Wildcard
+        $disabled_wildcards{$sname . ".$casename"}= $text;
+      }
+      else {
+        $disabled{$sname . ".$casename"}= $text;
+      }
     }
     close DISABLED;
   }
@@ -721,6 +730,14 @@ sub collect_one_test_case {
   # Check for disabled tests
   # ----------------------------------------------------------------------
   my $disable = $disabled{".$tname"} || $disabled{$name};
+  if (not $disable) {
+    foreach my $w (keys %disabled_wildcards) {
+      if ($name =~ /^$w/) {
+        $disable= $disabled_wildcards{$w};
+        last;
+      }
+    }
+  }
   if (not defined $disable and $suite->{parent}) {
     $disable = $disabled{$suite->{parent}->{name} . ".$tname"};
   }
diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests
new file mode 100644
index 00000000000..6a46602eb07
--- /dev/null
+++ b/mysql-test/unstable-tests
@@ -0,0 +1,254 @@
+##############################################################################
+#
+#  List the test cases which, unlike tests from disabled.def files,
+#  can still be run on the current tree meaningfully, but are known 
+#  or suspected to fail sporadically on different reasons.
+#
+#  Most common reasons are either test failures observed in buildbot,
+#  or recent modifications to the tests which make their stability 
+#  unknown. 
+#
+#  Tests included due to recent modifications are later removed from the 
+#  list, if during a certain period they do not fail (and are not 
+#  modified again). Tests included due to intermittent failures are 
+#  removed when corresponding bug reports are closed. 
+#
+#  Separate the test case name and the comment with ':'.
+#
+#    <suitename>.<testcasename> : MDEV-xxxxx - <comment>
+#
+#  '*' wildcard in testcase names is supported.
+#
+#  To use the list, run MTR with --skip-test-list=unstable-tests option. 
+#
+##############################################################################
+
+main.bootstrap                     : Modified on 2016-06-18 (MDEV-9969)
+main.create_delayed                : MDEV-10605 - failed with timeout
+main.create_or_replace             : Modified on 2016-06-23 (MDEV-9728)
+main.ctype_recoding                : Modified on 2016-06-10 (MDEV-10181)
+main.ctype_utf8                    : Modified on 2016-06-21 (merge)
+main.ctype_utf8mb4                 : Modified on 2016-06-21 (merge)
+main.events_1                      : Modified on 2016-06-21 (MDEV-9524)
+main.func_group                    : Modified on 2016-08-08 (MDEV-10468)
+main.func_in                       : Modified on 2016-06-20 (MDEV-10020)
+main.func_math                     : Modified on 2016-08-10 (merge)
+main.func_misc                     : Modified on 2016-08-10 (merge)
+main.grant2                        : Modified on 2016-07-18 (MDEV-8569)
+main.help                          : Modified on 2016-06-21 (MDEV-9524)
+main.host_cache_size_functionality : MDEV-10606 - sporadic failure on shutdown
+main.index_intersect_innodb        : MDEV-10643 - failed with timeout
+main.index_merge_innodb            : MDEV-7142 - sporadic wrong execution plan
+main.information_schema_stats      : Modified on 2016-07-25 (MDEV-10428)
+main.innodb_mysql_lock             : MDEV-7861 - sporadic lock detection failure
+main.insert_innodb                 : Modified on 2016-06-14 (merge from upstream)
+main.loaddata                      : Modified on 2016-08-10 (merge)
+main.locale                        : Modified on 2016-06-21 (merge)
+main.mdev-504                      : MDEV-10607 - sporadic "can't connect"
+main.mdev375                       : MDEV-10607 - sporadic "can't connect"
+main.merge                         : MDEV-10607 - sporadic "can't connect"
+main.multi_update                  : Modified on 2016-06-20 (MDEV-5973)
+main.myisam_enable_keys-10506      : New test, added on 2016-08-10 (MDEV-10506)
+main.mysqlcheck                    : Modified on 2016-08-10 (merge)
+main.mysqldump                     : MDEV-10512 - sporadic assertion failure
+main.mysqltest                     : MDEV-9269 - fails on Alpha
+main.named_pipe                    : Modified on 2016-08-02 (MDEV-10383)
+main.openssl_1                     : Modified on 2016-07-11 (MDEV-10211)
+main.parser                        : Modified on 2016-06-21 (merge)
+main.pool_of_threads               : MDEV-10100 - sporadic error on detecting max connections
+main.ps_1general                   : Modified on 2016-07-12 (merge)
+main.range                         : Modified on 2016-08-10 (merge)
+main.range_mrr_icp                 : Modified on 2016-08-10 (merge)
+main.query_cache                   : MDEV-10611 - sporadic mutex problem
+main.shutdown                      : MDEV-10612 - sporadic crashes
+main.sp-prelocking                 : Modified on 2016-08-10 (merge)
+main.sp-security                   : MDEV-10607 - sporadic "can't connect"
+main.ssl                           : MDEV-10211 - different ciphers on some platforms
+main.ssl_ca                        : Modified on 2016-07-11 (MDEV-10211)
+main.ssl_compress                  : Modified on 2016-07-11 (MDEV-10211)
+main.ssl_timeout                   : Modified on 2016-07-11 (MDEV-10211)
+main.stat_tables_par_innodb        : MDEV-10515 - sporadic wrong results
+main.status_user                   : Modified on 2016-06-20 (MDEV-8633)
+main.subselect_innodb              : MDEV-10614 - sporadic wrong results
+main.temp_table                    : Modified on 2016-06-18 (MDEV-8569)
+main.type_date                     : Modified on 2016-08-10 (merge)
+main.type_datetime                 : Modified on 2016-06-16 (MDEV-9374)
+main.view                          : Modified on 2016-08-10 (merge)
+main.xtradb_mrr                    : Modified on 2016-08-04 (MDEV-9946)
+
+#----------------------------------------------------------------
+
+archive.archive-big : MDEV-10615 - table is marked as crashed
+archive.discover    : MDEV-10510 - table is marked as crashed
+
+#----------------------------------------------------------------
+
+binlog.binlog_commit_wait                 : MDEV-10150 - Error: too much time elapsed
+binlog.binlog_dmls_on_tmp_tables_readonly : New test, added on 2016-05-04 (upstream)
+binlog.binlog_xa_recover                  : MDEV-8517 - Extra checkpoint
+
+#----------------------------------------------------------------
+
+connect.tbl             : MDEV-9844, MDEV-10179 - sporadic crashes, valgrind warnings, wrong results
+connect.jdbc            : New test, added on 2016-07-15
+connect.jdbc-new        : New test, added on 2016-07-14
+connect.jdbc-oracle     : New test, added on 2016-07-13
+connect.jdbc-postgresql : New test, added on 2016-07-13
+
+#----------------------------------------------------------------
+
+federated.federatedx             : MDEV-10617 - Wrong checksum, timeouts
+federated.federated_innodb       : MDEV-10617, MDEV-10417 - Wrong checksum, timeouts, fails on Mips
+federated.federated_partition    : MDEV-10417 - Fails on Mips
+federated.federated_transactions : MDEV-10617, MDEV-10417 - Wrong checksum, timeouts, fails on Mips
+
+#----------------------------------------------------------------
+
+funcs_1.processlist_priv_no_prot : Include file modified on 2016-07-12 (merge)
+funcs_1.processlist_priv_ps      : Include file modified on 2016-07-12 (merge)
+
+#----------------------------------------------------------------
+
+innodb.binlog_consistent              : MDEV-10618 - Server fails to start 
+innodb.innodb-alter-table             : MDEV-10619 - Testcase timeout
+innodb.innodb-alter-tempfile          : Modified on 2016-08-09 (MDEV-10469)
+innodb.innodb_corrupt_bit             : Modified on 2016-06-21 (merge)
+innodb.innodb_bug30423                : MDEV-7311 - Wrong number of rows in the plan
+innodb.innodb-fk-warnings             : Modified on 2016-07-18 (MDEV-8569)
+innodb.innodb-fkcheck                 : Modified on 2016-06-13 (MDEV-10083)
+innodb.innodb-wl5522                  : rdiff file modified on 2016-08-10 (merge)
+innodb.innodb-wl5522-debug-zip        : MDEV-10427 - Warning: database page corruption
+
+#----------------------------------------------------------------
+
+mroonga/storage.column_datetime_32bit_2038                         : Wrong result on Alpha
+mroonga/storage.column_datetime_32bit_before_unix_epoch            : Wrong result on Alpha
+mroonga/storage.column_datetime_32bit_max                          : Wrong result on Alpha
+mroonga/storage.column_datetime_32bit_out_of_range                 : Wrong result on Alpha
+mroonga/storage.index_multiple_column_unique_date_32bit_equal      : Wrong result on Alpha
+mroonga/storage.index_multiple_column_unique_date_order_32bit_desc : Wrong result on Alpha
+
+#----------------------------------------------------------------
+
+multi_source.gtid        : MDEV-10620, MDEV-10417 - Timeout in wait condition, fails on Mips
+multi_source.multisource : MDEV-10417 - Fails on Mips
+multi_source.simple      : MDEV-4633 - Wrong slave status output
+multi_source.status_vars : MDEV-4632 - failed while waiting for Slave_received_heartbeats
+
+#----------------------------------------------------------------
+
+parts.partition_float_myisam : MDEV-10621 - Testcase timeout
+parts.partition_int_myisam   : MDEV-10621 - Testcase timeout
+
+#----------------------------------------------------------------
+
+perfschema.digest_table_full                 : Modified on 2016-06-21 (merge)
+perfschema.func_file_io                      : MDEV-5708 - fails for s390x
+perfschema.func_mutex                        : MDEV-5708 - fails for s390x
+perfschema.rpl_gtid_func                     : Modified on 2016-06-21 (merge)
+perfschema.sizing_low                        : Modified on 2016-04-26 (5.6.30 merge)
+perfschema.socket_summary_by_event_name_func : MDEV-10622 - Socket summary tables do not match
+perfschema.start_server_low_digest           : Modified on 2016-06-21 (merge)
+perfschema.statement_digest                  : Modified on 2016-06-21 (merge)
+perfschema.statement_digest_consumers        : Modified on 2016-06-21 (merge)
+perfschema.statement_digest_long_query       : Modified on 2016-06-21 (merge)
+perfschema.table_name                        : New test, added on 2016-04-26 (5.6.30 merge)
+
+#----------------------------------------------------------------
+
+plugins.feedback_plugin_send     : MDEV-7932 - ssl failed for url
+plugins.pam                      : Modified on 2016-08-03 (MDEV-7329)
+plugins.pam_cleartext            : Modified on 2016-08-03
+plugins.server_audit             : MDEV-9562 - crashes on sol10-sparc
+plugins.thread_pool_server_audit : MDEV-9562 - crashes on sol10-sparc
+
+#----------------------------------------------------------------
+
+roles.rpl_grant_revoke_current_role-8638 : New test, added on 2016-06-20 (MDEV-8638)
+roles.set_role-9614                      : New test, added on 2016-05-30 (MDEV-9614)
+
+#----------------------------------------------------------------
+
+rpl.last_insert_id                    : MDEV-10625 - warnings in error log
+rpl.rpl_auto_increment                : MDEV-10417 - Fails on Mips
+rpl.rpl_auto_increment_bug45679       : MDEV-10417 - Fails on Mips
+rpl.rpl_auto_increment_update_failure : MDEV-10625 - warnings in error log
+rpl.rpl_binlog_index                  : MDEV-9501 - Warning: failed registering on master
+rpl.rpl_checksum_cache                : MDEV-10626 - Testcase timeout
+rpl.rpl_circular_for_4_hosts          : MDEV-10627 - Testcase timeout
+rpl.rpl_ddl                           : MDEV-10417 - Fails on Mips
+rpl.rpl_gtid_crash                    : MDEV-9501 - Warning: failed registering on master
+rpl.rpl_gtid_master_promote           : MDEV-10628 - Timeout in sync_with_master
+rpl.rpl_gtid_stop_start               : MDEV-10629 - Crash on shutdown
+rpl.rpl_gtid_until                    : MDEV-10625 - warnings in error log
+rpl.rpl_ignore_table                  : Modified on 2016-06-22
+rpl.rpl_innodb_bug30888               : MDEV-10417 - Fails on Mips
+rpl.rpl_insert                        : MDEV-9329 - Fails on Ubuntu/s390x
+rpl.rpl_insert_delayed                : MDEV-9329 - Fails on Ubuntu/s390x
+rpl.rpl_invoked_features              : MDEV-10417 - Fails on Mips
+rpl.rpl_mdev6020                      : MDEV-10630, MDEV-10417 - Timeouts, fails on Mips
+rpl.rpl_mdev6386                      : MDEV-10631 - Wrong result on slave
+rpl.rpl_parallel                      : MDEV-10632, MDEV-10653 - Failures to sync, timeouts
+rpl.rpl_parallel_temptable            : MDEV-10356 - Crash in close_thread_tables
+rpl.rpl_partition_innodb              : MDEV-10417 - Fails on Mips
+rpl.rpl_row_drop_create_temp_table    : MDEV-10626 - Testcase timeout
+rpl.rpl_row_sp001                     : MDEV-9329 - Fails on Ubuntu/s390x
+rpl.rpl_semi_sync_uninstall_plugin    : MDEV-7140 - Wrong plugin status
+rpl.rpl_slave_grp_exec                : MDEV-10514 - Unexpected deadlock
+rpl.rpl_switch_stm_row_mixed          : MDEV-10611 - Wrong usage of mutex
+rpl.rpl_sync                          : MDEV-10633 - Database page corruption
+rpl.rpl_temporary_error2              : MDEV-10634 - Wrong number of retries
+rpl.sec_behind_master-5114            : MDEV-8518 - Wrong value of Seconds_Behind_Master
+rpl.rpl_skip_replication              : MDEV-9268 - Fails with timeout in sync_slave_with_master on Alpha
+
+#----------------------------------------------------------------
+
+spider.* : MDEV-9329 - tests are too memory-consuming
+
+spider/bg.direct_aggregate      : MDEV-7098 - Trying to unlock mutex that wasn't locked
+spider/bg.direct_aggregate_part : MDEV-7098 - Trying to unlock mutex that wasn't locked
+spider/bg.ha                    : MDEV-7914, MDEV-9329 - Crash, failures on s390x
+spider/bg.ha_part               : MDEV-9329 - Fails on Ubuntu/s390x
+spider/bg.spider_fixes          : MDEV-7098, MDEV-9329 - Mutex problem, failures on s390x
+spider/bg.vp_fixes              : MDEV-9329 - Fails on Ubuntu/s390x
+
+#----------------------------------------------------------------
+
+stress.ddl_innodb : MDEV-10635 - Testcase timeout
+
+#----------------------------------------------------------------
+
+sys_vars.autocommit_func2                  : MDEV-9329 - Fails on Ubuntu/s390x
+sys_vars.general_log_file_basic            : Modified on 2016-08-09 (MDEV-10465)
+sys_vars.slow_query_log_file_basic         : Modified on 2016-08-09 (MDEV-10465)
+sys_vars.innodb_buffer_pool_dump_pct_basic : MDEV-10651 - sporadic failure on file_exists
+
+#----------------------------------------------------------------
+
+tokudb.background_job_manager        : MDEV-10327 - Assertion failure on server shutdown
+tokudb.cluster_filter_unpack_varchar : MDEV-10636 - Wrong execution plan
+tokudb.*                             : MDEV-9891 - massive crashes on shutdown
+tokudb_alter_table.*                 : MDEV-9891 - massive crashes on shutdown
+tokudb_bugs.checkpoint_lock          : MDEV-10637 - Wrong processlist output
+tokudb_bugs.checkpoint_lock_3        : MDEV-10637 - Wrong processlist output
+tokudb_bugs.*                        : MDEV-9891 - massive crashes on shutdown
+tokudb_parts.*                       : MDEV-9891 - massive crashes on shutdown
+rpl-tokudb.*                         : MDEV-9891 - massive crashes on shutdown, also modified on 2016-06-10 (Merge)
+tokudb/tokudb_add_index.*            : MDEV-9891 - massive crashes on shutdown
+tokudb/tokudb_backup.*               : MDEV-9891 - massive crashes on shutdown
+tokudb/tokudb_mariadb.*              : MDEV-9891 - massive crashes on shutdown
+tokudb/tokudb_sys_vars.*             : MDEV-9891 - massive crashes on shutdown
+tokudb/tokudb_rpl.*                  : MDEV-9891 - massive crashes on shutdown
+
+
+#----------------------------------------------------------------
+
+unit.ma_test_loghandler : MDEV-10638 - record read not ok
+
+#----------------------------------------------------------------
+
+vcol.charsets         : Added on 2016-06-23
+vcol.not_supported    : MDEV-10639 - Testcase timeout
+vcol.vcol_keys_innodb : MDEV-10639 - Testcase timeout
+
+#----------------------------------------------------------------