diff options
Diffstat (limited to 'contrib/testsuite-management')
| -rwxr-xr-x | contrib/testsuite-management/validate_failures.py | 95 | ||||
| -rw-r--r-- | contrib/testsuite-management/x86_64-unknown-linux-gnu.xfail | 78 |
2 files changed, 146 insertions, 27 deletions
diff --git a/contrib/testsuite-management/validate_failures.py b/contrib/testsuite-management/validate_failures.py index c07be84cd03..f231e93f835 100755 --- a/contrib/testsuite-management/validate_failures.py +++ b/contrib/testsuite-management/validate_failures.py @@ -1,4 +1,4 @@ -#!/usr/bin/python2.6 +#!/usr/bin/python # Script to compare testsuite failures against a list of known-to-fail # tests. @@ -46,6 +46,7 @@ executed it will: with exit code 0. Otherwise, it exits with error code 1. """ +import datetime import optparse import os import re @@ -135,15 +136,37 @@ class TestResult(object): attrs = '%s | ' % self.attrs return '%s%s: %s %s' % (attrs, self.state, self.name, self.description) + def ExpirationDate(self): + # Return a datetime.date object with the expiration date for this + # test result. Return None, if no expiration has been set. + if re.search(r'expire=', self.attrs): + expiration = re.search(r'expire=(\d\d\d\d)(\d\d)(\d\d)', self.attrs) + if not expiration: + Error('Invalid expire= format in "%s". Must be of the form ' + '"expire=YYYYMMDD"' % self) + return datetime.date(int(expiration.group(1)), + int(expiration.group(2)), + int(expiration.group(3))) + return None + + def HasExpired(self): + # Return True if the expiration date of this result has passed. + expiration_date = self.ExpirationDate() + if expiration_date: + now = datetime.date.today() + return now > expiration_date + def GetMakefileValue(makefile_name, value_name): if os.path.exists(makefile_name): - with open(makefile_name) as makefile: - for line in makefile: - if line.startswith(value_name): - (_, value) = line.split('=', 1) - value = value.strip() - return value + makefile = open(makefile_name) + for line in makefile: + if line.startswith(value_name): + (_, value) = line.split('=', 1) + value = value.strip() + makefile.close() + return value + makefile.close() return None @@ -173,10 +196,18 @@ def IsInterestingResult(line): def ParseSummary(sum_fname): """Create a set of TestResult instances from the given summary file.""" result_set = set() - with open(sum_fname) as sum_file: - for line in sum_file: - if IsInterestingResult(line): - result_set.add(TestResult(line)) + sum_file = open(sum_fname) + for line in sum_file: + if IsInterestingResult(line): + result = TestResult(line) + if result.HasExpired(): + # Tests that have expired are not added to the set of expected + # results. If they are still present in the set of actual results, + # they will cause an error to be reported. + print 'WARNING: Expected failure "%s" has expired.' % line.strip() + continue + result_set.add(result) + sum_file.close() return result_set @@ -195,7 +226,7 @@ def GetManifest(manifest_name): return set() -def GetSumFiles(builddir): +def CollectSumFiles(builddir): sum_files = [] for root, dirs, files in os.walk(builddir): if '.svn' in dirs: @@ -217,16 +248,20 @@ def GetResults(sum_files): def CompareResults(manifest, actual): """Compare sets of results and return two lists: - - List of results present in MANIFEST but missing from ACTUAL. - List of results present in ACTUAL but missing from MANIFEST. + - List of results present in MANIFEST but missing from ACTUAL. """ - # Report all the actual results not present in the manifest. + # Collect all the actual results not present in the manifest. + # Results in this set will be reported as errors. actual_vs_manifest = set() for actual_result in actual: if actual_result not in manifest: actual_vs_manifest.add(actual_result) - # Simlarly for all the tests in the manifest. + # Collect all the tests in the manifest that were not found + # in the actual results. + # Results in this set will be reported as warnings (since + # they are expected failures that are not failing anymore). manifest_vs_actual = set() for expected_result in manifest: # Ignore tests marked flaky. @@ -255,6 +290,16 @@ def PrintSummary(msg, summary): print result +def GetSumFiles(results, build_dir): + if not results: + print 'Getting actual results from build' + sum_files = CollectSumFiles(build_dir) + else: + print 'Getting actual results from user-provided results' + sum_files = results.split() + return sum_files + + def CheckExpectedResults(options): if not options.manifest: (srcdir, target, valid_build) = GetBuildData(options) @@ -268,13 +313,7 @@ def CheckExpectedResults(options): print 'Manifest: %s' % manifest_name manifest = GetManifest(manifest_name) - - if not options.results: - print 'Getting actual results from build' - sum_files = GetSumFiles(options.build_dir) - else: - print 'Getting actual results from user-provided results' - sum_files = options.results.split() + sum_files = GetSumFiles(options.results, options.build_dir) actual = GetResults(sum_files) if options.verbosity >= 1: @@ -311,11 +350,13 @@ def ProduceManifest(options): Error('Manifest file %s already exists.\nUse --force to overwrite.' % manifest_name) - actual = GetResults(options.build_dir) - with open(manifest_name, 'w') as manifest_file: - for result in sorted(actual): - print result - manifest_file.write('%s\n' % result) + sum_files = GetSumFiles(options.results, options.build_dir) + actual = GetResults(sum_files) + manifest_file = open(manifest_name, 'w') + for result in sorted(actual): + print result + manifest_file.write('%s\n' % result) + manifest_file.close() return True diff --git a/contrib/testsuite-management/x86_64-unknown-linux-gnu.xfail b/contrib/testsuite-management/x86_64-unknown-linux-gnu.xfail new file mode 100644 index 00000000000..c823fb20fbb --- /dev/null +++ b/contrib/testsuite-management/x86_64-unknown-linux-gnu.xfail @@ -0,0 +1,78 @@ +FAIL: gcc.dg/attr-weakref-1.c (test for excess errors) +FAIL: gcc.dg/torture/pr51106-2.c -O0 (internal compiler error) +FAIL: gcc.dg/torture/pr51106-2.c -O0 (test for excess errors) +FAIL: gcc.dg/torture/pr51106-2.c -O1 (internal compiler error) +FAIL: gcc.dg/torture/pr51106-2.c -O1 (test for excess errors) +FAIL: gcc.dg/torture/pr51106-2.c -O2 (internal compiler error) +FAIL: gcc.dg/torture/pr51106-2.c -O2 (test for excess errors) +FAIL: gcc.dg/torture/pr51106-2.c -O2 -flto -fno-use-linker-plugin -flto-partition=none (internal compiler error) +FAIL: gcc.dg/torture/pr51106-2.c -O2 -flto -fno-use-linker-plugin -flto-partition=none (test for excess errors) +FAIL: gcc.dg/torture/pr51106-2.c -O3 -fomit-frame-pointer (internal compiler error) +FAIL: gcc.dg/torture/pr51106-2.c -O3 -fomit-frame-pointer (test for excess errors) +FAIL: gcc.dg/torture/pr51106-2.c -O3 -g (internal compiler error) +FAIL: gcc.dg/torture/pr51106-2.c -O3 -g (test for excess errors) +FAIL: gcc.dg/torture/pr51106-2.c -Os (internal compiler error) +FAIL: gcc.dg/torture/pr51106-2.c -Os (test for excess errors) +FAIL: gfortran.dg/lto/pr45586 f_lto_pr45586_0.o-f_lto_pr45586_0.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin (internal compiler error) +FAIL: gfortran.dg/lto/pr45586 f_lto_pr45586_0.o-f_lto_pr45586_0.o link, -O0 -flto -flto-partition=none -fuse-linker-plugin (internal compiler error) +FAIL: gfortran.dg/lto/pr45586 f_lto_pr45586_0.o-f_lto_pr45586_0.o link, -O0 -flto -fuse-linker-plugin -fno-fat-lto-objects (internal compiler error) +FAIL: gfortran.dg/lto/pr45586-2 f_lto_pr45586-2_0.o-f_lto_pr45586-2_0.o link, -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin (internal compiler error) +FAIL: gfortran.dg/lto/pr45586-2 f_lto_pr45586-2_0.o-f_lto_pr45586-2_0.o link, -O0 -flto -flto-partition=none -fuse-linker-plugin (internal compiler error) +FAIL: gfortran.dg/lto/pr45586-2 f_lto_pr45586-2_0.o-f_lto_pr45586-2_0.o link, -O0 -flto -fuse-linker-plugin -fno-fat-lto-objects (internal compiler error) +FAIL: gnat.dg/array11.adb (test for warnings, line 12) +FAIL: gnat.dg/object_overflow.adb (test for warnings, line 8) +FAIL: libmudflap.c++/pass55-frag.cxx ( -O) execution test +FAIL: libmudflap.c++/pass55-frag.cxx (-O2) execution test +FAIL: libmudflap.c++/pass55-frag.cxx (-O3) execution test +UNRESOLVED: gcc.dg/attr-weakref-1.c compilation failed to produce executable +UNRESOLVED: gfortran.dg/lto/pr45586 f_lto_pr45586_0.o-f_lto_pr45586_0.o execute -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin +UNRESOLVED: gfortran.dg/lto/pr45586 f_lto_pr45586_0.o-f_lto_pr45586_0.o execute -O0 -flto -flto-partition=none -fuse-linker-plugin +UNRESOLVED: gfortran.dg/lto/pr45586 f_lto_pr45586_0.o-f_lto_pr45586_0.o execute -O0 -flto -fuse-linker-plugin -fno-fat-lto-objects +UNRESOLVED: gfortran.dg/lto/pr45586-2 f_lto_pr45586-2_0.o-f_lto_pr45586-2_0.o execute -O0 -flto -flto-partition=1to1 -fno-use-linker-plugin +UNRESOLVED: gfortran.dg/lto/pr45586-2 f_lto_pr45586-2_0.o-f_lto_pr45586-2_0.o execute -O0 -flto -flto-partition=none -fuse-linker-plugin +UNRESOLVED: gfortran.dg/lto/pr45586-2 f_lto_pr45586-2_0.o-f_lto_pr45586-2_0.o execute -O0 -flto -fuse-linker-plugin -fno-fat-lto-objects +XPASS: gcc.dg/Wstrict-overflow-18.c (test for bogus messages, line 20) +XPASS: gcc.dg/guality/example.c -O0 execution test +XPASS: gcc.dg/guality/example.c -O1 execution test +XPASS: gcc.dg/guality/example.c -O2 execution test +XPASS: gcc.dg/guality/example.c -O2 -flto -fno-use-linker-plugin -flto-partition=none execution test +XPASS: gcc.dg/guality/example.c -Os execution test +XPASS: gcc.dg/guality/guality.c -O0 execution test +XPASS: gcc.dg/guality/guality.c -O1 execution test +XPASS: gcc.dg/guality/guality.c -O2 execution test +XPASS: gcc.dg/guality/guality.c -O2 -flto -fno-use-linker-plugin -flto-partition=none execution test +XPASS: gcc.dg/guality/guality.c -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects execution test +XPASS: gcc.dg/guality/guality.c -O3 -fomit-frame-pointer execution test +XPASS: gcc.dg/guality/guality.c -O3 -g execution test +XPASS: gcc.dg/guality/guality.c -Os execution test +XPASS: gcc.dg/guality/inline-params.c -O2 execution test +XPASS: gcc.dg/guality/inline-params.c -O2 -flto -fno-use-linker-plugin -flto-partition=none execution test +XPASS: gcc.dg/guality/inline-params.c -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects execution test +XPASS: gcc.dg/guality/inline-params.c -O3 -fomit-frame-pointer execution test +XPASS: gcc.dg/guality/inline-params.c -O3 -g execution test +XPASS: gcc.dg/guality/inline-params.c -Os execution test +XPASS: gcc.dg/guality/pr41447-1.c -O0 execution test +XPASS: gcc.dg/guality/pr41447-1.c -O1 execution test +XPASS: gcc.dg/guality/pr41447-1.c -O2 execution test +XPASS: gcc.dg/guality/pr41447-1.c -O2 -flto -fno-use-linker-plugin -flto-partition=none execution test +XPASS: gcc.dg/guality/pr41447-1.c -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects execution test +XPASS: gcc.dg/guality/pr41447-1.c -O3 -fomit-frame-pointer execution test +XPASS: gcc.dg/guality/pr41447-1.c -O3 -g execution test +XPASS: gcc.dg/guality/pr41447-1.c -Os execution test +XPASS: gcc.dg/guality/pr41616-1.c -O0 execution test +XPASS: gcc.dg/guality/pr41616-1.c -O1 execution test +XPASS: gcc.dg/guality/pr41616-1.c -O2 execution test +XPASS: gcc.dg/guality/pr41616-1.c -O2 -flto -fno-use-linker-plugin -flto-partition=none execution test +XPASS: gcc.dg/guality/pr41616-1.c -O3 -fomit-frame-pointer execution test +XPASS: gcc.dg/guality/pr41616-1.c -O3 -g execution test +XPASS: gcc.dg/guality/pr41616-1.c -Os execution test +XPASS: gcc.dg/inline_3.c (test for excess errors) +XPASS: gcc.dg/inline_4.c (test for excess errors) +XPASS: gcc.dg/unroll_2.c (test for excess errors) +XPASS: gcc.dg/unroll_3.c (test for excess errors) +XPASS: gcc.dg/unroll_4.c (test for excess errors) +FAIL: gcc.dg/pr52558-1.c scan-tree-dump-times lim1 "MEM count_lsm.. count_lsm_flag" 1 +FAIL: gcc.dg/pr52558-2.c scan-tree-dump-times lim1 "MEM.*g_2_lsm_flag" 1 +FAIL: gcc.dg/tm/reg-promotion.c scan-tree-dump-times lim1 "MEM count_lsm.. count_lsm_flag" 1 +FAIL: go.test/test/stack.go execution, -O2 -g +FAIL: gcc.target/i386/pad-10.c scan-assembler-not nop |