diff options
author | dalyd <david.daly@mongodb.com> | 2015-09-10 16:24:07 -0400 |
---|---|---|
committer | dalyd <david.daly@mongodb.com> | 2015-09-17 08:31:07 -0400 |
commit | 91a539d8beaa48877ecf46381a79edc43037d878 (patch) | |
tree | d5c331ca84c3255dc256d8c80548a855b3afd388 /buildscripts | |
parent | 31d1c9ee2892aa8070d7141ee6105d21c647ca37 (diff) | |
download | mongo-91a539d8beaa48877ecf46381a79edc43037d878.tar.gz |
SERVER-20035: Update perf_regresison_check.py script to output report.json summarizing results
(cherry picked from commit 6fa5547c842a31712af91cf2927a631dbbcbdbf4)
Diffstat (limited to 'buildscripts')
-rw-r--r-- | buildscripts/perf_regression_check.py | 38 |
1 files changed, 32 insertions, 6 deletions
diff --git a/buildscripts/perf_regression_check.py b/buildscripts/perf_regression_check.py index 5d44b41602d..a6cf29141ba 100644 --- a/buildscripts/perf_regression_check.py +++ b/buildscripts/perf_regression_check.py @@ -110,10 +110,14 @@ def main(args): j = get_json(args.file) h = History(j) testnames = h.testnames() - failed = False + failed = 0 + results = [] for test in testnames: + # The first entry is valid. The rest is dummy data to match the existing format + result = {'test_file' : test, 'exit_code' : 0, 'elapsed' : 5, 'start': 1441227291.962453, 'end': 1441227293.428761} this_one = h.seriesAtRevision(test, args.rev) + testFailed = False print "checking %s.." % (test) if not this_one: print "\tno data at this revision, skipping" @@ -128,17 +132,39 @@ def main(args): continue if compareResults(this_one, previous[0], args.threshold, "Previous", h.noiseLevels(test), args.noise, args.threadThreshold, args.threadNoise): - failed = True + testFailed = True + result['PreviousCompare'] = 'fail' + else : + result['PreviousCompare'] = 'pass' + daysprevious = h.seriesItemsNDaysBefore(test, args.rev,args.ndays) reference = h.seriesAtRevision(test, args.reference) if compareResults(this_one, daysprevious, args.threshold, "NDays", h.noiseLevels(test), args.noise, args.threadThreshold, args.threadNoise): - failed = True + testFailed = True + result['NDayCompare'] = 'fail' + else : + result['NDayCompare'] = 'pass' if compareResults(this_one, reference, args.threshold, "Reference", h.noiseLevels(test), args.noise, args.threadThreshold, args.threadNoise): - failed = True - - if failed: + testFailed = True + result['BaselineCompare'] = 'fail' + else : + result['BaselineCompare'] = 'pass' + if testFailed : + result['status'] = 'fail' + failed += 1 + else : + result['status'] = 'pass' + results.append(result) + + report = {} + report['failures'] = failed + report['results'] = results + + reportFile = open('report.json', 'w') + json.dump(report, reportFile, indent=4, separators=(',', ': ')) + if failed > 0 : sys.exit(1) else: sys.exit(0) |