Add a script for calibrating settings for timing configurations.

Update the timings scripts with calibrated settings that run
a full build between 9.5 and 10.0 seconds on the buildbot slave.

5 files changed, 142 insertions, 22 deletions

diff --git a/QMTest/TestSCons.py b/QMTest/TestSCons.py
index 002baaae..c1726325 100644
--- a/QMTest/TestSCons.py
+++ b/QMTest/TestSCons.py
@@ -1011,7 +1011,9 @@ class TimeSCons(TestSCons):
                 self.variables[variable] = value
             del kw['variables']
 
-        if not kw.has_key('verbose'):
+        self.calibrate = os.environ.get('TIMESCONS_CALIBRATE', '0') != '0'
+
+        if not kw.has_key('verbose') and not self.calibrate:
             kw['verbose'] = True
 
         # TODO(1.5)
@@ -1048,8 +1050,11 @@ class TimeSCons(TestSCons):
             for variable, value in self.variables.items():
                 options.append('%s=%s' % (variable, value))
             kw['options'] = ' '.join(options)
-        calibrate = os.environ.get('TIMESCONS_CALIBRATE')
-        if calibrate in (None, '0'):
+        if self.calibrate:
+            # TODO(1.5)
+            #self.calibration(*args, **kw)
+            apply(self.calibration, args, kw)
+        else:
             # TODO(1.5)
             #self.help(*args, **kw)
             #self.full(*args, **kw)
@@ -1057,10 +1062,6 @@ class TimeSCons(TestSCons):
             apply(self.help, args, kw)
             apply(self.full, args, kw)
             apply(self.null, args, kw)
-        else:
-            # TODO(1.5)
-            #self.calibration(*args, **kw)
-            apply(self.calibration, args, kw)
 
     def trace(self, graph, name, value, units, sort=None):
         fmt = "TRACE: graph=%s name=%s value=%s units=%s"
diff --git a/bin/calibrate.py b/bin/calibrate.py
new file mode 100644
index 00000000..c1b4f115
--- /dev/null
+++ b/bin/calibrate.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2009 The SCons Foundation
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import optparse
+import os
+import re
+import subprocess
+import sys
+
+variable_re = re.compile('^VARIABLE: (.*)$', re.M)
+elapsed_re = re.compile('^ELAPSED: (.*)$', re.M)
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv
+
+    parser = optparse.OptionParser(usage="calibrate.py [-h] [--min time] [--max time] timings/*/*-run.py")
+    parser.add_option('--min', type='float', default=9.5,
+                      help="minimum acceptable execution time (default 9.5)")
+    parser.add_option('--max', type='float', default=10.00,
+                      help="maximum acceptable execution time (default 10.00)")
+    opts, args = parser.parse_args(argv[1:])
+
+    os.environ['TIMESCONS_CALIBRATE'] = '1'
+
+    for arg in args:
+        if len(args) > 1:
+            print arg + ':'
+
+        command = [sys.executable, 'runtest.py', '--noqmtest', arg]
+
+        run = 1
+        good = 0
+        while good < 3:
+            p = subprocess.Popen(command,
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.STDOUT)
+            output = p.communicate()[0]
+            vm = variable_re.search(output)
+            em = elapsed_re.search(output)
+            elapsed = float(em.group(1))
+            print "run %3d: %7.3f:  %s" % (run, elapsed, ' '.join(vm.groups()))
+            if opts.min < elapsed and elapsed < opts.max:
+                good += 1
+            else:
+                good = 0
+                for v in vm.groups():
+                    var, value = v.split('=', 1)
+                    value = int((int(value) * opts.max) / elapsed)
+                    os.environ[var] = str(value)
+            run += 1
+
+    return 0
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/timings/CPPPATH/TimeSCons-run.py b/timings/CPPPATH/TimeSCons-run.py
index dcee7c5c..97d9e053 100644
--- a/timings/CPPPATH/TimeSCons-run.py
+++ b/timings/CPPPATH/TimeSCons-run.py
@@ -24,15 +24,28 @@
 """
 This configuration times searching long lists of CPPPATH directories.
 
-We create 5000 on-disk directories.  A single checked-in .h file exists
-in the 'include' directory.  The SConstruct sets CPPPATH to a list of Dir
-Nodes for the created directories, followed by 'include'.  A checked-in .c
-file #includes the .h file to be found in the last directory in the list.
+We create $DIR_COUNT on-disk directories.  A single checked-in .h file
+exists in the 'include' directory.  The SConstruct sets CPPPATH to a
+list of Dir Nodes for the created directories, followed by 'include'.
+A checked-in .c file #includes the .h file to be found in the last
+directory in the list.
 """
 
 import TestSCons
 
-test = TestSCons.TimeSCons(variables={'DIR_COUNT':5000})
+# Full-build time of just under 10 seconds on ubuntu-timings slave,
+# as determined by bin/calibrate.py on 9 December 2009:
+#
+# run   1:   2.235:  DIR_COUNT=50
+# run   2:   3.976:  DIR_COUNT=223
+# run   3:   7.353:  DIR_COUNT=560
+# run   4:   9.569:  DIR_COUNT=761
+# run   5:   9.353:  DIR_COUNT=761
+# run   6:   9.972:  DIR_COUNT=813
+# run   7:   9.930:  DIR_COUNT=813
+# run   8:   9.983:  DIR_COUNT=813
+
+test = TestSCons.TimeSCons(variables={'DIR_COUNT':813})
 
 for d in xrange(test.variables['DIR_COUNT']):
     test.subdir('inc_%04d' % d)
diff --git a/timings/JTimer/TimeSCons-run.py b/timings/JTimer/TimeSCons-run.py
index 05ffbfb9..7fe1bf40 100644
--- a/timings/JTimer/TimeSCons-run.py
+++ b/timings/JTimer/TimeSCons-run.py
@@ -25,10 +25,10 @@
 This configuration is for timing how we evaluate long chains of
 dependencies, specifically when -j is used.
 
-We set up a chain of 500 targets that get built from a Python function
-action with no source files (equivalent to "echo junk > $TARGET").
-Each target explicitly depends on the next target in turn, so the
-Taskmaster will do a deep walk of the dependency graph.
+We set up a chain of $TARGET_COUNT targets that get built from a
+Python function action with no source files (equivalent to "echo junk >
+$TARGET").  Each target explicitly depends on the next target in turn,
+so the Taskmaster will do a deep walk of the dependency graph.
 
 This test case was contributed by Kevin Massey.  Prior to revision 1468,
 we had a serious O(N^2) problem in the Taskmaster when handling long
@@ -38,7 +38,26 @@ to the Taskmaster so it could be smarter about not re-evaluating Nodes.
 
 import TestSCons
 
-test = TestSCons.TimeSCons(variables={'TARGET_COUNT':500})
+# Full-build time of just under 10 seconds on ubuntu-timings slave,
+# as determined by bin/calibrate.py on 9 December 2009:
+#
+# run   1:   3.211:  TARGET_COUNT=50
+# run   2:  11.920:  TARGET_COUNT=155
+# run   3:   9.182:  TARGET_COUNT=130
+# run   4:  10.185:  TARGET_COUNT=141
+# run   5:   9.945:  TARGET_COUNT=138
+# run   6:  10.035:  TARGET_COUNT=138
+# run   7:   9.898:  TARGET_COUNT=137
+# run   8:   9.840:  TARGET_COUNT=137
+# run   9:  10.054:  TARGET_COUNT=137
+# run  10:   9.747:  TARGET_COUNT=136
+# run  11:   9.778:  TARGET_COUNT=136
+# run  12:   9.743:  TARGET_COUNT=136
+#
+# The fact that this varies so much suggests that it's pretty
+# non-deterministic, which makes sense for a test involving -j.
+
+test = TestSCons.TimeSCons(variables={'TARGET_COUNT':136})
 
 test.main()
 
diff --git a/timings/hundred/TimeSCons-run.py b/timings/hundred/TimeSCons-run.py
index 915c1319..c90c26b9 100644
--- a/timings/hundred/TimeSCons-run.py
+++ b/timings/hundred/TimeSCons-run.py
@@ -25,15 +25,26 @@
 This configuration is for timing how we handle the NxM interaction when
 we build a lot of targets from a lot of source files.
 
-We create a list of 500 target files that will each be built by copying
-a file from a corresponding list of 500 source files.  The source
-files themselves are each built by a Python function action that's the
-equivalent of "echo contents > $TARGET".
+We create a list of $TARGET_COUNT target files that will each be built by
+copying a file from a corresponding list of $TARGET_COUNT source files.
+The source files themselves are each built by a Python function action
+that's the equivalent of "echo contents > $TARGET".
 """
 
 import TestSCons
 
-test = TestSCons.TimeSCons(variables={'TARGET_COUNT':500})
+# Full-build time of just under 10 seconds on ubuntu-timings slave,
+# as determined by bin/calibrate.py on 9 December 2009:
+#
+# run   1:   3.124:  TARGET_COUNT=50
+# run   2:  11.936:  TARGET_COUNT=160
+# run   3:   9.175:  TARGET_COUNT=134
+# run   4:  10.489:  TARGET_COUNT=146
+# run   5:   9.798:  TARGET_COUNT=139
+# run   6:   9.695:  TARGET_COUNT=139
+# run   7:   9.670:  TARGET_COUNT=139
+
+test = TestSCons.TimeSCons(variables={'TARGET_COUNT':139})
 
 for t in xrange(test.variables['TARGET_COUNT']):
     open('source_%04d' % t, 'wb' ).write('contents\n')