Add slow=yes option for regression tests in an attempt to avoid flakeouts.

6 files changed, 49 insertions, 12 deletions

diff --git a/SConstruct b/SConstruct
index 9df08938..33c9e9c6 100644
--- a/SConstruct
+++ b/SConstruct
@@ -166,6 +166,8 @@ boolopts = (
     ("chrpath",       False, "use chrpath to edit library load paths"),
     ("manbuild",      True,  "build help in man and HTML formats"),
     ("leapfetch",     True,  "fetch up-to-date data on leap seconds."),
+    # Test control
+    ("slow",          False, "run tests with realistic (slow) delays"),
     )
 for (name, default, help) in boolopts:
     opts.Add(BoolVariable(name, help, default))
@@ -271,6 +273,13 @@ for key, value in os.environ.iteritems():
 # $SRCDIR replaces occurrences for $(srcdir) in the autotools build.
 env['SRCDIR'] = '.'
 
+# We may need to force slow regression tests to get around race
+# conditions in the pty layer, especially on a loaded machine.
+if env["slow"]:
+    env['REGRESSOPTS'] = "-S"
+else:
+    env['REGRESSOPTS'] = ""
+
 def announce(msg):
     if not env.GetOption("silent"):
         print msg
@@ -1514,7 +1523,7 @@ else:
 if env['socket_export']:
     # Regression-test the daemon
     gps_regress = Utility("gps-regress", [gpsd, python_built_extensions],
-            '$SRCDIR/regress-driver test/daemon/*.log')
+            '$SRCDIR/regress-driver $REGRESSOPTS test/daemon/*.log')
 
     # Build the regression tests for the daemon.
     # Note: You'll have to do this whenever the default leap second
diff --git a/build.txt b/build.txt
index 934c0230..f2fb8e05 100644
--- a/build.txt
+++ b/build.txt
@@ -328,6 +328,7 @@ framework shove data through the pty and socket layers *way* faster
 than would ever occur in production.  If you get regression-test
 failures that aren't repeatable and look like the test framework is
 sporadically failing to feed the last line or two of test loads, try
+using the slow=yes option with scons check.  If that fails, try
 increasing the delay constants in fake.py.  If you have to do this, 
 please report your experience to the GPSD maintainers.
 
diff --git a/gps/fake.py b/gps/fake.py
index 2b51f5ad..445ff3a2 100644
--- a/gps/fake.py
+++ b/gps/fake.py
@@ -91,7 +91,7 @@ import packet as sniffer
 # implementation underneath will return with precision finer than 1
 # second. (Linux and *BSD return full precision.)
 
-# Field reports:
+# Field reports on minima:
 #
 # Eric Raymond  on Linux 3.11.0 under an Intel Core Duo at 2.66GHz.
 #  WRITE_PAD = 0.0 / CLOSE_DELAY = 0.1    Works, 112s real
@@ -122,6 +122,10 @@ else:
     WRITE_PAD = 0.004
     CLOSE_DELAY = 0.8
 
+# Additional delays in slow mode
+WRITE_PAD_SLOWDOWN = 0.01
+CLOSE_DELAY_SLOWDOWN = 2.0
+
 class TestLoadError(exceptions.Exception):
     def __init__(self, msg):
         exceptions.Exception.__init__(self)
@@ -129,7 +133,7 @@ class TestLoadError(exceptions.Exception):
 
 class TestLoad:
     "Digest a logfile into a list of sentences we can cycle through."
-    def __init__(self, logfp, predump=False):
+    def __init__(self, logfp, predump=False, slow=False):
         self.sentences = []	# This is the interesting part
         if type(logfp) == type(""):
             logfp = open(logfp, "r")
@@ -140,6 +144,8 @@ class TestLoad:
         self.sourcetype = "pty"
         self.serial = None
         self.delay = WRITE_PAD
+        if slow:
+            self.delay += WRITE_PAD_SLOWDOWN
         self.delimiter = None
         # Stash away a copy in case we need to resplit
         text = logfp.read()
@@ -509,7 +515,7 @@ class TestSessionError(exceptions.Exception):
 
 class TestSession:
     "Manage a session including a daemon with fake GPSes and clients."
-    def __init__(self, prefix=None, port=None, options=None, verbose=0, predump=False, udp=False, tcp=False):
+    def __init__(self, prefix=None, port=None, options=None, verbose=0, predump=False, udp=False, tcp=False, slow=False):
         "Initialize the test session by launching the daemon."
         self.prefix = prefix
         self.port = port
@@ -518,6 +524,7 @@ class TestSession:
         self.predump = predump
         self.udp = udp
         self.tcp = tcp
+        self.slow = slow
         self.daemon = DaemonInstance()
         self.fakegpslist = {}
         self.client_id = 0
@@ -530,6 +537,9 @@ class TestSession:
             self.port = port
         else:
             self.port = gps.GPSD_PORT
+        self.close_delay = CLOSE_DELAY
+        if slow:
+            self.close_delay += CLOSE_DELAY_SLOWDOWN
         self.progress = lambda x: None
         self.reporter = lambda x: None
         self.default_predicate = None
@@ -547,7 +557,7 @@ class TestSession:
         "Add a simulated GPS being fed by the specified logfile."
         self.progress("gpsfake: gps_add(%s, %d)\n" % (logfile, speed))
         if logfile not in self.fakegpslist:
-            testload = TestLoad(logfile, predump=self.predump)
+            testload = TestLoad(logfile, predump=self.predump, slow=self.slow)
             if testload.sourcetype == "UDP" or self.udp:
                 newgps = FakeUDP(testload, ipaddr="127.0.0.1",
                                  port=self.baseport,
@@ -625,7 +635,7 @@ class TestSession:
                 had_output = False
                 chosen = self.choose()
                 if isinstance(chosen, FakeGPS):
-                    if chosen.exhausted and (time.time() - chosen.exhausted > CLOSE_DELAY) and chosen.byname in self.fakegpslist:
+                    if chosen.exhausted and (time.time() - chosen.exhausted > self.close_delay) and chosen.byname in self.fakegpslist:
                         self.gps_remove(chosen.byname)
                         self.progress("gpsfake: GPS %s removed (timeout)\n" % chosen.byname)
                     elif not chosen.go_predicate(chosen.index, chosen):
diff --git a/gpsfake b/gpsfake
index 9ae5d7ba..63da8bd8 100755
--- a/gpsfake
+++ b/gpsfake
@@ -94,7 +94,7 @@ def fakehook(linenumber, fakegps):
 
 if __name__ == '__main__':
     try:
-        (options, arguments) = getopt.getopt(sys.argv[1:], "1bc:D:fghilm:no:pP:r:s:tuvx")
+        (options, arguments) = getopt.getopt(sys.argv[1:], "1bc:D:fghilm:no:pP:r:s:Stuvx")
     except getopt.GetoptError, msg:
         print "gpsfake: " + str(msg)
         raise SystemExit, 1
@@ -114,6 +114,7 @@ if __name__ == '__main__':
     tcp = False
     udp = False
     verbose = 0
+    slow = False
     for (switch, val) in options:
         if (switch == '-1'):
             singleshot = True
@@ -147,6 +148,8 @@ if __name__ == '__main__':
             client_init = val
         elif (switch == '-s'):
             speed = int(val)
+        elif (switch == '-S'):
+            slow = True
         elif (switch == '-t'):
             tcp = True
         elif (switch == '-u'):
@@ -154,7 +157,7 @@ if __name__ == '__main__':
         elif (switch == '-v'):
             verbose += 1
         elif (switch == '-h'):
-            sys.stderr.write("usage: gpsfake [-h] [-l] [-m monitor] [--D debug] [-o options] [-p] [-s speed] [-c cycle] [-b] logfile\n")
+            sys.stderr.write("usage: gpsfake [-h] [-l] [-m monitor] [--D debug] [-o options] [-p] [-s speed] [-S] [-c cycle] [-b] logfile\n")
             raise SystemExit,0
 
     try:
@@ -174,7 +177,7 @@ if __name__ == '__main__':
 
     test = gpsfake.TestSession(prefix=monitor, port=port, options=doptions,
                                tcp=tcp, udp=udp, verbose=verbose,
-                               predump=predump)
+                               predump=predump, slow=slow)
 
     if pipe:
         test.reporter = sys.stdout.write
diff --git a/gpsfake.xml b/gpsfake.xml
index 8faaba6f..ed051fb7 100644
--- a/gpsfake.xml
+++ b/gpsfake.xml
@@ -38,6 +38,7 @@ BSD terms apply: see the file COPYING in the distribution root for details.
       <arg choice='opt'>-P <replaceable>port</replaceable></arg>
       <arg choice='opt'>-r <replaceable>initcmd</replaceable></arg>
       <arg choice='opt'>-s <replaceable>speed</replaceable></arg>
+      <arg choice='opt'>-S</arg>
       <arg choice='opt'>-u</arg>
       <arg choice='opt'>-t</arg>
       <arg choice='opt'>-v</arg>
@@ -145,6 +146,12 @@ The default is <command>?WATCH={"enable":true,"json":true}</command>.</para>
 <para>The <option>-s</option> sets the baud rate for the slave tty.  The
 default is 4800.</para>
 
+<para>The option -S tells gpsfake to insert realistic delays in the
+test input rather than trying to stuff it through the daemon as fast
+as possible.  This will make the test(s) run much slower, but avoids
+flaky failures due to machine lode and possible race conditions in
+the pty layer.</para>
+
 <para>The <option>-t</option> forces the test framework to use TCP
 rather than pty devices. Besides being a test of TCP source handling,
 this may be useful for testing from within chroot jails where access
diff --git a/regress-driver b/regress-driver
index 453b932a..8b14c025 100755
--- a/regress-driver
+++ b/regress-driver
@@ -45,12 +45,13 @@ opts=""
 logfile=""
 help="0"
 baton=false
-while getopts cl:stbuvo:h opt
+while getopts cl:sStbuvo:h opt
 do
     case $opt in
 	c) testing=clientlib ;;	# Can be 'daemon'
 	l) logfile=$OPTARG ;;	# Logfile to save diffs to
 	s) mode=regress ;;	# Run regression tests
+	S) mode=slowregress ;;	# Run regression tests with big delays
 	t) baton=true mode=regress ;;	# Run regression tests w/baton
 	b) mode=build ;;	# Rebuild regression check files
 	u) opts="$opts -u" ;;	# Force UDP
@@ -69,7 +70,8 @@ then
 	echo "-h  - this help"
 	echo "-c  - can be 'daemon'"
 	echo "-l <filename>  - where to log diffs to"
-	echo "-s  - run regreession tests"
+	echo "-s  - run regression tests"
+	echo "-S  - run regression tests with realistic timing delays"
 	echo "-t <ots>  - Run regression tests w/baton"
 	echo "-b  - Rebuild regression check files"
 	echo "-u - Force UDP"
@@ -107,8 +109,13 @@ then
     opts="$opts -b"
 fi
 
+if [ $mode = slowregress ]
+then
+    opts="$opts -S"
+fi
+
 case $mode in
-    regress)
+    regress|slowregress)
         echo "Testing the $testing..." >&2
         errors=0; total=0; notfound=0;error_list="";
         for f in $*; do