From c0d020644bf925fb1010c86756e5051e50f18e51 Mon Sep 17 00:00:00 2001 From: "Eric S. Raymond" Date: Sun, 24 Aug 2014 15:29:50 -0400 Subject: Add slow=yes option for regression tests in an attempt to avoid flakeouts. --- SConstruct | 11 ++++++++++- build.txt | 1 + gps/fake.py | 20 +++++++++++++++----- gpsfake | 9 ++++++--- gpsfake.xml | 7 +++++++ regress-driver | 13 ++++++++++--- 6 files changed, 49 insertions(+), 12 deletions(-) diff --git a/SConstruct b/SConstruct index 9df08938..33c9e9c6 100644 --- a/SConstruct +++ b/SConstruct @@ -166,6 +166,8 @@ boolopts = ( ("chrpath", False, "use chrpath to edit library load paths"), ("manbuild", True, "build help in man and HTML formats"), ("leapfetch", True, "fetch up-to-date data on leap seconds."), + # Test control + ("slow", False, "run tests with realistic (slow) delays"), ) for (name, default, help) in boolopts: opts.Add(BoolVariable(name, help, default)) @@ -271,6 +273,13 @@ for key, value in os.environ.iteritems(): # $SRCDIR replaces occurrences for $(srcdir) in the autotools build. env['SRCDIR'] = '.' +# We may need to force slow regression tests to get around race +# conditions in the pty layer, especially on a loaded machine. +if env["slow"]: + env['REGRESSOPTS'] = "-S" +else: + env['REGRESSOPTS'] = "" + def announce(msg): if not env.GetOption("silent"): print msg @@ -1514,7 +1523,7 @@ else: if env['socket_export']: # Regression-test the daemon gps_regress = Utility("gps-regress", [gpsd, python_built_extensions], - '$SRCDIR/regress-driver test/daemon/*.log') + '$SRCDIR/regress-driver $REGRESSOPTS test/daemon/*.log') # Build the regression tests for the daemon. # Note: You'll have to do this whenever the default leap second diff --git a/build.txt b/build.txt index 934c0230..f2fb8e05 100644 --- a/build.txt +++ b/build.txt @@ -328,6 +328,7 @@ framework shove data through the pty and socket layers *way* faster than would ever occur in production. If you get regression-test failures that aren't repeatable and look like the test framework is sporadically failing to feed the last line or two of test loads, try +using the slow=yes option with scons check. If that fails, try increasing the delay constants in fake.py. If you have to do this, please report your experience to the GPSD maintainers. diff --git a/gps/fake.py b/gps/fake.py index 2b51f5ad..445ff3a2 100644 --- a/gps/fake.py +++ b/gps/fake.py @@ -91,7 +91,7 @@ import packet as sniffer # implementation underneath will return with precision finer than 1 # second. (Linux and *BSD return full precision.) -# Field reports: +# Field reports on minima: # # Eric Raymond on Linux 3.11.0 under an Intel Core Duo at 2.66GHz. # WRITE_PAD = 0.0 / CLOSE_DELAY = 0.1 Works, 112s real @@ -122,6 +122,10 @@ else: WRITE_PAD = 0.004 CLOSE_DELAY = 0.8 +# Additional delays in slow mode +WRITE_PAD_SLOWDOWN = 0.01 +CLOSE_DELAY_SLOWDOWN = 2.0 + class TestLoadError(exceptions.Exception): def __init__(self, msg): exceptions.Exception.__init__(self) @@ -129,7 +133,7 @@ class TestLoadError(exceptions.Exception): class TestLoad: "Digest a logfile into a list of sentences we can cycle through." - def __init__(self, logfp, predump=False): + def __init__(self, logfp, predump=False, slow=False): self.sentences = [] # This is the interesting part if type(logfp) == type(""): logfp = open(logfp, "r") @@ -140,6 +144,8 @@ class TestLoad: self.sourcetype = "pty" self.serial = None self.delay = WRITE_PAD + if slow: + self.delay += WRITE_PAD_SLOWDOWN self.delimiter = None # Stash away a copy in case we need to resplit text = logfp.read() @@ -509,7 +515,7 @@ class TestSessionError(exceptions.Exception): class TestSession: "Manage a session including a daemon with fake GPSes and clients." - def __init__(self, prefix=None, port=None, options=None, verbose=0, predump=False, udp=False, tcp=False): + def __init__(self, prefix=None, port=None, options=None, verbose=0, predump=False, udp=False, tcp=False, slow=False): "Initialize the test session by launching the daemon." self.prefix = prefix self.port = port @@ -518,6 +524,7 @@ class TestSession: self.predump = predump self.udp = udp self.tcp = tcp + self.slow = slow self.daemon = DaemonInstance() self.fakegpslist = {} self.client_id = 0 @@ -530,6 +537,9 @@ class TestSession: self.port = port else: self.port = gps.GPSD_PORT + self.close_delay = CLOSE_DELAY + if slow: + self.close_delay += CLOSE_DELAY_SLOWDOWN self.progress = lambda x: None self.reporter = lambda x: None self.default_predicate = None @@ -547,7 +557,7 @@ class TestSession: "Add a simulated GPS being fed by the specified logfile." self.progress("gpsfake: gps_add(%s, %d)\n" % (logfile, speed)) if logfile not in self.fakegpslist: - testload = TestLoad(logfile, predump=self.predump) + testload = TestLoad(logfile, predump=self.predump, slow=self.slow) if testload.sourcetype == "UDP" or self.udp: newgps = FakeUDP(testload, ipaddr="127.0.0.1", port=self.baseport, @@ -625,7 +635,7 @@ class TestSession: had_output = False chosen = self.choose() if isinstance(chosen, FakeGPS): - if chosen.exhausted and (time.time() - chosen.exhausted > CLOSE_DELAY) and chosen.byname in self.fakegpslist: + if chosen.exhausted and (time.time() - chosen.exhausted > self.close_delay) and chosen.byname in self.fakegpslist: self.gps_remove(chosen.byname) self.progress("gpsfake: GPS %s removed (timeout)\n" % chosen.byname) elif not chosen.go_predicate(chosen.index, chosen): diff --git a/gpsfake b/gpsfake index 9ae5d7ba..63da8bd8 100755 --- a/gpsfake +++ b/gpsfake @@ -94,7 +94,7 @@ def fakehook(linenumber, fakegps): if __name__ == '__main__': try: - (options, arguments) = getopt.getopt(sys.argv[1:], "1bc:D:fghilm:no:pP:r:s:tuvx") + (options, arguments) = getopt.getopt(sys.argv[1:], "1bc:D:fghilm:no:pP:r:s:Stuvx") except getopt.GetoptError, msg: print "gpsfake: " + str(msg) raise SystemExit, 1 @@ -114,6 +114,7 @@ if __name__ == '__main__': tcp = False udp = False verbose = 0 + slow = False for (switch, val) in options: if (switch == '-1'): singleshot = True @@ -147,6 +148,8 @@ if __name__ == '__main__': client_init = val elif (switch == '-s'): speed = int(val) + elif (switch == '-S'): + slow = True elif (switch == '-t'): tcp = True elif (switch == '-u'): @@ -154,7 +157,7 @@ if __name__ == '__main__': elif (switch == '-v'): verbose += 1 elif (switch == '-h'): - sys.stderr.write("usage: gpsfake [-h] [-l] [-m monitor] [--D debug] [-o options] [-p] [-s speed] [-c cycle] [-b] logfile\n") + sys.stderr.write("usage: gpsfake [-h] [-l] [-m monitor] [--D debug] [-o options] [-p] [-s speed] [-S] [-c cycle] [-b] logfile\n") raise SystemExit,0 try: @@ -174,7 +177,7 @@ if __name__ == '__main__': test = gpsfake.TestSession(prefix=monitor, port=port, options=doptions, tcp=tcp, udp=udp, verbose=verbose, - predump=predump) + predump=predump, slow=slow) if pipe: test.reporter = sys.stdout.write diff --git a/gpsfake.xml b/gpsfake.xml index 8faaba6f..ed051fb7 100644 --- a/gpsfake.xml +++ b/gpsfake.xml @@ -38,6 +38,7 @@ BSD terms apply: see the file COPYING in the distribution root for details. -P port -r initcmd -s speed + -S -u -t -v @@ -145,6 +146,12 @@ The default is ?WATCH={"enable":true,"json":true}. The sets the baud rate for the slave tty. The default is 4800. +The option -S tells gpsfake to insert realistic delays in the +test input rather than trying to stuff it through the daemon as fast +as possible. This will make the test(s) run much slower, but avoids +flaky failures due to machine lode and possible race conditions in +the pty layer. + The forces the test framework to use TCP rather than pty devices. Besides being a test of TCP source handling, this may be useful for testing from within chroot jails where access diff --git a/regress-driver b/regress-driver index 453b932a..8b14c025 100755 --- a/regress-driver +++ b/regress-driver @@ -45,12 +45,13 @@ opts="" logfile="" help="0" baton=false -while getopts cl:stbuvo:h opt +while getopts cl:sStbuvo:h opt do case $opt in c) testing=clientlib ;; # Can be 'daemon' l) logfile=$OPTARG ;; # Logfile to save diffs to s) mode=regress ;; # Run regression tests + S) mode=slowregress ;; # Run regression tests with big delays t) baton=true mode=regress ;; # Run regression tests w/baton b) mode=build ;; # Rebuild regression check files u) opts="$opts -u" ;; # Force UDP @@ -69,7 +70,8 @@ then echo "-h - this help" echo "-c - can be 'daemon'" echo "-l - where to log diffs to" - echo "-s - run regreession tests" + echo "-s - run regression tests" + echo "-S - run regression tests with realistic timing delays" echo "-t - Run regression tests w/baton" echo "-b - Rebuild regression check files" echo "-u - Force UDP" @@ -107,8 +109,13 @@ then opts="$opts -b" fi +if [ $mode = slowregress ] +then + opts="$opts -S" +fi + case $mode in - regress) + regress|slowregress) echo "Testing the $testing..." >&2 errors=0; total=0; notfound=0;error_list=""; for f in $*; do -- cgit v1.2.1