test: Implement --fatal-timeouts to allow a timely VM restart.

2 files changed, 21 insertions, 3 deletions

diff --git a/test/lib/journal.h b/test/lib/journal.h
index 8913d7cd4..1e0a74232 100644
--- a/test/lib/journal.h
+++ b/test/lib/journal.h
@@ -89,6 +89,7 @@ struct Journal {
 	Status status, written;
 
 	std::string location, list;
+	int timeouts;
 
 	void append( std::string path ) {
 		std::ofstream of( path.c_str(), std::fstream::app );
@@ -126,6 +127,10 @@ struct Journal {
 
 	void done( std::string n, R r ) {
 		status[ n ] = r;
+		if ( r == TIMEOUT )
+			++ timeouts;
+		else
+			timeouts = 0;
 		sync();
 	}
 
@@ -165,7 +170,8 @@ struct Journal {
 
 	Journal( std::string dir )
 		: location( dir + "/journal" ),
-		  list( dir + "/list" )
+		  list( dir + "/list" ),
+		  timeouts( 0 )
 	{}
 };
 
diff --git a/test/lib/runner.cpp b/test/lib/runner.cpp
index ffc5398a3..67a9b8773 100644
--- a/test/lib/runner.cpp
+++ b/test/lib/runner.cpp
@@ -60,10 +60,11 @@ bool fatal_signal = false;
 bool interrupt = false;
 
 struct Options {
-	bool verbose, batch, interactive, cont;
+	bool verbose, batch, interactive, cont, fatal_timeouts;
 	std::string testdir, outdir;
 	std::vector< std::string > flavours, filter;
-	Options() : verbose( false ), batch( false ), interactive( false ), cont( false ) {}
+	Options() : verbose( false ), batch( false ), interactive( false ),
+		    cont( false ), fatal_timeouts( false ) {}
 };
 
 struct TestProcess
@@ -409,6 +410,14 @@ struct Main {
 
 			i->run();
 
+			if ( options.fatal_timeouts && journal.timeouts >= 2 ) {
+				journal.started( i->id() ); // retry the test on --continue
+				std::cerr << "E: Hit 2 timeouts in a row with --fatal-timeouts" << std::endl;
+				std::cerr << "Suspending (please restart the VM)." << std::endl;
+				sleep( 3600 );
+				die = 1;
+			}
+
 			if ( time(0) - start > 3 * 3600 ) {
 				std::cerr << "3 hours passed, giving up..." << std::endl;
 				die = 1;
@@ -503,6 +512,9 @@ int main(int argc, char **argv)
 	if ( args.has( "--only" ) )
 		split( args.opt( "--only" ), opt.filter );
 
+	if ( args.has( "--fatal-timeouts" ) )
+		opt.fatal_timeouts = true;
+
 	if ( args.has( "--batch" ) || hasenv( "BATCH" ) ) {
 		opt.verbose = false;
 		opt.batch = true;