Abort dbtests if a single test runs for more than 30 min

1 files changed, 36 insertions, 0 deletions

diff --git a/dbtests/framework.cpp b/dbtests/framework.cpp
index f57d3eec647..bb96279d97a 100644
--- a/dbtests/framework.cpp
+++ b/dbtests/framework.cpp
@@ -26,6 +26,7 @@
 #include "framework.h"
 #include "../util/file_allocator.h"
 #include "../db/dur.h"
+#include "../util/background.h"
 
 #ifndef _WIN32
 #include <cxxabi.h>
@@ -78,6 +79,10 @@ namespace mongo {
 
         Result * Result::cur = 0;
 
+        int minutesRunning = 0; // reset to 0 each time a new test starts
+        mutex minutesRunningMutex("minutesRunningMutex");
+        string currentTestName;
+
         Result * Suite::run( const string& filter ) {
             tlogLevel = -1;
 
@@ -107,6 +112,12 @@ namespace mongo {
                 stringstream err;
                 err << tc->getName() << "\t";
 
+                {
+                    scoped_lock lk(minutesRunningMutex);
+                    minutesRunning = 0;
+                    currentTestName = tc->getName();
+                }
+
                 try {
                     tc->run();
                     passes = true;
@@ -146,6 +157,28 @@ namespace mongo {
                  << options << "suite: run the specified test suite(s) only" << endl;
         }
 
+        class TestWatchDog : public BackgroundJob {
+        public:
+            virtual string name() const { return "TestWatchDog"; }
+            virtual void run(){
+
+                while (true) {
+                    sleepsecs(60);
+
+                    scoped_lock lk(minutesRunningMutex);
+                    minutesRunning++; //reset to 0 when new test starts
+
+                    if (minutesRunning > 30){
+                        log() << currentTestName << " has been running for more than 30 minutes. aborting." << endl;
+                        abort();
+                    }
+                    else if (minutesRunning > 1){
+                        warning() << currentTestName << " has been running for more than " << minutesRunning-1 << " minutes." << endl;
+                    }
+                }
+            }
+        };
+
         int Suite::run( int argc , char** argv , string default_dbpath ) {
             unsigned long long seed = time( 0 );
             string dbpathSpec;
@@ -276,6 +309,9 @@ namespace mongo {
 //                cmdLine.durOptions |= 8;
             }
 
+            TestWatchDog twd;
+            twd.go();
+
             int ret = run(suites,filter);
 
 #if !defined(_WIN32) && !defined(__sunos__)