rs better fatal handling

4 files changed, 15 insertions, 4 deletions

diff --git a/db/repl/rs.cpp b/db/repl/rs.cpp
index 09a0e728330..1c0444a3dfd 100644
--- a/db/repl/rs.cpp
+++ b/db/repl/rs.cpp
@@ -494,6 +494,7 @@ namespace mongo {
                 startupStatusMsg = "replSet error loading set config (BADCONFIG)";
                 log() << "replSet error loading configurations " << e.toString() << rsLog;
                 log() << "replSet error replication will not start" << rsLog;
+                sethbmsg("error loading set config");
                 _fatal();
                 throw;
             }
@@ -507,8 +508,8 @@ namespace mongo {
     { 
         //lock l(this);
         box.set(MemberState::RS_FATAL, 0);
-        sethbmsg("fatal error");
-        log() << "replSet error fatal error, stopping replication" << rsLog; 
+        //sethbmsg("fatal error");
+        log() << "replSet error fatal, stopping replication" << rsLog; 
     }
 
     void ReplSet::haveNewConfig(ReplSetConfig& newConfig, bool addComment) { 
diff --git a/db/repl/rs.h b/db/repl/rs.h
index 0719ab9818e..ed88c227d18 100644
--- a/db/repl/rs.h
+++ b/db/repl/rs.h
@@ -206,7 +206,7 @@ namespace mongo {
         }
         void noteRemoteIsPrimary(const Member *remote) { 
             scoped_lock lk(m);
-            if( !sp.state.secondary() )
+            if( !sp.state.secondary() && !sp.state.fatal() )
                 sp.state = MemberState::RS_RECOVERING;
             sp.primary = remote;
         }
diff --git a/db/repl/rs_rollback.cpp b/db/repl/rs_rollback.cpp
index c55ec835915..6b2544c4305 100644
--- a/db/repl/rs_rollback.cpp
+++ b/db/repl/rs_rollback.cpp
@@ -431,6 +431,7 @@ namespace mongo {
                    log() << "replSet error rolling back : " << err << ". A full resync will be necessary." << rsLog;
                    /* todo: reset minvalid so that we are permanently in fatal state */
                    /* todo: don't be fatal, but rather, get all the data first. */
+                   sethbmsg("rollback error");
                    throw rsfatal();
                }
            }
@@ -649,6 +650,11 @@ namespace mongo {
             try { 
                 syncFixUp(how, r);
             }
+            catch( rsfatal& ) { 
+                sethbmsg("rollback fixup error");
+                _fatal();
+                return 2;
+            }
             catch(...) { 
                 incRBID(); throw;
             }
diff --git a/jstests/replsets/rollback3.js b/jstests/replsets/rollback3.js
index e09ce84a11c..5c2f2f1b6b3 100755
--- a/jstests/replsets/rollback3.js
+++ b/jstests/replsets/rollback3.js
@@ -30,7 +30,11 @@ function wait(f) {
         if (n % 4 == 0)
             print("rollback3.js waiting " + w);
         if (++n == 4) {
-            print("" + f);
+            print("" + f);

+        }

+        if (n == 200) {

+            print("rollback3.js failing waited too long");

+            throw "wait error";
         }
         sleep(1000);
     }