Fix replication Javascript test

The test was flaky for a variety of reasons:

 * waitForSeq only waited for 3 seconds and on failure it never explitly
   indicated an error and just waited for the comparison below to fail. So made
   it wait for 30 seconds and also throw an exception right away if it fails.

 * Last waitForSeq was used after task was canceled. So it just wasted time
   waiting until timeout as the task was null. So created a function to
   wait for task to be null.

 * waitForSeq spun in a tight do/while loop querying _active_tasks. In some test
   environment with minimal CPU resources that's not the greatest thing to do.
   So made it wait for 0.5 seconds between retries.

 * waitForSeq waited for replication task's through_seq value to match source
   update sequence from source db info. Those don't necessarily match. Instead
   made waitForSeq use the changes feed last sequence since that's what the
   replication task uses to update through_seq.

1 files changed, 24 insertions, 3 deletions

diff --git a/test/javascript/tests/replication.js b/test/javascript/tests/replication.js
index c7861b342..b6f2c4e12 100644
--- a/test/javascript/tests/replication.js
+++ b/test/javascript/tests/replication.js
@@ -218,12 +218,15 @@ couchTests.replication = function(debug) {
     return null;
   }
 
+  function getSourceLastSeq(sourceDb) {
+      return sourceDb.changes({"since":"now"}).last_seq;
+  }
 
   function waitForSeq(sourceDb, targetDb, rep_id) {
-    var sourceSeq = sourceDb.info().update_seq,
+    var sourceSeq = getSourceLastSeq(sourceDb),
         t0 = new Date(),
         t1,
-        ms = 3000;
+        ms = 30000;
 
     do {
       var task = getTask(rep_id, 0);
@@ -231,7 +234,24 @@ couchTests.replication = function(debug) {
         return;
       }
       t1 = new Date();
+      sleep(500);
     } while (((t1 - t0) <= ms));
+    throw(Error('Timeout waiting for replication through_seq = source update seq'));
+  }
+
+  function waitReplicationTaskStop(rep_id) {
+      var t0 = new Date(),
+          t1,
+          ms = 30000;
+      do {
+        var task = getTask(rep_id, 0);
+        if(task == null) {
+            return;
+        }
+        t1 = new Date();
+        sleep(500);
+      } while (((t1 - t0) <= ms));
+      throw(Error('Timeout waiting for replication task stop' + rep_id));
   }
 
   // test simple replications (not continuous, not filtered), including
@@ -1423,7 +1443,8 @@ couchTests.replication = function(debug) {
     };
     TEquals(true, sourceDb.save(doc).ok);
 
-    waitForSeq(sourceDb, targetDb, rep_id);
+    waitReplicationTaskStop(rep_id);
+
     copy = targetDb.open(doc._id);
     TEquals(null, copy);
   }