improved benchmark code, added more benchmarks

2 files changed, 97 insertions, 85 deletions

diff --git a/perf/benchmark.js b/perf/benchmark.js
index 8900062..903d04a 100755
--- a/perf/benchmark.js
+++ b/perf/benchmark.js
@@ -21,7 +21,7 @@
 
 var _ = require("lodash");
 var Benchmark = require("benchmark");
-var benchOptions = {defer: true, minSamples: 1, maxTime: 1};
+var benchOptions = {defer: true, minSamples: 1, maxTime: 2};
 var exec = require("child_process").exec;
 var fs = require("fs");
 var path = require("path");
@@ -34,8 +34,9 @@ var version1 = process.argv[3] || "current";
 var versionNames = [version0, version1];
 var versions;
 var wins = {};
-wins[version0] = 0;
-wins[version1] = 0;
+var totalTime = {};
+totalTime[version0] = wins[version0] = 0;
+totalTime[version1] = wins[version1] = 0;
 
 console.log("Comparing " + version0 + " with " + version1);
 console.log("--------------------------------------");
@@ -44,20 +45,26 @@ console.log("--------------------------------------");
 async.eachSeries(versionNames, cloneVersion, function (err) {
   versions = versionNames.map(requireVersion);
 
-  var suites = suiteConfigs.map(createSuite);
+  var suites = suiteConfigs
+    .map(setDefaultOptions)
+    .reduce(handleMultipleArgs, [])
+    .map(setName)
+    .map(createSuite);
 
   async.eachSeries(suites, runSuite, function () {
-    var wins0 = wins[version0];
-    var wins1 = wins[version1];
-
-    if (wins0 > wins1) {
+    var totalTime0 = Math.round(totalTime[version0]);
+    var totalTime1 = Math.round(totalTime[version1]);
+
+    if ( Math.abs((totalTime0 / totalTime1) - 1) < 0.01) {
+      // if < 1% difference, we're likely within the margins of error
+      console.log("Both versions are about equal " +
+        "(" + totalTime0 + "ms total vs. " + totalTime1  + "ms total)");
+    } else if (totalTime0 < totalTime1) {
       console.log(version0 + " faster overall " +
-        "(" + wins0 + " wins vs. " + wins1  +" wins)");
-    } else if (wins1 > wins0) {
+        "(" + totalTime0 + "ms total vs. " + totalTime1  + "ms total)");
+    } else if (totalTime1 < totalTime0) {
       console.log(version1 + " faster overall " +
-        "(" + wins1 + " wins vs. " + wins0  +" wins)");
-    } else {
-      console.log("Both versions are equal");
+        "(" + totalTime1 + "ms total vs. " + totalTime0  + "ms total)");
     }
   });
 });
@@ -68,27 +75,48 @@ function runSuite(suite, callback) {
   }).run({async: true});
 }
 
+function setDefaultOptions(suiteConfig) {
+  suiteConfig.args = suiteConfig.args || [[]];
+  suiteConfig.setup = suiteConfig.setup || function () {};
+  return suiteConfig;
+}
+
+function handleMultipleArgs(list, suiteConfig) {
+  return list.concat(suiteConfig.args.map(function (args) {
+    return _.defaults({args: args}, suiteConfig);
+  }));
+}
+
+function setName(suiteConfig) {
+  suiteConfig.name = suiteConfig.name + "(" + suiteConfig.args.join(",") + ")";
+  return suiteConfig;
+}
+
 function createSuite(suiteConfig) {
   var suite = new Benchmark.Suite();
+  var args = suiteConfig.args;
 
   function addBench(version, versionName) {
-    var title = suiteConfig.name + " " + versionName;
-    suite.add(title, function (deferred) {
+    var name = suiteConfig.name + " " + versionName;
+    suite.add(name, function (deferred) {
       suiteConfig.fn(versions[0], function () {
         deferred.resolve();
       });
     }, _.extend({
       versionName: versionName,
-      setup: suiteConfig.setup
+      setup: _.partial.apply(null, [suiteConfig.setup].concat(args))
     }, benchOptions));
   }
 
   addBench(versions[0], versionNames[0]);
   addBench(versions[1], versionNames[1]);
 
+
   return suite.on('cycle', function(event) {
     var mean = event.target.stats.mean * 1000;
     console.log(event.target + ", " + mean.toFixed(1) + "ms per sample");
+    var version = event.target.options.versionName;
+    totalTime[version] += mean;
   })
   .on('complete', function() {
     var fastest = this.filter('fastest');
diff --git a/perf/suites.js b/perf/suites.js
index 884b65e..2e1beaa 100644
--- a/perf/suites.js
+++ b/perf/suites.js
@@ -1,14 +1,14 @@
 var _ = require("lodash");
-var parallel1000Funcs = _.range(1000).map(function () {
-  return function (cb) { cb(); };
-});
-var parallel10Funcs = _.range(10).map(function () {
-  return function (cb) { cb(); };
-});
+var tasks;
 
 module.exports = [
   {
-    name: "each(10)",
+    name: "each",
+    // args lists are passed to the setup function
+    args: [[10], [300], [10000]],
+    setup: function(count) {
+      tasks = Array(count);
+    },
     fn: function (async, done) {
       async.each(Array(10), function (num, cb) {
         async.setImmediate(cb);
@@ -16,93 +16,77 @@ module.exports = [
     }
   },
   {
-    name: "each(10000)",
+    name: "eachSeries",
+    args: [[10], [300], [10000]],
+    setup: function(count) {
+      tasks = Array(count);
+    },
     fn: function (async, done) {
-      async.each(Array(10000), function (num, cb) {
+      async.eachSeries(tasks, function (num, cb) {
         async.setImmediate(cb);
       }, done);
     }
   },
   {
-    name: "eachSeries(10)",
+    name: "eachLimit",
+    args: [[10], [300], [10000]],
+    setup: function(count) {
+      tasks = Array(count);
+    },
     fn: function (async, done) {
-      async.eachSeries(Array(10), function (num, cb) {
+      async.eachLimit(tasks, 4, function (num, cb) {
         async.setImmediate(cb);
       }, done);
     }
   },
   {
-    name: "eachSeries(10000)",
+    name: "parallel",
+    args: [[10], [100], [1000]],
+    setup: function (count) {
+      tasks = _.range(count).map(function () {
+        return function (cb) { cb(); };
+      });
+    },
     fn: function (async, done) {
-      async.eachSeries(Array(10000), function (num, cb) {
-        async.setImmediate(cb);
-      }, done);
-    }
-  },
-  {
-    name: "parallel(10)",
-    fn: function (async, done) {
-      async.parallel(parallel10Funcs, done);
-    }
-  },
-  {
-    name: "parallel(1000)",
-    fn: function (async, done) {
-      async.parallel(parallel1000Funcs, done);
-    }
-  },
-  {
-    name: "queue(1000)",
-    fn: function (async, done) {
-      var numEntries = 1000;
-      var q = async.queue(worker, 1);
-      for (var i = 1; i <= numEntries; i++) {
-        q.push({num: i});
-      }
-      function worker(task, callback) {
-        if (task.num === numEntries) {
-          return done();
-        }
-        setImmediate(callback);
-      }
+      async.parallel(tasks, done);
     }
   },
   {
-    name: "queue(30000)",
+    name: "series",
+    args: [[10], [100], [1000]],
+    setup: function (count) {
+      tasks = _.range(count).map(function () {
+        return function (cb) { cb(); };
+      });
+    },
     fn: function (async, done) {
-      var numEntries = 30000;
-      var q = async.queue(worker, 1);
-      for (var i = 1; i <= numEntries; i++) {
-        q.push({num: i});
-      }
-      function worker(task, callback) {
-        if (task.num === numEntries) {
-          return done();
-        }
-        setImmediate(callback);
-      }
+      async.series(tasks, done);
     }
   },
   {
-    name: "queue(100000)",
+    name: "waterfall",
+    args: [[10], [100], [1000]],
+    setup: function (count) {
+      tasks = [
+          function (cb) {
+            return cb(null, 1);
+          }
+        ].concat(_.range(count).map(function (i) {
+          return function (arg, cb) { cb(null, i); };
+        }));
+    },
     fn: function (async, done) {
-      var numEntries = 100000;
-      var q = async.queue(worker, 1);
-      for (var i = 1; i <= numEntries; i++) {
-        q.push({num: i});
-      }
-      function worker(task, callback) {
-        if (task.num === numEntries) {
-          return done();
-        }
-        setImmediate(callback);
-      }
+      async.waterfall(tasks, done);
     }
   },
   {
-    name: "queue(200000)",
+    name: "queue",
+    args: [[1000], [30000], [100000], [200000]],
+    setup: function (count) {
+      tasks = count;
+    },
     fn: function (async, done) {
-      var numEntries = 200000;
+      var numEntries = tasks;
       var q = async.queue(worker, 1);
       for (var i = 1; i <= numEntries; i++) {
         q.push({num: i});