From f5c6331d3e0c8af238a143b16cf7271902acf797 Mon Sep 17 00:00:00 2001 From: Alexander Early Date: Wed, 20 May 2015 15:31:25 -0700 Subject: improved benchmark code, added more benchmarks --- perf/benchmark.js | 60 ++++++++++++++++++++------- perf/suites.js | 122 ++++++++++++++++++++++++------------------------------ 2 files changed, 97 insertions(+), 85 deletions(-) (limited to 'perf') diff --git a/perf/benchmark.js b/perf/benchmark.js index 8900062..903d04a 100755 --- a/perf/benchmark.js +++ b/perf/benchmark.js @@ -21,7 +21,7 @@ var _ = require("lodash"); var Benchmark = require("benchmark"); -var benchOptions = {defer: true, minSamples: 1, maxTime: 1}; +var benchOptions = {defer: true, minSamples: 1, maxTime: 2}; var exec = require("child_process").exec; var fs = require("fs"); var path = require("path"); @@ -34,8 +34,9 @@ var version1 = process.argv[3] || "current"; var versionNames = [version0, version1]; var versions; var wins = {}; -wins[version0] = 0; -wins[version1] = 0; +var totalTime = {}; +totalTime[version0] = wins[version0] = 0; +totalTime[version1] = wins[version1] = 0; console.log("Comparing " + version0 + " with " + version1); console.log("--------------------------------------"); @@ -44,20 +45,26 @@ console.log("--------------------------------------"); async.eachSeries(versionNames, cloneVersion, function (err) { versions = versionNames.map(requireVersion); - var suites = suiteConfigs.map(createSuite); + var suites = suiteConfigs + .map(setDefaultOptions) + .reduce(handleMultipleArgs, []) + .map(setName) + .map(createSuite); async.eachSeries(suites, runSuite, function () { - var wins0 = wins[version0]; - var wins1 = wins[version1]; - - if (wins0 > wins1) { + var totalTime0 = Math.round(totalTime[version0]); + var totalTime1 = Math.round(totalTime[version1]); + + if ( Math.abs((totalTime0 / totalTime1) - 1) < 0.01) { + // if < 1% difference, we're likely within the margins of error + console.log("Both versions are about equal " + + "(" + totalTime0 + "ms total vs. " + totalTime1 + "ms total)"); + } else if (totalTime0 < totalTime1) { console.log(version0 + " faster overall " + - "(" + wins0 + " wins vs. " + wins1 +" wins)"); - } else if (wins1 > wins0) { + "(" + totalTime0 + "ms total vs. " + totalTime1 + "ms total)"); + } else if (totalTime1 < totalTime0) { console.log(version1 + " faster overall " + - "(" + wins1 + " wins vs. " + wins0 +" wins)"); - } else { - console.log("Both versions are equal"); + "(" + totalTime1 + "ms total vs. " + totalTime0 + "ms total)"); } }); }); @@ -68,27 +75,48 @@ function runSuite(suite, callback) { }).run({async: true}); } +function setDefaultOptions(suiteConfig) { + suiteConfig.args = suiteConfig.args || [[]]; + suiteConfig.setup = suiteConfig.setup || function () {}; + return suiteConfig; +} + +function handleMultipleArgs(list, suiteConfig) { + return list.concat(suiteConfig.args.map(function (args) { + return _.defaults({args: args}, suiteConfig); + })); +} + +function setName(suiteConfig) { + suiteConfig.name = suiteConfig.name + "(" + suiteConfig.args.join(",") + ")"; + return suiteConfig; +} + function createSuite(suiteConfig) { var suite = new Benchmark.Suite(); + var args = suiteConfig.args; function addBench(version, versionName) { - var title = suiteConfig.name + " " + versionName; - suite.add(title, function (deferred) { + var name = suiteConfig.name + " " + versionName; + suite.add(name, function (deferred) { suiteConfig.fn(versions[0], function () { deferred.resolve(); }); }, _.extend({ versionName: versionName, - setup: suiteConfig.setup + setup: _.partial.apply(null, [suiteConfig.setup].concat(args)) }, benchOptions)); } addBench(versions[0], versionNames[0]); addBench(versions[1], versionNames[1]); + return suite.on('cycle', function(event) { var mean = event.target.stats.mean * 1000; console.log(event.target + ", " + mean.toFixed(1) + "ms per sample"); + var version = event.target.options.versionName; + totalTime[version] += mean; }) .on('complete', function() { var fastest = this.filter('fastest'); diff --git a/perf/suites.js b/perf/suites.js index 884b65e..2e1beaa 100644 --- a/perf/suites.js +++ b/perf/suites.js @@ -1,14 +1,14 @@ var _ = require("lodash"); -var parallel1000Funcs = _.range(1000).map(function () { - return function (cb) { cb(); }; -}); -var parallel10Funcs = _.range(10).map(function () { - return function (cb) { cb(); }; -}); +var tasks; module.exports = [ { - name: "each(10)", + name: "each", + // args lists are passed to the setup function + args: [[10], [300], [10000]], + setup: function(count) { + tasks = Array(count); + }, fn: function (async, done) { async.each(Array(10), function (num, cb) { async.setImmediate(cb); @@ -16,93 +16,77 @@ module.exports = [ } }, { - name: "each(10000)", + name: "eachSeries", + args: [[10], [300], [10000]], + setup: function(count) { + tasks = Array(count); + }, fn: function (async, done) { - async.each(Array(10000), function (num, cb) { + async.eachSeries(tasks, function (num, cb) { async.setImmediate(cb); }, done); } }, { - name: "eachSeries(10)", + name: "eachLimit", + args: [[10], [300], [10000]], + setup: function(count) { + tasks = Array(count); + }, fn: function (async, done) { - async.eachSeries(Array(10), function (num, cb) { + async.eachLimit(tasks, 4, function (num, cb) { async.setImmediate(cb); }, done); } }, { - name: "eachSeries(10000)", + name: "parallel", + args: [[10], [100], [1000]], + setup: function (count) { + tasks = _.range(count).map(function () { + return function (cb) { cb(); }; + }); + }, fn: function (async, done) { - async.eachSeries(Array(10000), function (num, cb) { - async.setImmediate(cb); - }, done); - } - }, - { - name: "parallel(10)", - fn: function (async, done) { - async.parallel(parallel10Funcs, done); - } - }, - { - name: "parallel(1000)", - fn: function (async, done) { - async.parallel(parallel1000Funcs, done); - } - }, - { - name: "queue(1000)", - fn: function (async, done) { - var numEntries = 1000; - var q = async.queue(worker, 1); - for (var i = 1; i <= numEntries; i++) { - q.push({num: i}); - } - function worker(task, callback) { - if (task.num === numEntries) { - return done(); - } - setImmediate(callback); - } + async.parallel(tasks, done); } }, { - name: "queue(30000)", + name: "series", + args: [[10], [100], [1000]], + setup: function (count) { + tasks = _.range(count).map(function () { + return function (cb) { cb(); }; + }); + }, fn: function (async, done) { - var numEntries = 30000; - var q = async.queue(worker, 1); - for (var i = 1; i <= numEntries; i++) { - q.push({num: i}); - } - function worker(task, callback) { - if (task.num === numEntries) { - return done(); - } - setImmediate(callback); - } + async.series(tasks, done); } }, { - name: "queue(100000)", + name: "waterfall", + args: [[10], [100], [1000]], + setup: function (count) { + tasks = [ + function (cb) { + return cb(null, 1); + } + ].concat(_.range(count).map(function (i) { + return function (arg, cb) { cb(null, i); }; + })); + }, fn: function (async, done) { - var numEntries = 100000; - var q = async.queue(worker, 1); - for (var i = 1; i <= numEntries; i++) { - q.push({num: i}); - } - function worker(task, callback) { - if (task.num === numEntries) { - return done(); - } - setImmediate(callback); - } + async.waterfall(tasks, done); } }, { - name: "queue(200000)", + name: "queue", + args: [[1000], [30000], [100000], [200000]], + setup: function (count) { + tasks = count; + }, fn: function (async, done) { - var numEntries = 200000; + var numEntries = tasks; var q = async.queue(worker, 1); for (var i = 1; i <= numEntries; i++) { q.push({num: i}); -- cgit v1.2.1