benchmark: add `test` and `all` options and improve errors

This adds a new `test` option. Using it automatically uses a single
minimal option matrix to verify the benchmark works as expected.

Using the new `all` option makes sure all test suites are run.

On top of that the benchmarks will from now on report properly
what category might have a typo, if any.

The http duration was also refactored to use a option instead of
relying on a configuration setting.

The fixture folder is ignored as test suite from now on.

PR-URL: https://github.com/nodejs/node/pull/31396
Fixes: https://github.com/nodejs/node/issues/31083
Reviewed-By: Sam Roberts <vieuxtech@gmail.com>
Reviewed-By: Rich Trott <rtrott@gmail.com>

4 files changed, 17 insertions, 9 deletions

diff --git a/benchmark/http2/compat.js b/benchmark/http2/compat.js
index 5d06ccf317..2c7e732b07 100644
--- a/benchmark/http2/compat.js
+++ b/benchmark/http2/compat.js
@@ -9,10 +9,11 @@ const bench = common.createBenchmark(main, {
   requests: [100, 1000, 5000],
   streams: [1, 10, 20, 40, 100, 200],
   clients: [2],
-  benchmarker: ['h2load']
+  benchmarker: ['test-double-http2'],
+  duration: 5
 }, { flags: ['--no-warnings'] });
 
-function main({ requests, streams, clients }) {
+function main({ requests, streams, clients, duration }) {
   const http2 = require('http2');
   const server = http2.createServer();
   server.on('request', (req, res) => {
@@ -29,7 +30,8 @@ function main({ requests, streams, clients }) {
       requests,
       maxConcurrentStreams: streams,
       clients,
-      threads: clients
+      threads: clients,
+      duration
     }, () => { server.close(); });
   });
 }
diff --git a/benchmark/http2/respond-with-fd.js b/benchmark/http2/respond-with-fd.js
index 35856490f7..5bf5988d16 100644
--- a/benchmark/http2/respond-with-fd.js
+++ b/benchmark/http2/respond-with-fd.js
@@ -10,10 +10,11 @@ const bench = common.createBenchmark(main, {
   requests: [100, 1000, 5000],
   streams: [1, 10, 20, 40, 100, 200],
   clients: [2],
-  benchmarker: ['h2load']
+  benchmarker: ['test-double-http2'],
+  duration: 5
 }, { flags: ['--no-warnings'] });
 
-function main({ requests, streams, clients }) {
+function main({ requests, streams, clients, duration }) {
   fs.open(file, 'r', (err, fd) => {
     if (err)
       throw err;
@@ -30,6 +31,7 @@ function main({ requests, streams, clients }) {
         requests,
         maxConcurrentStreams: streams,
         clients,
+        duration,
         threads: clients
       }, () => server.close());
     });
diff --git a/benchmark/http2/simple.js b/benchmark/http2/simple.js
index aab7c6b609..929c4c655e 100644
--- a/benchmark/http2/simple.js
+++ b/benchmark/http2/simple.js
@@ -9,10 +9,11 @@ const bench = common.createBenchmark(main, {
   requests: [100, 1000, 5000],
   streams: [1, 10, 20, 40, 100, 200],
   clients: [2],
-  benchmarker: ['h2load']
+  benchmarker: ['test-double-http2'],
+  duration: 5
 }, { flags: ['--no-warnings'] });
 
-function main({ requests, streams, clients }) {
+function main({ requests, streams, clients, duration }) {
   const http2 = require('http2');
   const server = http2.createServer();
   server.on('stream', (stream) => {
@@ -27,6 +28,7 @@ function main({ requests, streams, clients }) {
       requests,
       maxConcurrentStreams: streams,
       clients,
+      duration,
       threads: clients
     }, () => { server.close(); });
   });
diff --git a/benchmark/http2/write.js b/benchmark/http2/write.js
index fc3203c6e5..7ea8b2c02d 100644
--- a/benchmark/http2/write.js
+++ b/benchmark/http2/write.js
@@ -6,10 +6,11 @@ const bench = common.createBenchmark(main, {
   streams: [100, 200, 1000],
   length: [64 * 1024, 128 * 1024, 256 * 1024, 1024 * 1024],
   size: [100000],
-  benchmarker: ['h2load']
+  benchmarker: ['test-double-http2'],
+  duration: 5
 }, { flags: ['--no-warnings'] });
 
-function main({ streams, length, size }) {
+function main({ streams, length, size, duration }) {
   const http2 = require('http2');
   const server = http2.createServer();
   server.on('stream', (stream) => {
@@ -29,6 +30,7 @@ function main({ streams, length, size }) {
     bench.http({
       path: '/',
       requests: 10000,
+      duration,
       maxConcurrentStreams: streams,
     }, () => { server.close(); });
   });