Initial benchmarking script

Has code to generate a Docker image for each version under test. Does
not yet reuse these images between runs or clean them up.

Does not run any serious tests yet.

1 files changed, 226 insertions, 0 deletions

diff --git a/contrib/benchmark/__main__.py b/contrib/benchmark/__main__.py
new file mode 100644
index 000000000..9d7c14b20
--- /dev/null
+++ b/contrib/benchmark/__main__.py
@@ -0,0 +1,226 @@
+# buildstream benchmark script
+
+import click
+import docker
+
+import json
+import logging
+import sys
+
+
+class BstVersionSpec():
+    """Specifies a version of BuildStream to be tested.
+
+    Note that the test harness uses Docker to ensure that the necessary
+    dependencies for a particular version of BuildStream are made available in a
+    reproducible way.
+
+    Args:
+       name (str): Identifier for this version, e.g. '1.0.0' or 'master'
+       base_docker_image (str): Name of the Docker image containing
+                                dependencies, or 'None' to use the default
+       base_docker_ref (str): Tag or digest checksum specifing the exact version
+                              of the Docker image to use. Defaults to 'name'.
+       buildstream_repo (str): Git repository holding the version of
+                               BuildStream to test, or 'None' to use the default
+       buildstream_ref (str): Git ref pointing to the version of BuildStream to
+                              be tested. Defaults to value of 'name'.
+    """
+
+    DEFAULT_BASE_DOCKER_IMAGE = 'docker.io/buildstream/buildstream-fedora'
+    DEFAULT_BUILDSTREAM_REPO = 'https://gitlab.com/BuildStream/BuildStream'
+
+    def __init__(self, name, base_docker_image=None, base_docker_ref=None,
+                 buildstream_repo=None, buildstream_ref=None):
+        self.name = name
+        self.base_docker_image = base_docker_image or self.DEFAULT_BASE_DOCKER_IMAGE
+        self.base_docker_ref = base_docker_ref or name
+        self.buildstream_repo = buildstream_repo or self.DEFAULT_BUILDSTREAM_REPO
+        self.buildstream_ref = buildstream_ref or name
+
+
+class BstVersion():
+    """Represents a version of BuildStream being tested.
+
+    Args:
+        spec (BstVersionSpec): Version specification (provided by test config).
+        image (docker.models.images.Image): Docker image to be used for testing the specified BuildStream version.
+        base_docker_digest (str): Exact sha256 digest of the base Docker image used.
+        buildstream_commit (str): Exact sha1 hash of the BuildStream commit used.
+        bst_version_string (str): Output of `bst --version` inside the container.
+    """
+    def __init__(self, spec, image, base_docker_digest, buildstream_commit, bst_version_string):
+        self.spec = spec
+        self.image = image
+        self.base_docker_digest = base_docker_digest
+        self.buildstream_commit = buildstream_commit
+        self.bst_version_string = bst_version_string
+
+    def describe(self):
+        return {
+            'name': self.spec.name,
+            'base_docker_image': self.spec.base_docker_image,
+            'base_docker_ref': self.spec.base_docker_ref,
+            'base_docker_digest': self.base_docker_digest,
+            'buildstream_repo': self.spec.buildstream_repo,
+            'buildstream_ref': self.spec.buildstream_ref,
+            'buildstream_commit': self.buildstream_commit,
+            'bst_version_string': self.bst_version_string,
+        }
+
+
+class BenchmarkTest():
+    """Specifies a test to be run."""
+    def __init__(self, name, commandline):
+        self.name = name
+        self.commandline = commandline
+
+
+# FIXME: put these in a config file
+
+# Using exact digests saves a little time in the 'Preparing' stage as the Docker
+# daemon doesn't check for updates ... not sure it's worth the effort though.
+
+versions_to_test = [
+    BstVersionSpec(name='1.0.0', base_docker_ref='1.0.0'),
+    BstVersionSpec(name='1.0.1', base_docker_ref='1.0.1'),
+    BstVersionSpec(name='stable', base_docker_ref='1.0.1', buildstream_ref='bst-1.0'),
+    BstVersionSpec(name='master', base_docker_ref='latest', buildstream_ref='master'),
+]
+
+tests_to_run = [
+    BenchmarkTest(name='Startup time', commandline=['bst', '--help'])
+]
+
+
+def parse_output_var(line, prefix):
+    if line.startswith(prefix + ': '):
+        return line.split(': ', 1)[1]
+    else:
+        raise RuntimeError("Line didn't start with expected prefix {}: {}".format(prefix, line))
+
+
+# Creates a Docker image that can be used to test a specific version of
+# BuildStream, as described by 'version_spec'.
+def prepare_version(docker_client, version_spec):
+    logging.info("Preparing version '{}'".format(version_spec.name))
+
+    # The base image may have a symbolic name, so check that we have the latest
+    # version.
+    logging.debug("Pulling Docker image {}:{}".format(version_spec.base_docker_image, version_spec.base_docker_ref))
+    docker_client.images.pull(version_spec.base_docker_image, version_spec.base_docker_ref)
+
+    base_docker_image_with_version = '{}:{}'.format(version_spec.base_docker_image, version_spec.base_docker_ref)
+
+    # The digest identifies the *exact* Docker image that we are using as a base.
+    base_docker_image = docker_client.images.get(base_docker_image_with_version)
+    base_docker_digest = base_docker_image.id
+
+    # Start a container from the base image and run the setup script.
+    #
+    # Due to Docker limitations, we run setup as a single shell script. Luckily
+    # we are only interpolating URLs and Git refs into this which should not
+    # contain spaces or other weird characters.
+    script = '\n'.join([
+        'set -e',
+        'git clone {} ./buildstream --branch {}'.format(version_spec.buildstream_repo, version_spec.buildstream_ref),
+        'pip3 install --user ./buildstream',
+        'echo "buildstream_commit: $(git -C ./buildstream rev-parse HEAD)"',
+        'echo "bst_version_string: $(bst --version)"',
+    ])
+
+    try:
+        container = docker_client.containers.run(base_docker_image_with_version,
+                                                 command=['/bin/sh', '-c', script],
+                                                 detach=True)
+
+        exit_code = container.wait()
+        if exit_code != 0:
+            error = container.logs(stdout=True, stderr=True).decode('unicode-escape')
+            raise RuntimeError("Container setup script exited with code. Error output:\n{}".format(error))
+
+        output = container.logs(stdout=True).decode('unicode-escape')
+        logging.debug("Output from setup script: {}".format(output))
+
+        lines = output.splitlines()
+
+        buildstream_commit = parse_output_var(lines[-2], 'buildstream_commit')
+        bst_version_string = parse_output_var(lines[-1], 'bst_version_string')
+
+        # FIXME: we never clean up these images; which is perhaps OK but we
+        # should at least label them, and if possible reuse them.
+        image = container.commit()
+    except:
+        container.remove()
+        raise
+
+    return BstVersion(version_spec, image, base_docker_digest,
+                      buildstream_commit, bst_version_string)
+
+
+def run_benchmark(docker_client, version, test_spec):
+    logging.info("Running test '{}' on version '{}'".format(test_spec.name, version.spec.name))
+
+    logging.debug("Starting container from {}".format(version.image))
+
+    container = docker_client.containers.run(
+        version.image, command='echo FOOOOOO', auto_remove=True,
+        detach=True, labels=['buildstream-benchmark'])
+
+    logging.debug("Waiting for container to finish")
+    result = container.wait()
+
+    logging.debug("Output: {}".format(container.logs()))
+    logging.debug("Returncode: {}".format(result))
+
+    logging.info("Test completed")
+
+
+@click.command()
+@click.option('--debug/--no-debug', default=False)
+def run(debug):
+    if debug:
+        logging.basicConfig(stream=sys.stderr, level=logging.DEBUG)
+    else:
+        logging.basicConfig(stream=sys.stderr, level=logging.INFO)
+
+    logging.info("BuildStream benchmark runner started")
+
+    docker_client = docker.from_env()
+
+    versions = []
+
+    for version_spec in versions_to_test:
+        version = prepare_version(docker_client, version_spec)
+        versions.append(version)
+
+    for version in versions:
+        for test in tests_to_run:
+            run_benchmark(docker_client, version, test)
+
+    logging.info("Writing results to stdout")
+    output = {
+        'start_time': 'x',
+        'end_time': 'y',
+        'host_info': {'foo': 'bar'},
+        'versions': [ version.describe() for version in versions],
+        'tests': [
+            {
+                'name': 'Startup time',
+                'results': [
+                    {
+                        'version': '1.0.1',
+                        'time': 0.1,
+                    },
+                    {
+                        'version': 'master',
+                        'time': 0.5,
+                    }
+                ]
+            }
+        ]
+    }
+    json.dump(output, sys.stdout, indent=4)
+
+
+run(prog_name="benchmark")