Merge branch 'tcanabrava/cache-server-test' into 'master'

CI For the Cache Server

Closes #1362

See merge request BuildStream/buildstream!1997

16 files changed, 254 insertions, 8 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 02c7f6155..55a740ccb 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -171,10 +171,9 @@ tests-fedora-update-deps:
 
     - su buildstream -c "${TEST_COMMAND}"
 
-tests-remote-execution:
+.compose-test-boilerplate: &remote-test
   allow_failure: true
   image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:31-${DOCKER_IMAGE_VERSION}
-  <<: *tests
   before_script:
     - dnf install -y docker docker-compose
     - docker-compose --file ${COMPOSE_MANIFEST} up --detach
@@ -184,17 +183,33 @@ tests-remote-execution:
     - docker-compose --file ${COMPOSE_MANIFEST} down
   services:
     - docker:stable-dind
+
+.docker-variables-boilerplate: &docker-variables
+  DOCKER_HOST: tcp://docker:2375
+  DOCKER_DRIVER: overlay2
+  # Required to be able to connect to the docker daemon. See https://gitlab.com/gitlab-org/gitlab-runner/issues/4501
+  DOCKER_TLS_CERTDIR: ""
+
+tests-remote-execution:
+  <<: *tests
+  <<: *remote-test
   variables:
-    DOCKER_HOST: tcp://docker:2375
-    DOCKER_DRIVER: overlay2
-    # Required to be able to connect to the docker daemon. See https://gitlab.com/gitlab-org/gitlab-runner/issues/4501
-    DOCKER_TLS_CERTDIR: ""
-    COMPOSE_MANIFEST: .gitlab-ci/buildgrid-compose.yml
+    <<: *docker-variables
+    COMPOSE_MANIFEST: .gitlab-ci/buildgrid-remote-execution.yml
     ARTIFACT_CACHE_SERVICE: http://docker:50052
     REMOTE_EXECUTION_SERVICE: http://docker:50051
     SOURCE_CACHE_SERVICE: http://docker:50052
     PYTEST_ARGS: "--color=yes --remote-execution"
 
+tests-remote-cache:
+  <<: *tests
+  <<: *remote-test
+  variables:
+    <<: *docker-variables
+    COMPOSE_MANIFEST: .gitlab-ci/cache-server.yml
+    ARTIFACT_CACHE_SERVICE: http://docker:50052
+    PYTEST_ARGS: "--color=yes --remote-cache"
+
 tests-no-usedevelop:
   # Ensure that tests also pass without `--develop` flag.
   image: registry.gitlab.com/buildstream/buildstream-docker-images/testsuite-fedora:31-${DOCKER_IMAGE_VERSION}
diff --git a/.gitlab-ci/buildgrid-compose.yml b/.gitlab-ci/buildgrid-remote-execution.yml
index fd45c45af..fd45c45af 100644
--- a/.gitlab-ci/buildgrid-compose.yml
+++ b/.gitlab-ci/buildgrid-remote-execution.yml
diff --git a/.gitlab-ci/cache-server.yml b/.gitlab-ci/cache-server.yml
new file mode 100644
index 000000000..24ccd1672
--- /dev/null
+++ b/.gitlab-ci/cache-server.yml
@@ -0,0 +1,36 @@
+##
+# A Builstream Artifact Server
+#
+# Spins-up a unnamed and unauthenticated grid:
+#  - artifacts server at: http://localhost:50052
+#
+# BuildStream configuration snippet:
+#
+#    artifacts:
+#      url: http://localhost:50052
+#      push: true
+#
+# Basic usage:
+#  - docker-compose -f cache-server.yml up
+#  - docker-compose -f cache-server.yml down
+#
+version: "3.2"
+
+services:
+  controller:
+    image: buildstream/buildstream:dev
+    command: ["bst-artifact-server","--port",
+      "50052",
+      "--enable-push",
+      "/artifacts"
+    ]
+    ports:
+      - 50052:50052
+    networks:
+      - grid
+networks:
+  grid:
+    driver: bridge
+
+volumes:
+  cache:
diff --git a/setup.cfg b/setup.cfg
index 5a3a75d67..e8ba5ac35 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -18,6 +18,7 @@ markers =
     datafiles: data files for tests
     integration: run test only if --integration option is specified
     remoteexecution: run test only if --remote-execution option is specified
+    remotecache: run tests only if --remote-cache option is specified
 xfail_strict=True
 
 [mypy]
diff --git a/src/buildstream/testing/runcli.py b/src/buildstream/testing/runcli.py
index 0f08bd141..af15b56f1 100644
--- a/src/buildstream/testing/runcli.py
+++ b/src/buildstream/testing/runcli.py
@@ -776,7 +776,7 @@ def cli_remote_execution(tmpdir, remote_services):
     fixture = CliRemote(directory)
 
     if remote_services.artifact_service:
-        fixture.configure({"artifacts": [{"url": remote_services.artifact_service,}]})
+        fixture.configure({"artifacts": [{"url": remote_services.artifact_service, "push": True,}]})
 
     remote_execution = {}
     if remote_services.action_service:
diff --git a/tests/conftest.py b/tests/conftest.py
index bb4611a11..28e120d5e 100755
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -49,6 +49,7 @@ def pytest_addoption(parser):
     parser.addoption("--integration", action="store_true", default=False, help="Run integration tests")
     parser.addoption("--plugins", action="store_true", default=False, help="Run only plugins tests")
     parser.addoption("--remote-execution", action="store_true", default=False, help="Run remote-execution tests only")
+    parser.addoption("--remote-cache", action="store_true", default=False, help="Run remote-cache tests only")
 
 
 def pytest_runtest_setup(item):
@@ -67,6 +68,16 @@ def pytest_runtest_setup(item):
         if item.get_closest_marker("remoteexecution"):
             pytest.skip("skipping remote-execution test")
 
+    # With --remote-cache: only run tests marked with 'remotecache'
+    if item.config.getvalue("remote_cache"):
+        if not item.get_closest_marker("remotecache"):
+            pytest.skip("skipping non remote-cache test")
+
+    # Without --remote-cache: skip tests marked with 'remotecache'
+    else:
+        if item.get_closest_marker("remotecache"):
+            pytest.skip("skipping remote-cache test")
+
     # With --plugins only run plugins tests
     if item.config.getvalue("plugins"):
         if not item.get_closest_marker("generic_source_test"):
diff --git a/tests/remotecache/project/elements/autotools/amhello.bst b/tests/remotecache/project/elements/autotools/amhello.bst
new file mode 100644
index 000000000..3af67ef3c
--- /dev/null
+++ b/tests/remotecache/project/elements/autotools/amhello.bst
@@ -0,0 +1,26 @@
+kind: autotools
+description: Autotools test
+
+depends:
+- base.bst
+
+sources:
+- kind: tar
+  url: project_dir:/files/amhello.tar.gz
+  ref: 9ba123fa4e660929e9a0aa99f0c487b7eee59c5e7594f3284d015640b90f5590
+
+config:
+
+  configure-commands:
+    - |
+      %{autogen}
+    - |
+      %{configure}
+    - |
+      date +%s > config-time
+
+  build-commands:
+    - |
+      %{make}
+    - |
+      date +%s > build-time
diff --git a/tests/remotecache/project/elements/base.bst b/tests/remotecache/project/elements/base.bst
new file mode 100644
index 000000000..428afa736
--- /dev/null
+++ b/tests/remotecache/project/elements/base.bst
@@ -0,0 +1,5 @@
+# elements/base.bst
+
+kind: stack
+depends:
+  - base/base-alpine.bst
diff --git a/tests/remotecache/project/elements/base/base-alpine.bst b/tests/remotecache/project/elements/base/base-alpine.bst
new file mode 100644
index 000000000..c5833095d
--- /dev/null
+++ b/tests/remotecache/project/elements/base/base-alpine.bst
@@ -0,0 +1,17 @@
+kind: import
+
+description: |
+  Alpine Linux base for tests
+
+  Generated using the `tests/integration-tests/base/generate-base.sh` script.
+
+sources:
+  - kind: tar
+    base-dir: ''
+    (?):
+    - arch == "x86-64":
+        ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639
+        url: "alpine:integration-tests-base.v1.x86_64.tar.xz"
+    - arch == "aarch64":
+        ref: 431fb5362032ede6f172e70a3258354a8fd71fcbdeb1edebc0e20968c792329a
+        url: "alpine:integration-tests-base.v1.aarch64.tar.xz"
diff --git a/tests/remotecache/project/elements/build-shell/buildtree-fail.bst b/tests/remotecache/project/elements/build-shell/buildtree-fail.bst
new file mode 100644
index 000000000..a3b515a9a
--- /dev/null
+++ b/tests/remotecache/project/elements/build-shell/buildtree-fail.bst
@@ -0,0 +1,13 @@
+kind: manual
+description: |
+  Puts a file in the build tree so that build tree caching and staging can be tested,
+  then deliberately failing to build so we can check the output.
+
+depends:
+  - filename: base.bst
+    type: build
+
+config:
+  build-commands:
+    - "echo 'Hi' > %{build-root}/test"
+    - "false"
diff --git a/tests/remotecache/project/elements/build-shell/buildtree.bst b/tests/remotecache/project/elements/build-shell/buildtree.bst
new file mode 100644
index 000000000..d5cf256be
--- /dev/null
+++ b/tests/remotecache/project/elements/build-shell/buildtree.bst
@@ -0,0 +1,11 @@
+kind: manual
+description: |
+  Puts a file in the build tree so that build tree caching and staging can be tested.
+
+depends:
+  - filename: base.bst
+    type: build
+
+config:
+  build-commands:
+    - "echo 'Hi' > %{build-root}/test"
diff --git a/tests/remotecache/project/elements/no-runtime-deps.bst b/tests/remotecache/project/elements/no-runtime-deps.bst
new file mode 100644
index 000000000..ca76aaf9f
--- /dev/null
+++ b/tests/remotecache/project/elements/no-runtime-deps.bst
@@ -0,0 +1,9 @@
+kind: manual
+description: Test element without runtime dependencies
+
+build-depends:
+- autotools/amhello.bst
+
+config:
+  install-commands:
+  - echo Test > %{install-root}/test
diff --git a/tests/remotecache/project/files/amhello.tar.gz b/tests/remotecache/project/files/amhello.tar.gz
new file mode 100644
index 000000000..afe189908
--- /dev/null
+++ b/tests/remotecache/project/files/amhello.tar.gz
diff --git a/tests/remotecache/project/files/dev-files/usr/include/pony.h b/tests/remotecache/project/files/dev-files/usr/include/pony.h
new file mode 100644
index 000000000..40bd0c2e7
--- /dev/null
+++ b/tests/remotecache/project/files/dev-files/usr/include/pony.h
@@ -0,0 +1,12 @@
+#ifndef __PONY_H__
+#define __PONY_H__
+
+#define PONY_BEGIN "Once upon a time, there was a pony."
+#define PONY_END "And they lived happily ever after, the end."
+
+#define MAKE_PONY(story)  \
+  PONY_BEGIN \
+  story \
+  PONY_END
+
+#endif /* __PONY_H__ */
diff --git a/tests/remotecache/project/project.conf b/tests/remotecache/project/project.conf
new file mode 100644
index 000000000..27b5f5e34
--- /dev/null
+++ b/tests/remotecache/project/project.conf
@@ -0,0 +1,25 @@
+# Project config for frontend build test
+name: test
+min-version: 2.0
+element-path: elements
+aliases:
+  alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/
+  project_dir: file://{project_dir}
+options:
+  linux:
+    type: bool
+    description: Whether to expect a linux platform
+    default: True
+  arch:
+    type: arch
+    description: Current architecture
+    values:
+      - x86-64
+      - aarch64
+split-rules:
+  test:
+    - |
+      /tests
+    - |
+      /tests/*
+
diff --git a/tests/remotecache/simple.py b/tests/remotecache/simple.py
new file mode 100644
index 000000000..74c44ae1e
--- /dev/null
+++ b/tests/remotecache/simple.py
@@ -0,0 +1,65 @@
+# Pylint doesn't play well with fixtures and dependency injection from pytest
+# pylint: disable=redefined-outer-name
+
+import os
+import pytest
+
+from buildstream.testing import cli_remote_execution as cli  # pylint: disable=unused-import
+from buildstream.testing.integration import assert_contains
+
+
+pytestmark = pytest.mark.remotecache
+
+
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
+
+# Test building an executable with a remote cache:
+@pytest.mark.datafiles(DATA_DIR)
+def test_remote_autotools_build(cli, datafiles):
+    project = str(datafiles)
+    checkout = os.path.join(cli.directory, "checkout")
+    element_name = "autotools/amhello.bst"
+
+    result = cli.run(project=project, args=["build", element_name])
+    result.assert_success()
+    assert element_name in result.get_pushed_elements()
+
+    result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
+    result.assert_success()
+
+    assert_contains(
+        checkout,
+        [
+            "/usr",
+            "/usr/lib",
+            "/usr/bin",
+            "/usr/share",
+            "/usr/bin/hello",
+            "/usr/share/doc",
+            "/usr/share/doc/amhello",
+            "/usr/share/doc/amhello/README",
+        ],
+    )
+
+    # then remove it locally
+    result = cli.run(project=project, args=["artifact", "delete", element_name])
+    result.assert_success()
+
+    result = cli.run(project=project, args=["build", element_name])
+    result.assert_success()
+    assert element_name in result.get_pulled_elements()
+
+
+# Test building an executable with a remote cache:
+@pytest.mark.datafiles(DATA_DIR)
+def test_remote_autotools_build_no_cache(cli, datafiles):
+    project = str(datafiles)
+    checkout = os.path.join(cli.directory, "checkout")
+    element_name = "autotools/amhello.bst"
+
+    cli.configure({"artifacts": {"url": "http://fake.url.service", "push": True}})
+    result = cli.run(project=project, args=["build", element_name])
+    result.assert_success()
+
+    assert """WARNING Failed to initialize remote""" in result.stderr
+    assert """Remote initialisation failed with status UNAVAILABLE: DNS resolution failed""" in result.stderr