diff options
author | Tom Pollard <tom.pollard@codethink.co.uk> | 2018-11-14 17:33:24 +0000 |
---|---|---|
committer | Tom Pollard <tom.pollard@codethink.co.uk> | 2019-01-29 15:11:15 +0000 |
commit | 51bb1341e7db35538aca0aa0a1ca273ee373be0c (patch) | |
tree | fc86c2c6e570e55e2541593fb2348f95e2d2bcea | |
parent | 99f0d726f993f2c3d5e2a4683fa35d7a28efc2ad (diff) | |
download | buildstream-tpollard/752.tar.gz |
Add ability to build without interacting with available artifacttpollard/752
servers.
By adding a new user context and bst main option, we can provide
more granular control of what remotes are utilised during relevant
bst pull and push actions.
_artifactcache/artifactcache.py: Expand setup_remotes() to consider
the user context value of 'use_remotes'. Allowing the user to omit
interacting with all, or all non user defined remotes.
tests/frontend: pull.py & push.py addition of tests for checking
expected behaviour. Ensures interaction with user, project or no
remotes.
-rw-r--r-- | NEWS | 9 | ||||
-rw-r--r-- | buildstream/_artifactcache.py | 17 | ||||
-rw-r--r-- | tests/frontend/pull.py | 72 | ||||
-rw-r--r-- | tests/frontend/push.py | 65 |
4 files changed, 157 insertions, 6 deletions
@@ -122,6 +122,15 @@ buildstream 1.3.1 'shell', 'show', 'source-checkout', 'track', 'workspace close' and 'workspace reset' commands are affected. + o bst interaction with defined artifact servers can be controlled more granularly. + This can be done via the user configuration option `useremotes` or via the bst cli + main option '--use-remotes'. This can be set as 'none', 'user' or the default value + 'all'. Unless specifically overriden, when considering wether to pull or push to + available artifact servers (be it user or project defined) this optional config option + will be used. Setting this value to 'user' for example and performing a build would + lead to any project or junction defined artifact server to be ignored, whilst still + attempting to any user defined remotes. + ================= buildstream 1.1.5 diff --git a/buildstream/_artifactcache.py b/buildstream/_artifactcache.py index 48ab278c4..c162c3974 100644 --- a/buildstream/_artifactcache.py +++ b/buildstream/_artifactcache.py @@ -148,7 +148,7 @@ class ArtifactCache(): # Sets up which remotes to use # # Args: - # use_config (bool): Whether to use project configuration + # use_config (bool): Whether to use configuration # remote_url (str): Remote artifact cache URL # # This requires that all of the projects which are to be processed in the session @@ -167,11 +167,16 @@ class ArtifactCache(): self._set_remotes([ArtifactCacheSpec(remote_url, push=True)]) has_remote_caches = True if use_config: - for project in self.context.get_projects(): - artifact_caches = _configured_remote_artifact_cache_specs(self.context, project) - if artifact_caches: # artifact_caches is a list of ArtifactCacheSpec instances - self._set_remotes(artifact_caches, project=project) - has_remote_caches = True + if self.context.use_remotes == 'all': + for project in self.context.get_projects(): + artifact_caches = _configured_remote_artifact_cache_specs(self.context, project) + if artifact_caches: # artifact_caches is a list of ArtifactCacheSpec instances + self._set_remotes(artifact_caches, project=project) + has_remote_caches = True + # If configured to only use user configured remotes, pass existing user cache spec + elif self.context.use_remotes == 'user' and self.context.artifact_cache_specs: + self._set_remotes(self.context.artifact_cache_specs) + has_remote_caches = True if has_remote_caches: self._initialize_remotes() diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py index 20b740948..57e1bfdb4 100644 --- a/tests/frontend/pull.py +++ b/tests/frontend/pull.py @@ -408,3 +408,75 @@ def test_pull_missing_notifies_user(caplog, cli, tmpdir, datafiles): assert "INFO Remote ({}) does not have".format(share.repo) in result.stderr assert "SKIPPED Pull" in result.stderr + + +# Tests that: +# +# * The bst main option --use-remotes limits remote action +# as expected for pull jobs +# +@pytest.mark.datafiles(DATA_DIR) +def test_useremotes_cli_options(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + + with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\ + create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject: + + # Add shareproject repo url to project.conf + with open(os.path.join(project, "project.conf"), "a") as projconf: + projconf.write("artifacts:\n url: {}\n push: True".format(shareproject.repo)) + + # First build the target element and push to the remotes. + # We need the artifact available in the remotes to test against. + cli.configure({ + 'artifacts': {'url': shareuser.repo, 'push': True} + }) + result = cli.run(project=project, args=['build', 'target.bst']) + result.assert_success() + assert cli.get_element_state(project, 'target.bst') == 'cached' + + # Assert that everything is now cached in the remotes. + all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst'] + for element_name in all_elements: + assert_shared(cli, shareuser, project, element_name) + assert_shared(cli, shareproject, project, element_name) + + # Now we've pushed, delete the user's local artifact cache + artifacts = os.path.join(cli.directory, 'artifacts') + shutil.rmtree(artifacts) + + # Assert that nothing is cached locally anymore + for element_name in all_elements: + assert cli.get_element_state(project, element_name) != 'cached' + + # Attempt bst build with --use-remotes set as none, this should lead to + # a complete rebuild without pulling from either artifact remote cache + result = cli.run(project=project, args=['--use-remotes', 'none', 'build', 'target.bst']) + result.assert_success() + for element_name in all_elements: + assert element_name not in result.get_pulled_elements() + + # Delete local cache again + artifacts = os.path.join(cli.directory, 'artifacts') + shutil.rmtree(artifacts) + + # Attempt bst build with --use-remotes set as user, as the shareuser is + # passed in as user config and not via a project, assert project remote + # was not attempted by it not being in the output + result = cli.run(project=project, args=['--use-remotes', 'user', 'build', 'target.bst']) + result.assert_success() + for element_name in all_elements: + assert element_name in result.get_pulled_elements() + assert shareproject.repo not in result.stderr + + # Delete local cache again + artifacts = os.path.join(cli.directory, 'artifacts') + shutil.rmtree(artifacts) + + # Attempt bst build with --use-remotes set as all, this time + # assert that project remote is attempted and in the output + result = cli.run(project=project, args=['--use-remotes', 'all', 'build', 'target.bst']) + result.assert_success() + for element_name in all_elements: + assert element_name in result.get_pulled_elements() + assert shareproject.repo in result.stderr diff --git a/tests/frontend/push.py b/tests/frontend/push.py index f3f9b52f1..b728a2f58 100644 --- a/tests/frontend/push.py +++ b/tests/frontend/push.py @@ -416,3 +416,68 @@ def test_push_already_cached(caplog, cli, tmpdir, datafiles): assert not result.get_pushed_elements(), "No elements should have been pushed since the cache was populated" assert "INFO Remote ({}) already has ".format(share.repo) in result.stderr assert "SKIPPED Push" in result.stderr + + +# Tests that: +# +# * The bst main option --use-remotes limits remote action +# as expected for push jobs +# +@pytest.mark.datafiles(DATA_DIR) +def test_useremotes_cli_options(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + + with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\ + create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject: + + # Add shareproject repo url to project.conf + with open(os.path.join(project, "project.conf"), "a") as projconf: + projconf.write("artifacts:\n url: {}\n push: True".format(shareproject.repo)) + + # Configure shareuser remote in user conf + cli.configure({ + 'artifacts': {'url': shareuser.repo, 'push': True} + }) + + # First build the target element with --use-remotes set as none. + # This should lead to a complete build without pushing to either artifact + # remote cache + result = cli.run(project=project, args=['--use-remotes', 'none', 'build', 'target.bst']) + result.assert_success() + assert not result.get_pushed_elements() + assert cli.get_element_state(project, 'target.bst') == 'cached' + + # Delete the artifacts from the local artifact cache + all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst'] + for element_name in all_elements: + cli.remove_artifact_from_cache(project, element_name) + + # Assert that nothing is cached locally anymore + for element_name in all_elements: + assert cli.get_element_state(project, element_name) != 'cached' + + # Attempt bst build with --use-remotes set as user, this should lead to + # a complete rebuild, with artifacts pushed to the shareuser remote artifact cache + # only. Assert project remote was not attempted by it not being in the output + result = cli.run(project=project, args=['--use-remotes', 'user', 'build', 'target.bst']) + result.assert_success() + for element_name in all_elements: + assert element_name in result.get_pushed_elements() + for element_name in all_elements: + assert_shared(cli, shareuser, project, element_name) + assert shareproject.repo not in result.stderr + + # Delete the artifacts from the local artifact cache + all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst'] + for element_name in all_elements: + cli.remove_artifact_from_cache(project, element_name) + + # Attempt bst build with --use-remotes set as all, this should lead to + # a complete rebuild, with artifacts pushed to both the shareuser and + # shareproject remote artifacts caches + result = cli.run(project=project, args=['--use-remotes', 'all', 'build', 'target.bst']) + result.assert_success() + for element_name in all_elements: + assert element_name in result.get_pushed_elements() + for element_name in all_elements: + assert_shared(cli, shareproject, project, element_name) |