diff options
author | Jürg Billeter <j@bitron.ch> | 2019-01-24 15:10:08 +0000 |
---|---|---|
committer | Jürg Billeter <j@bitron.ch> | 2019-01-24 15:10:08 +0000 |
commit | 05587f22bd157e188e2083f11893e7c801b3f50e (patch) | |
tree | 416f3215a19c06a52a890427815c842c6eca3797 | |
parent | 24bd8994180fbf1b4bc00754d62ccc31ed6136ee (diff) | |
parent | 50c5159f1cf760a75abd15cf655ec82351b378ac (diff) | |
download | buildstream-05587f22bd157e188e2083f11893e7c801b3f50e.tar.gz |
Merge branch 'issue-638-validate-all-files' into 'master'
Add support for default targets
See merge request BuildStream/buildstream!925
32 files changed, 519 insertions, 45 deletions
@@ -30,6 +30,10 @@ buildstream 1.3.1 specific. Recommendation if you are building in Linux is to use the ones being used in freedesktop-sdk project, for example + o Running commands without elements specified will now attempt to use + the default targets defined in the project configuration. + If no default target is defined, all elements in the project will be used. + o All elements must now be suffixed with `.bst` Attempting to use an element that does not have the `.bst` extension, will result in a warning. diff --git a/buildstream/_context.py b/buildstream/_context.py index 8b9f47bd6..f14f6b746 100644 --- a/buildstream/_context.py +++ b/buildstream/_context.py @@ -32,7 +32,7 @@ from ._message import Message, MessageType from ._profile import Topics, profile_start, profile_end from ._artifactcache import ArtifactCache from ._cas import CASCache -from ._workspaces import Workspaces, WorkspaceProjectCache, WORKSPACE_PROJECT_FILE +from ._workspaces import Workspaces, WorkspaceProjectCache from .plugin import _plugin_lookup from .sandbox import SandboxRemote @@ -657,20 +657,6 @@ class Context(): self._cascache = CASCache(self.artifactdir) return self._cascache - # guess_element() - # - # Attempts to interpret which element the user intended to run commands on - # - # Returns: - # (str) The name of the element, or None if no element can be guessed - def guess_element(self): - workspace_project_dir, _ = utils._search_upward_for_files(self._directory, [WORKSPACE_PROJECT_FILE]) - if workspace_project_dir: - workspace_project = self._workspace_project_cache.get(workspace_project_dir) - return workspace_project.get_default_element() - else: - return None - # _node_get_option_str() # diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py index fd088b63c..ab190aae4 100644 --- a/buildstream/_frontend/cli.py +++ b/buildstream/_frontend/cli.py @@ -342,7 +342,15 @@ def init(app, project_name, format_version, element_path, force): type=click.Path(readable=False)) @click.pass_obj def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions): - """Build elements in a pipeline""" + """Build elements in a pipeline + + Specifying no elements will result in building the default targets + of the project. If no default targets are configured, all project + elements will be built. + + When this command is executed from a workspace directory, the default + is to build the workspace element. + """ if (track_except or track_cross_junctions) and not (track_ or track_all): click.echo("ERROR: The --track-except and --track-cross-junctions options " @@ -353,10 +361,12 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac click.echo("WARNING: --track-save is deprecated, saving is now unconditional", err=True) with app.initialized(session_name="Build"): - if not all_ and not elements: - guessed_target = app.context.guess_element() - if guessed_target: - elements = (guessed_target,) + ignore_junction_targets = False + + if not elements: + elements = app.project.get_default_targets() + # Junction elements cannot be built, exclude them from default targets + ignore_junction_targets = True if track_all: track_ = elements @@ -365,6 +375,7 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac track_targets=track_, track_except=track_except, track_cross_junctions=track_cross_junctions, + ignore_junction_targets=ignore_junction_targets, build_all=all_) @@ -390,6 +401,13 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac def show(app, elements, deps, except_, order, format_): """Show elements in the pipeline + Specifying no elements will result in showing the default targets + of the project. If no default targets are configured, all project + elements will be shown. + + When this command is executed from a workspace directory, the default + is to show the workspace element. + By default this will show all of the dependencies of the specified target element. @@ -436,9 +454,7 @@ def show(app, elements, deps, except_, order, format_): """ with app.initialized(): if not elements: - guessed_target = app.context.guess_element() - if guessed_target: - elements = (guessed_target,) + elements = app.project.get_default_targets() dependencies = app.stream.load_selection(elements, selection=deps, @@ -478,6 +494,9 @@ def show(app, elements, deps, except_, order, format_): def shell(app, element, sysroot, mount, isolate, build_, cli_buildtree, command): """Run a command in the target element's sandbox environment + When this command is executed from a workspace directory, the default + is to shell into the workspace element. + This will stage a temporary sysroot for running the target element, assuming it has already been built and all required artifacts are in the local cache. @@ -511,7 +530,7 @@ def shell(app, element, sysroot, mount, isolate, build_, cli_buildtree, command) with app.initialized(): if not element: - element = app.context.guess_element() + element = app.project.get_default_target() if not element: raise AppError('Missing argument "ELEMENT".') @@ -581,6 +600,13 @@ def source(): def source_fetch(app, elements, deps, track_, except_, track_cross_junctions): """Fetch sources required to build the pipeline + Specifying no elements will result in fetching the default targets + of the project. If no default targets are configured, all project + elements will be fetched. + + When this command is executed from a workspace directory, the default + is to fetch the workspace element. + By default this will only try to fetch sources which are required for the build plan of the specified target element, omitting sources for any elements which are already built @@ -606,9 +632,7 @@ def source_fetch(app, elements, deps, track_, except_, track_cross_junctions): with app.initialized(session_name="Fetch"): if not elements: - guessed_target = app.context.guess_element() - if guessed_target: - elements = (guessed_target,) + elements = app.project.get_default_targets() app.stream.fetch(elements, selection=deps, @@ -636,6 +660,15 @@ def source_track(app, elements, deps, except_, cross_junctions): """Consults the specified tracking branches for new versions available to build and updates the project with any newly available references. + Specifying no elements will result in tracking the default targets + of the project. If no default targets are configured, all project + elements will be tracked. + + When this command is executed from a workspace directory, the default + is to track the workspace element. + + If no default is declared, all elements in the project will be tracked + By default this will track just the specified element, but you can also update a whole tree of dependencies in one go. @@ -647,9 +680,7 @@ def source_track(app, elements, deps, except_, cross_junctions): """ with app.initialized(session_name="Track"): if not elements: - guessed_target = app.context.guess_element() - if guessed_target: - elements = (guessed_target,) + elements = app.project.get_default_targets() # Substitute 'none' for 'redirect' so that element redirections # will be done @@ -685,6 +716,9 @@ def source_track(app, elements, deps, except_, cross_junctions): def source_checkout(app, element, location, force, deps, fetch_, except_, tar, build_scripts): """Checkout sources of an element to the specified location + + When this command is executed from a workspace directory, the default + is to checkout the sources of the workspace element. """ if not element and not location: click.echo("ERROR: LOCATION is not specified", err=True) @@ -697,7 +731,7 @@ def source_checkout(app, element, location, force, deps, fetch_, except_, with app.initialized(): if not element: - element = app.context.guess_element() + element = app.project.get_default_target() if not element: raise AppError('Missing argument "ELEMENT".') @@ -763,7 +797,7 @@ def workspace_close(app, remove_dir, all_, elements): if not (all_ or elements): # NOTE: I may need to revisit this when implementing multiple projects # opening one workspace. - element = app.context.guess_element() + element = app.project.get_default_target() if element: elements = (element,) else: @@ -824,7 +858,7 @@ def workspace_reset(app, soft, track_, all_, elements): with app.initialized(): if not (all_ or elements): - element = app.context.guess_element() + element = app.project.get_default_target() if element: elements = (element,) else: @@ -921,7 +955,11 @@ def artifact(): type=click.Path(readable=False)) @click.pass_obj def artifact_checkout(app, force, deps, integrate, hardlinks, tar, directory, element): - """Checkout contents of an artifact""" + """Checkout contents of an artifact + + When this command is executed from a workspace directory, the default + is to checkout the artifact of the workspace element. + """ from ..element import Scope if hardlinks and tar is not None: @@ -952,7 +990,7 @@ def artifact_checkout(app, force, deps, integrate, hardlinks, tar, directory, el with app.initialized(): if not element: - element = app.context.guess_element() + element = app.project.get_default_target() if not element: raise AppError('Missing argument "ELEMENT".') @@ -980,6 +1018,13 @@ def artifact_checkout(app, force, deps, integrate, hardlinks, tar, directory, el def artifact_pull(app, elements, deps, remote): """Pull a built artifact from the configured remote artifact cache. + Specifying no elements will result in pulling the default targets + of the project. If no default targets are configured, all project + elements will be pulled. + + When this command is executed from a workspace directory, the default + is to pull the workspace element. + By default the artifact will be pulled one of the configured caches if possible, following the usual priority order. If the `--remote` flag is given, only the specified cache will be queried. @@ -992,12 +1037,15 @@ def artifact_pull(app, elements, deps, remote): """ with app.initialized(session_name="Pull"): + ignore_junction_targets = False + if not elements: - guessed_target = app.context.guess_element() - if guessed_target: - elements = (guessed_target,) + elements = app.project.get_default_targets() + # Junction elements cannot be pulled, exclude them from default targets + ignore_junction_targets = True - app.stream.pull(elements, selection=deps, remote=remote) + app.stream.pull(elements, selection=deps, remote=remote, + ignore_junction_targets=ignore_junction_targets) ################################################################## @@ -1015,6 +1063,13 @@ def artifact_pull(app, elements, deps, remote): def artifact_push(app, elements, deps, remote): """Push a built artifact to a remote artifact cache. + Specifying no elements will result in pushing the default targets + of the project. If no default targets are configured, all project + elements will be pushed. + + When this command is executed from a workspace directory, the default + is to push the workspace element. + The default destination is the highest priority configured cache. You can override this by passing a different cache URL with the `--remote` flag. @@ -1029,12 +1084,15 @@ def artifact_push(app, elements, deps, remote): all: All dependencies """ with app.initialized(session_name="Push"): + ignore_junction_targets = False + if not elements: - guessed_target = app.context.guess_element() - if guessed_target: - elements = (guessed_target,) + elements = app.project.get_default_targets() + # Junction elements cannot be pushed, exclude them from default targets + ignore_junction_targets = True - app.stream.push(elements, selection=deps, remote=remote) + app.stream.push(elements, selection=deps, remote=remote, + ignore_junction_targets=ignore_junction_targets) ################################################################ diff --git a/buildstream/_project.py b/buildstream/_project.py index 1492fde77..3ec141d58 100644 --- a/buildstream/_project.py +++ b/buildstream/_project.py @@ -104,6 +104,9 @@ class Project(): # Absolute path to where elements are loaded from within the project self.element_path = None + # Default target elements + self._default_targets = None + # ProjectRefs for the main refs and also for junctions self.refs = ProjectRefs(self.directory, 'project.refs') self.junction_refs = ProjectRefs(self.directory, 'junction.refs') @@ -228,7 +231,7 @@ class Project(): 'element-path', 'variables', 'environment', 'environment-nocache', 'split-rules', 'elements', 'plugins', - 'aliases', 'name', + 'aliases', 'name', 'defaults', 'artifacts', 'options', 'fail-on-overlap', 'shell', 'fatal-warnings', 'ref-storage', 'sandbox', 'mirrors', 'remote-execution', @@ -391,6 +394,44 @@ class Project(): # Reset the element loader state Element._reset_load_state() + # get_default_target() + # + # Attempts to interpret which element the user intended to run a command on. + # This is for commands that only accept a single target element and thus, + # this only uses the workspace element (if invoked from workspace directory) + # and does not use the project default targets. + # + def get_default_target(self): + return self._invoked_from_workspace_element + + # get_default_targets() + # + # Attempts to interpret which elements the user intended to run a command on. + # This is for commands that accept multiple target elements. + # + def get_default_targets(self): + + # If _invoked_from_workspace_element has a value, + # a workspace element was found before a project config + # Therefore the workspace does not contain a project + if self._invoked_from_workspace_element: + return (self._invoked_from_workspace_element,) + + # Default targets from project configuration + if self._default_targets: + return tuple(self._default_targets) + + # If default targets are not configured, default to all project elements + default_targets = [] + for root, _, files in os.walk(self.element_path): + for file in files: + if file.endswith(".bst"): + rel_dir = os.path.relpath(root, self.element_path) + rel_file = os.path.join(rel_dir, file).lstrip("./") + default_targets.append(rel_file) + + return tuple(default_targets) + # _load(): # # Loads the project configuration file in the project @@ -456,6 +497,10 @@ class Project(): self.config.options = OptionPool(self.element_path) self.first_pass_config.options = OptionPool(self.element_path) + defaults = _yaml.node_get(pre_config_node, Mapping, 'defaults') + _yaml.node_validate(defaults, ['targets']) + self._default_targets = _yaml.node_get(defaults, list, "targets") + # Fatal warnings self._fatal_warnings = _yaml.node_get(pre_config_node, list, 'fatal-warnings', default_value=[]) diff --git a/buildstream/_stream.py b/buildstream/_stream.py index 62eff1d8b..36b496e77 100644 --- a/buildstream/_stream.py +++ b/buildstream/_stream.py @@ -161,6 +161,7 @@ class Stream(): # track_targets (list of str): Specified targets for tracking # track_except (list of str): Specified targets to except from tracking # track_cross_junctions (bool): Whether tracking should cross junction boundaries + # ignore_junction_targets (bool): Whether junction targets should be filtered out # build_all (bool): Whether to build all elements, or only those # which are required to build the target. # @@ -168,6 +169,7 @@ class Stream(): track_targets=None, track_except=None, track_cross_junctions=False, + ignore_junction_targets=False, build_all=False): if build_all: @@ -180,6 +182,7 @@ class Stream(): selection=selection, track_selection=PipelineSelection.ALL, track_except_targets=track_except, track_cross_junctions=track_cross_junctions, + ignore_junction_targets=ignore_junction_targets, use_artifact_config=True, fetch_subprojects=True, dynamic_plan=True) @@ -291,6 +294,7 @@ class Stream(): # Args: # targets (list of str): Targets to pull # selection (PipelineSelection): The selection mode for the specified targets + # ignore_junction_targets (bool): Whether junction targets should be filtered out # remote (str): The URL of a specific remote server to pull from, or None # # If `remote` specified as None, then regular configuration will be used @@ -298,6 +302,7 @@ class Stream(): # def pull(self, targets, *, selection=PipelineSelection.NONE, + ignore_junction_targets=False, remote=None): use_config = True @@ -306,6 +311,7 @@ class Stream(): elements, _ = self._load(targets, (), selection=selection, + ignore_junction_targets=ignore_junction_targets, use_artifact_config=use_config, artifact_remote_url=remote, fetch_subprojects=True) @@ -325,6 +331,7 @@ class Stream(): # Args: # targets (list of str): Targets to push # selection (PipelineSelection): The selection mode for the specified targets + # ignore_junction_targets (bool): Whether junction targets should be filtered out # remote (str): The URL of a specific remote server to push to, or None # # If `remote` specified as None, then regular configuration will be used @@ -336,6 +343,7 @@ class Stream(): # def push(self, targets, *, selection=PipelineSelection.NONE, + ignore_junction_targets=False, remote=None): use_config = True @@ -344,6 +352,7 @@ class Stream(): elements, _ = self._load(targets, (), selection=selection, + ignore_junction_targets=ignore_junction_targets, use_artifact_config=use_config, artifact_remote_url=remote, fetch_subprojects=True) @@ -851,6 +860,7 @@ class Stream(): # except_targets (list of str): Specified targets to except from fetching # track_except_targets (list of str): Specified targets to except from fetching # track_cross_junctions (bool): Whether tracking should cross junction boundaries + # ignore_junction_targets (bool): Whether junction targets should be filtered out # use_artifact_config (bool): Whether to initialize artifacts with the config # artifact_remote_url (bool): A remote url for initializing the artifacts # fetch_subprojects (bool): Whether to fetch subprojects while loading @@ -865,6 +875,7 @@ class Stream(): except_targets=(), track_except_targets=(), track_cross_junctions=False, + ignore_junction_targets=False, use_artifact_config=False, artifact_remote_url=None, fetch_subprojects=False, @@ -881,6 +892,10 @@ class Stream(): rewritable=rewritable, fetch_subprojects=fetch_subprojects) + # Optionally filter out junction elements + if ignore_junction_targets: + elements = [e for e in elements if e.get_kind() != 'junction'] + # Hold on to the targets self.targets = elements diff --git a/buildstream/_versions.py b/buildstream/_versions.py index 8472a5b33..2563fa504 100644 --- a/buildstream/_versions.py +++ b/buildstream/_versions.py @@ -23,7 +23,7 @@ # This version is bumped whenever enhancements are made # to the `project.conf` format or the core element format. # -BST_FORMAT_VERSION = 20 +BST_FORMAT_VERSION = 21 # The base BuildStream artifact version diff --git a/buildstream/data/projectconfig.yaml b/buildstream/data/projectconfig.yaml index e9bdaa160..2a6b09a15 100644 --- a/buildstream/data/projectconfig.yaml +++ b/buildstream/data/projectconfig.yaml @@ -167,3 +167,11 @@ shell: # Command to run when `bst shell` does not provide a command # command: [ 'sh', '-i' ] + +# Defaults for bst commands +# +defaults: + + # Set default target elements to use when none are passed on the command line. + # If none are configured in the project, default to all project elements. + targets: [] diff --git a/doc/source/format_project.rst b/doc/source/format_project.rst index c3555e0c1..f4dea333a 100644 --- a/doc/source/format_project.rst +++ b/doc/source/format_project.rst @@ -945,6 +945,44 @@ Host side environment variable expansion is also supported: - '${XDG_RUNTIME_DIR}/pulse/native' +.. _project_default_targets: + +Default targets +--------------- +When running BuildStream commands from a project directory or subdirectory +without specifying any target elements on the command line, the default targets +of the project will be used. The default targets can be configured in the +``defaults`` section as follows: + +.. code:: yaml + + defaults: + + # List of default target elements + targets: + - app.bst + +If no default targets are configured in ``project.conf``, BuildStream commands +will default to all ``.bst`` files in the configured element path. + +Commands that cannot support junctions as target elements (``bst build``, +``bst artifact push``, and ``bst artifact pull``) ignore junctions in the list +of default targets. + +When running BuildStream commands from a workspace directory (that is not a +BuildStream project directory), project default targets are not used and the +workspace element will be used as the default target instead. + +``bst artifact checkout``, ``bst source checkout``, and ``bst shell`` are +currently limited to a single target element and due to this, they currently +do not use project default targets. However, they still use the workspace +element as default target when run from a workspace directory. + +.. note:: + + The ``targets`` configuration is available since :ref:`format version 21 <project_format_version>` + + .. _project_builtin_defaults: Builtin defaults diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py index 8c7e22a85..b35b14820 100644 --- a/tests/frontend/buildcheckout.py +++ b/tests/frontend/buildcheckout.py @@ -2,6 +2,7 @@ import os import tarfile import hashlib import pytest +import subprocess from tests.testutils import cli, create_repo, ALL_REPO_KINDS, generate_junction from tests.testutils.site import IS_WINDOWS @@ -61,6 +62,35 @@ def test_build_checkout(datafiles, cli, strict, hardlinks): assert os.path.exists(filename) +@pytest.mark.datafiles(DATA_DIR + "_world") +def test_build_default_all(datafiles, cli): + project = os.path.join(datafiles.dirname, datafiles.basename) + result = cli.run(project=project, silent=True, args=['build']) + + result.assert_success() + target_dir = os.path.join(cli.directory, DATA_DIR + "_world", "elements") + output_dir = os.path.join(cli.directory, "logs", "test") + + expected = subprocess.Popen(('ls', target_dir), stdout=subprocess.PIPE) + expected = subprocess.check_output(("wc", "-w"), stdin=expected.stdout) + + results = subprocess.Popen(('ls', output_dir), stdout=subprocess.PIPE) + results = subprocess.check_output(("wc", "-w"), stdin=results.stdout) + + assert results == expected + + +@pytest.mark.datafiles(DATA_DIR + "_default") +def test_build_default(cli, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + result = cli.run(project=project, silent=True, args=['build']) + + result.assert_success() + results = cli.get_element_state(project, "target2.bst") + expected = "cached" + assert results == expected + + @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("strict,hardlinks", [ ("non-strict", "hardlinks"), @@ -550,6 +580,53 @@ def test_build_checkout_junction(cli, tmpdir, datafiles): assert contents == 'animal=Pony\n' +# Test that default targets work with projects with junctions +@pytest.mark.datafiles(DATA_DIR + "_world") +def test_build_checkout_junction_default_targets(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + subproject_path = os.path.join(project, 'files', 'sub-project') + junction_path = os.path.join(project, 'elements', 'junction.bst') + element_path = os.path.join(project, 'elements', 'junction-dep.bst') + checkout = os.path.join(cli.directory, 'checkout') + + # Create a repo to hold the subproject and generate a junction element for it + ref = generate_junction(tmpdir, subproject_path, junction_path) + + # Create a stack element to depend on a cross junction element + # + element = { + 'kind': 'stack', + 'depends': [ + { + 'junction': 'junction.bst', + 'filename': 'import-etc.bst' + } + ] + } + _yaml.dump(element, element_path) + + # Now try to build it, this should automatically result in fetching + # the junction itself at load time. + result = cli.run(project=project, args=['build']) + result.assert_success() + + # Assert that it's cached now + assert cli.get_element_state(project, 'junction-dep.bst') == 'cached' + + # Now check it out + result = cli.run(project=project, args=[ + 'artifact', 'checkout', 'junction-dep.bst', '--directory', checkout + ]) + result.assert_success() + + # Assert the content of /etc/animal.conf + filename = os.path.join(checkout, 'etc', 'animal.conf') + assert os.path.exists(filename) + with open(filename, 'r') as f: + contents = f.read() + assert contents == 'animal=Pony\n' + + @pytest.mark.datafiles(DATA_DIR) def test_build_checkout_workspaced_junction(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) diff --git a/tests/frontend/fetch.py b/tests/frontend/fetch.py index 96c4a9335..24d9a36a6 100644 --- a/tests/frontend/fetch.py +++ b/tests/frontend/fetch.py @@ -49,6 +49,41 @@ def test_fetch(cli, tmpdir, datafiles, kind): assert cli.get_element_state(project, element_name) == 'buildable' +@pytest.mark.datafiles(os.path.join(TOP_DIR, 'project_world')) +def test_fetch_default_targets(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + element_path = os.path.join(project, 'elements') + element_name = 'fetch-test.bst' + + # Create our repo object of the given source type with + # the bin files, and then collect the initial ref. + # + repo = create_repo('git', str(tmpdir)) + ref = repo.create(project) + + # Write out our test target + element = { + 'kind': 'import', + 'sources': [ + repo.source_config(ref=ref) + ] + } + _yaml.dump(element, + os.path.join(element_path, + element_name)) + + # Assert that a fetch is needed + assert cli.get_element_state(project, element_name) == 'fetch needed' + + # Now try to fetch it, using the default target feature + result = cli.run(project=project, args=['source', 'fetch']) + result.assert_success() + + # Assert that we are now buildable because the source is + # now cached. + assert cli.get_element_state(project, element_name) == 'buildable' + + @pytest.mark.datafiles(os.path.join(TOP_DIR, 'consistencyerror')) def test_fetch_consistency_error(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) diff --git a/tests/frontend/project_default/elements/target.bst b/tests/frontend/project_default/elements/target.bst new file mode 100644 index 000000000..d644c89ba --- /dev/null +++ b/tests/frontend/project_default/elements/target.bst @@ -0,0 +1,4 @@ +kind: stack +description: | + + Main stack target for the bst build test diff --git a/tests/frontend/project_default/elements/target2.bst b/tests/frontend/project_default/elements/target2.bst new file mode 100644 index 000000000..d644c89ba --- /dev/null +++ b/tests/frontend/project_default/elements/target2.bst @@ -0,0 +1,4 @@ +kind: stack +description: | + + Main stack target for the bst build test diff --git a/tests/frontend/project_default/project.conf b/tests/frontend/project_default/project.conf new file mode 100644 index 000000000..5987c82f1 --- /dev/null +++ b/tests/frontend/project_default/project.conf @@ -0,0 +1,11 @@ +# Project config for frontend build test +name: test + +element-path: elements + +fatal-warnings: +- bad-element-suffix + +defaults: + targets: + - target2.bst diff --git a/tests/frontend/project_fail/elements/compose-all.bst b/tests/frontend/project_fail/elements/compose-all.bst new file mode 100644 index 000000000..9b981dd73 --- /dev/null +++ b/tests/frontend/project_fail/elements/compose-all.bst @@ -0,0 +1,10 @@ +kind: compose + +depends: +- fileNAME: import-dev.bst + type: build + +config: + # Dont try running the sandbox, we dont have a + # runtime to run anything in this context. + integrate: False diff --git a/tests/frontend/project_fail/elements/import-dev.bst b/tests/frontend/project_fail/elements/import-dev.bst new file mode 100644 index 000000000..152a54667 --- /dev/null +++ b/tests/frontend/project_fail/elements/import-dev.bst @@ -0,0 +1,4 @@ +kind: import +sources: +- kind: local + path: files/dev-files diff --git a/tests/frontend/project_fail/elements/target.bst b/tests/frontend/project_fail/elements/target.bst new file mode 100644 index 000000000..154c477e6 --- /dev/null +++ b/tests/frontend/project_fail/elements/target.bst @@ -0,0 +1,7 @@ +kind: stack +description: | + + Main stack target for the bst build test + +depends: +- compose-all.bst diff --git a/tests/frontend/project_fail/files/dev-files/usr/include/pony.h b/tests/frontend/project_fail/files/dev-files/usr/include/pony.h new file mode 100644 index 000000000..40bd0c2e7 --- /dev/null +++ b/tests/frontend/project_fail/files/dev-files/usr/include/pony.h @@ -0,0 +1,12 @@ +#ifndef __PONY_H__ +#define __PONY_H__ + +#define PONY_BEGIN "Once upon a time, there was a pony." +#define PONY_END "And they lived happily ever after, the end." + +#define MAKE_PONY(story) \ + PONY_BEGIN \ + story \ + PONY_END + +#endif /* __PONY_H__ */ diff --git a/tests/frontend/project_fail/project.conf b/tests/frontend/project_fail/project.conf new file mode 100644 index 000000000..854e38693 --- /dev/null +++ b/tests/frontend/project_fail/project.conf @@ -0,0 +1,4 @@ +# Project config for frontend build test +name: test + +element-path: elements diff --git a/tests/frontend/project_world/elements/checkout-deps.bst b/tests/frontend/project_world/elements/checkout-deps.bst new file mode 100644 index 000000000..e3a548690 --- /dev/null +++ b/tests/frontend/project_world/elements/checkout-deps.bst @@ -0,0 +1,7 @@ +kind: stack +description: It is important for this element to have both build and runtime dependencies +depends: +- filename: import-dev.bst + type: build +- filename: import-bin.bst + type: runtime diff --git a/tests/frontend/project_world/elements/compose-all.bst b/tests/frontend/project_world/elements/compose-all.bst new file mode 100644 index 000000000..ba47081b3 --- /dev/null +++ b/tests/frontend/project_world/elements/compose-all.bst @@ -0,0 +1,12 @@ +kind: compose + +depends: +- filename: import-bin.bst + type: build +- filename: import-dev.bst + type: build + +config: + # Dont try running the sandbox, we dont have a + # runtime to run anything in this context. + integrate: False diff --git a/tests/frontend/project_world/elements/compose-exclude-dev.bst b/tests/frontend/project_world/elements/compose-exclude-dev.bst new file mode 100644 index 000000000..75c14378c --- /dev/null +++ b/tests/frontend/project_world/elements/compose-exclude-dev.bst @@ -0,0 +1,16 @@ +kind: compose + +depends: +- filename: import-bin.bst + type: build +- filename: import-dev.bst + type: build + +config: + # Dont try running the sandbox, we dont have a + # runtime to run anything in this context. + integrate: False + + # Exclude the dev domain + exclude: + - devel diff --git a/tests/frontend/project_world/elements/compose-include-bin.bst b/tests/frontend/project_world/elements/compose-include-bin.bst new file mode 100644 index 000000000..9571203c6 --- /dev/null +++ b/tests/frontend/project_world/elements/compose-include-bin.bst @@ -0,0 +1,16 @@ +kind: compose + +depends: +- filename: import-bin.bst + type: build +- filename: import-dev.bst + type: build + +config: + # Dont try running the sandbox, we dont have a + # runtime to run anything in this context. + integrate: False + + # Only include the runtim + include: + - runtime diff --git a/tests/frontend/project_world/elements/import-bin.bst b/tests/frontend/project_world/elements/import-bin.bst new file mode 100644 index 000000000..066a03328 --- /dev/null +++ b/tests/frontend/project_world/elements/import-bin.bst @@ -0,0 +1 @@ +kind: stack diff --git a/tests/frontend/project_world/elements/import-dev.bst b/tests/frontend/project_world/elements/import-dev.bst new file mode 100644 index 000000000..066a03328 --- /dev/null +++ b/tests/frontend/project_world/elements/import-dev.bst @@ -0,0 +1 @@ +kind: stack diff --git a/tests/frontend/project_world/elements/rebuild-target.bst b/tests/frontend/project_world/elements/rebuild-target.bst new file mode 100644 index 000000000..49a02c217 --- /dev/null +++ b/tests/frontend/project_world/elements/rebuild-target.bst @@ -0,0 +1,4 @@ +kind: compose + +build-depends: +- target.bst diff --git a/tests/frontend/project_world/elements/target.bst b/tests/frontend/project_world/elements/target.bst new file mode 100644 index 000000000..b9432fafa --- /dev/null +++ b/tests/frontend/project_world/elements/target.bst @@ -0,0 +1,8 @@ +kind: stack +description: | + + Main stack target for the bst build test + +depends: +- import-bin.bst +- compose-all.bst diff --git a/tests/frontend/project_world/files/sub-project/elements/import-etc.bst b/tests/frontend/project_world/files/sub-project/elements/import-etc.bst new file mode 100644 index 000000000..f0171990e --- /dev/null +++ b/tests/frontend/project_world/files/sub-project/elements/import-etc.bst @@ -0,0 +1,4 @@ +kind: import +sources: +- kind: local + path: files/etc-files diff --git a/tests/frontend/project_world/files/sub-project/files/etc-files/etc/animal.conf b/tests/frontend/project_world/files/sub-project/files/etc-files/etc/animal.conf new file mode 100644 index 000000000..db8c36cba --- /dev/null +++ b/tests/frontend/project_world/files/sub-project/files/etc-files/etc/animal.conf @@ -0,0 +1 @@ +animal=Pony diff --git a/tests/frontend/project_world/files/sub-project/project.conf b/tests/frontend/project_world/files/sub-project/project.conf new file mode 100644 index 000000000..bbb8414a3 --- /dev/null +++ b/tests/frontend/project_world/files/sub-project/project.conf @@ -0,0 +1,4 @@ +# Project config for frontend build test +name: subtest + +element-path: elements diff --git a/tests/frontend/project_world/project.conf b/tests/frontend/project_world/project.conf new file mode 100644 index 000000000..a7e4a023c --- /dev/null +++ b/tests/frontend/project_world/project.conf @@ -0,0 +1,7 @@ +# Project config for frontend build test +name: test + +element-path: elements + +fatal-warnings: +- bad-element-suffix diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py index f0d6fbf4e..20b740948 100644 --- a/tests/frontend/pull.py +++ b/tests/frontend/pull.py @@ -80,6 +80,56 @@ def test_push_pull_all(cli, tmpdir, datafiles): # Tests that: # +# * `bst push` (default targets) pushes all built elements to configured 'push' cache +# * `bst pull` (default targets) downloads everything from cache after local deletion +# +@pytest.mark.datafiles(DATA_DIR + '_world') +def test_push_pull_default_targets(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + + with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + + # First build the target elements + cli.configure({ + 'artifacts': {'url': share.repo} + }) + result = cli.run(project=project, args=['build']) + result.assert_success() + assert cli.get_element_state(project, 'target.bst') == 'cached' + + # Push all elements + cli.configure({ + 'artifacts': {'url': share.repo, 'push': True} + }) + result = cli.run(project=project, args=['artifact', 'push']) + result.assert_success() + + # Assert that everything is now cached in the remote. + all_elements = ['target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst'] + for element_name in all_elements: + assert_shared(cli, share, project, element_name) + + # Now we've pushed, delete the user's local artifact cache + # directory and try to redownload it from the share + # + artifacts = os.path.join(cli.directory, 'artifacts') + shutil.rmtree(artifacts) + + # Assert that nothing is cached locally anymore + states = cli.get_element_states(project, all_elements) + assert not any(states[e] == 'cached' for e in all_elements) + + # Now try bst pull + result = cli.run(project=project, args=['artifact', 'pull']) + result.assert_success() + + # And assert that it's again in the local cache, without having built + states = cli.get_element_states(project, all_elements) + assert not any(states[e] != 'cached' for e in all_elements) + + +# Tests that: +# # * `bst build` pushes all build elements ONLY to configured 'push' cache # * `bst pull` finds artifacts that are available only in the secondary cache # diff --git a/tests/frontend/show.py b/tests/frontend/show.py index 9c32dd664..88f38dd6a 100644 --- a/tests/frontend/show.py +++ b/tests/frontend/show.py @@ -45,6 +45,27 @@ def test_show_invalid_element_path(cli, datafiles): 'show', "foo.bst"]) + +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project_default')) +def test_show_default(cli, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + result = cli.run(project=project, silent=True, args=[ + 'show']) + + result.assert_success() + + # Get the result output of "[state sha element]" and turn into a list + results = result.output.strip().split(" ") + expected = 'target2.bst' + assert results[2] == expected + + +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project_fail')) +def test_show_fail(cli, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + result = cli.run(project=project, silent=True, args=[ + 'show']) + result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) |