summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorValentin David <valentin.david@codethink.co.uk>2018-05-14 18:48:32 +0200
committerTristan Van Berkom <tristan.van.berkom@gmail.com>2018-06-08 21:07:22 +0000
commitacde3ba8fa09605775a6627398bc2af26658d69a (patch)
tree21c76c943f28f13587d9f0dd43c43b85e06e1d4a
parent130bfbb84e0de2c2289b9dd708b3f79682d140f4 (diff)
downloadbuildstream-acde3ba8fa09605775a6627398bc2af26658d69a.tar.gz
Allow tracking dependencies within sub-projects.
--track-cross-junctions now concerns crossing junctions rather than forbidding elements in sub-project to be tracked. Part of #359.
-rw-r--r--buildstream/_pipeline.py16
-rw-r--r--buildstream/_stream.py24
-rw-r--r--tests/frontend/track.py43
-rw-r--r--tests/frontend/track_cross_junction.py156
4 files changed, 229 insertions, 10 deletions
diff --git a/buildstream/_pipeline.py b/buildstream/_pipeline.py
index ba27ca6b6..a280c22ee 100644
--- a/buildstream/_pipeline.py
+++ b/buildstream/_pipeline.py
@@ -346,6 +346,8 @@ class Pipeline():
# lists targetted at tracking.
#
# Args:
+ # project (Project): Project used for cross_junction filtering.
+ # All elements are expected to belong to that project.
# elements (list of Element): The list of elements to filter
# cross_junction_requested (bool): Whether the user requested
# cross junction tracking
@@ -353,12 +355,11 @@ class Pipeline():
# Returns:
# (list of Element): The filtered or asserted result
#
- def track_cross_junction_filter(self, elements, cross_junction_requested):
+ def track_cross_junction_filter(self, project, elements, cross_junction_requested):
# Filter out cross junctioned elements
- if cross_junction_requested:
- self._assert_junction_tracking(elements)
- else:
- elements = self._filter_cross_junctions(elements)
+ if not cross_junction_requested:
+ elements = self._filter_cross_junctions(project, elements)
+ self._assert_junction_tracking(elements)
return elements
@@ -403,16 +404,17 @@ class Pipeline():
# Filters out cross junction elements from the elements
#
# Args:
+ # project (Project): The project on which elements are allowed
# elements (list of Element): The list of elements to be tracked
#
# Returns:
# (list): A filtered list of `elements` which does
# not contain any cross junction elements.
#
- def _filter_cross_junctions(self, elements):
+ def _filter_cross_junctions(self, project, elements):
return [
element for element in elements
- if element._get_project() is self._project
+ if element._get_project() is project
]
# _assert_junction_tracking()
diff --git a/buildstream/_stream.py b/buildstream/_stream.py
index c2fce58c0..0680f2a1f 100644
--- a/buildstream/_stream.py
+++ b/buildstream/_stream.py
@@ -831,12 +831,30 @@ class Stream():
# done before resolving element states.
#
assert track_selection != PipelineSelection.PLAN
- track_selected = self._pipeline.get_selection(track_elements, track_selection)
+
+ # Tracked elements are split by owner projects in order to
+ # filter cross junctions tracking dependencies on their
+ # respective project.
+ track_projects = {}
+ for element in track_elements:
+ project = element._get_project()
+ if project not in track_projects:
+ track_projects[project] = [element]
+ else:
+ track_projects[project].append(element)
+
+ track_selected = []
+
+ for project, project_elements in track_projects.items():
+ selected = self._pipeline.get_selection(project_elements, track_selection)
+ selected = self._pipeline.track_cross_junction_filter(project,
+ selected,
+ track_cross_junctions)
+ track_selected.extend(selected)
+
track_selected = self._pipeline.except_elements(track_elements,
track_selected,
track_except_elements)
- track_selected = self._pipeline.track_cross_junction_filter(track_selected,
- track_cross_junctions)
for element in track_selected:
element._schedule_tracking()
diff --git a/tests/frontend/track.py b/tests/frontend/track.py
index 2defc2349..51768d650 100644
--- a/tests/frontend/track.py
+++ b/tests/frontend/track.py
@@ -437,3 +437,46 @@ def test_junction_element(cli, tmpdir, datafiles, ref_storage):
# Now assert element state (via bst show under the hood) of the dep again
assert cli.get_element_state(project, 'junction-dep.bst') == 'waiting'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
+@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+def test_cross_junction(cli, tmpdir, datafiles, ref_storage, kind):
+ project = os.path.join(datafiles.dirname, datafiles.basename)
+ subproject_path = os.path.join(project, 'files', 'sub-project')
+ junction_path = os.path.join(project, 'elements', 'junction.bst')
+ etc_files = os.path.join(subproject_path, 'files', 'etc-files')
+ repo_element_path = os.path.join(subproject_path, 'elements',
+ 'import-etc-repo.bst')
+
+ configure_project(project, {
+ 'ref-storage': ref_storage
+ })
+
+ repo = create_repo(kind, str(tmpdir.join('element_repo')))
+ ref = repo.create(etc_files)
+
+ generate_element(repo, repo_element_path)
+
+ generate_junction(str(tmpdir.join('junction_repo')),
+ subproject_path, junction_path, store_ref=False)
+
+ # Track the junction itself first.
+ result = cli.run(project=project, args=['track', 'junction.bst'])
+ result.assert_success()
+
+ assert cli.get_element_state(project, 'junction.bst:import-etc-repo.bst') == 'no reference'
+
+ # Track the cross junction element. -J is not given, it is implied.
+ result = cli.run(project=project, args=['track', 'junction.bst:import-etc-repo.bst'])
+
+ if ref_storage == 'inline':
+ # This is not allowed to track cross junction without project.refs.
+ result.assert_main_error(ErrorDomain.PIPELINE, 'untrackable-sources')
+ else:
+ result.assert_success()
+
+ assert cli.get_element_state(project, 'junction.bst:import-etc-repo.bst') == 'buildable'
+
+ assert os.path.exists(os.path.join(project, 'project.refs'))
diff --git a/tests/frontend/track_cross_junction.py b/tests/frontend/track_cross_junction.py
new file mode 100644
index 000000000..34c39ddd2
--- /dev/null
+++ b/tests/frontend/track_cross_junction.py
@@ -0,0 +1,156 @@
+import os
+import pytest
+from tests.testutils import cli, create_repo, ALL_REPO_KINDS
+from buildstream import _yaml
+
+from . import generate_junction
+
+
+def generate_element(repo, element_path, dep_name=None):
+ element = {
+ 'kind': 'import',
+ 'sources': [
+ repo.source_config()
+ ]
+ }
+ if dep_name:
+ element['depends'] = [dep_name]
+
+ _yaml.dump(element, element_path)
+
+
+def generate_import_element(tmpdir, kind, project, name):
+ element_name = 'import-{}.bst'.format(name)
+ repo_element_path = os.path.join(project, 'elements', element_name)
+ files = str(tmpdir.join("imported_files_{}".format(name)))
+ os.makedirs(files)
+
+ with open(os.path.join(files, '{}.txt'.format(name)), 'w') as f:
+ f.write(name)
+
+ subproject_path = os.path.join(str(tmpdir.join('sub-project-{}'.format(name))))
+
+ repo = create_repo(kind, str(tmpdir.join('element_{}_repo'.format(name))))
+ ref = repo.create(files)
+
+ generate_element(repo, repo_element_path)
+
+ return element_name
+
+
+def generate_project(tmpdir, name, config={}):
+ project_name = 'project-{}'.format(name)
+ subproject_path = os.path.join(str(tmpdir.join(project_name)))
+ os.makedirs(os.path.join(subproject_path, 'elements'))
+
+ project_conf = {
+ 'name': name,
+ 'element-path': 'elements'
+ }
+ project_conf.update(config)
+ _yaml.dump(project_conf, os.path.join(subproject_path, 'project.conf'))
+
+ return project_name, subproject_path
+
+
+def generate_simple_stack(project, name, dependencies):
+ element_name = '{}.bst'.format(name)
+ element_path = os.path.join(project, 'elements', element_name)
+ element = {
+ 'kind': 'stack',
+ 'depends': dependencies
+ }
+ _yaml.dump(element, element_path)
+
+ return element_name
+
+
+def generate_cross_element(project, subproject_name, import_name):
+ basename, _ = os.path.splitext(import_name)
+ return generate_simple_stack(project, 'import-{}-{}'.format(subproject_name, basename),
+ [{
+ 'junction': '{}.bst'.format(subproject_name),
+ 'filename': import_name
+ }])
+
+
+@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+def test_cross_junction_multiple_projects(cli, tmpdir, datafiles, kind):
+ tmpdir = tmpdir.join(kind)
+
+ # Generate 3 projects: main, a, b
+ _, project = generate_project(tmpdir, 'main', {'ref-storage': 'project.refs'})
+ project_a, project_a_path = generate_project(tmpdir, 'a')
+ project_b, project_b_path = generate_project(tmpdir, 'b')
+
+ # Generate an element with a trackable source for each project
+ element_a = generate_import_element(tmpdir, kind, project_a_path, 'a')
+ element_b = generate_import_element(tmpdir, kind, project_b_path, 'b')
+ element_c = generate_import_element(tmpdir, kind, project, 'c')
+
+ # Create some indirections to the elements with dependencies to test --deps
+ stack_a = generate_simple_stack(project_a_path, 'stack-a', [element_a])
+ stack_b = generate_simple_stack(project_b_path, 'stack-b', [element_b])
+
+ # Create junctions for projects a and b in main.
+ junction_a = '{}.bst'.format(project_a)
+ junction_a_path = os.path.join(project, 'elements', junction_a)
+ generate_junction(tmpdir.join('repo_a'), project_a_path, junction_a_path, store_ref=False)
+
+ junction_b = '{}.bst'.format(project_b)
+ junction_b_path = os.path.join(project, 'elements', junction_b)
+ generate_junction(tmpdir.join('repo_b'), project_b_path, junction_b_path, store_ref=False)
+
+ # Track the junctions.
+ result = cli.run(project=project, args=['track', junction_a, junction_b])
+ result.assert_success()
+
+ # Import elements from a and b in to main.
+ imported_a = generate_cross_element(project, project_a, stack_a)
+ imported_b = generate_cross_element(project, project_b, stack_b)
+
+ # Generate a top level stack depending on everything
+ all_bst = generate_simple_stack(project, 'all', [imported_a, imported_b, element_c])
+
+ # Track without following junctions. But explicitly also track the elements in project a.
+ result = cli.run(project=project, args=['track', '--deps', 'all', all_bst, '{}:{}'.format(junction_a, stack_a)])
+ result.assert_success()
+
+ # Elements in project b should not be tracked. But elements in project a and main should.
+ expected = [element_c,
+ '{}:{}'.format(junction_a, element_a)]
+ assert set(result.get_tracked_elements()) == set(expected)
+
+
+@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+def test_track_exceptions(cli, tmpdir, datafiles, kind):
+ tmpdir = tmpdir.join(kind)
+
+ _, project = generate_project(tmpdir, 'main', {'ref-storage': 'project.refs'})
+ project_a, project_a_path = generate_project(tmpdir, 'a')
+
+ element_a = generate_import_element(tmpdir, kind, project_a_path, 'a')
+ element_b = generate_import_element(tmpdir, kind, project_a_path, 'b')
+
+ all_bst = generate_simple_stack(project_a_path, 'all', [element_a,
+ element_b])
+
+ junction_a = '{}.bst'.format(project_a)
+ junction_a_path = os.path.join(project, 'elements', junction_a)
+ generate_junction(tmpdir.join('repo_a'), project_a_path, junction_a_path, store_ref=False)
+
+ result = cli.run(project=project, args=['track', junction_a])
+ result.assert_success()
+
+ imported_b = generate_cross_element(project, project_a, element_b)
+ indirection = generate_simple_stack(project, 'indirection', [imported_b])
+
+ result = cli.run(project=project,
+ args=['track', '--deps', 'all',
+ '--except', indirection,
+ '{}:{}'.format(junction_a, all_bst), imported_b])
+ result.assert_success()
+
+ expected = ['{}:{}'.format(junction_a, element_a),
+ '{}:{}'.format(junction_a, element_b)]
+ assert set(result.get_tracked_elements()) == set(expected)