diff options
Diffstat (limited to 'tests/frontend')
-rw-r--r-- | tests/frontend/buildcheckout.py | 19 | ||||
-rw-r--r-- | tests/frontend/cross_junction_workspace.py | 117 | ||||
-rw-r--r-- | tests/frontend/fetch.py | 38 | ||||
-rw-r--r-- | tests/frontend/pull.py | 98 | ||||
-rw-r--r-- | tests/frontend/push.py | 7 | ||||
-rw-r--r-- | tests/frontend/show.py | 47 | ||||
-rw-r--r-- | tests/frontend/track.py | 43 | ||||
-rw-r--r-- | tests/frontend/track_cross_junction.py | 156 |
8 files changed, 488 insertions, 37 deletions
diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py index 3eb98139f..5b46d3d52 100644 --- a/tests/frontend/buildcheckout.py +++ b/tests/frontend/buildcheckout.py @@ -390,3 +390,22 @@ def test_build_checkout_workspaced_junction(cli, tmpdir, datafiles): with open(filename, 'r') as f: contents = f.read() assert contents == 'animal=Horsy\n' + + +@pytest.mark.datafiles(DATA_DIR) +def test_build_checkout_cross_junction(datafiles, cli, tmpdir): + project = os.path.join(datafiles.dirname, datafiles.basename) + subproject_path = os.path.join(project, 'files', 'sub-project') + junction_path = os.path.join(project, 'elements', 'junction.bst') + checkout = os.path.join(cli.directory, 'checkout') + + generate_junction(tmpdir, subproject_path, junction_path) + + result = cli.run(project=project, args=['build', 'junction.bst:import-etc.bst']) + result.assert_success() + + result = cli.run(project=project, args=['checkout', 'junction.bst:import-etc.bst', checkout]) + result.assert_success() + + filename = os.path.join(checkout, 'etc', 'animal.conf') + assert os.path.exists(filename) diff --git a/tests/frontend/cross_junction_workspace.py b/tests/frontend/cross_junction_workspace.py new file mode 100644 index 000000000..eb2bc2eb8 --- /dev/null +++ b/tests/frontend/cross_junction_workspace.py @@ -0,0 +1,117 @@ +import os +from tests.testutils import cli, create_repo +from buildstream import _yaml + + +def prepare_junction_project(cli, tmpdir): + main_project = tmpdir.join("main") + sub_project = tmpdir.join("sub") + os.makedirs(str(main_project)) + os.makedirs(str(sub_project)) + + _yaml.dump({'name': 'main'}, str(main_project.join("project.conf"))) + _yaml.dump({'name': 'sub'}, str(sub_project.join("project.conf"))) + + import_dir = tmpdir.join("import") + os.makedirs(str(import_dir)) + with open(str(import_dir.join("hello.txt")), "w") as f: + f.write("hello!") + + import_repo_dir = tmpdir.join("import_repo") + os.makedirs(str(import_repo_dir)) + import_repo = create_repo("git", str(import_repo_dir)) + import_ref = import_repo.create(str(import_dir)) + + _yaml.dump({'kind': 'import', + 'sources': [import_repo.source_config(ref=import_ref)]}, + str(sub_project.join("data.bst"))) + + sub_repo_dir = tmpdir.join("sub_repo") + os.makedirs(str(sub_repo_dir)) + sub_repo = create_repo("git", str(sub_repo_dir)) + sub_ref = sub_repo.create(str(sub_project)) + + _yaml.dump({'kind': 'junction', + 'sources': [sub_repo.source_config(ref=sub_ref)]}, + str(main_project.join("sub.bst"))) + + args = ['fetch', 'sub.bst'] + result = cli.run(project=str(main_project), args=args) + result.assert_success() + + return str(main_project) + + +def open_cross_junction(cli, tmpdir): + project = prepare_junction_project(cli, tmpdir) + workspace = tmpdir.join("workspace") + + element = 'sub.bst:data.bst' + args = ['workspace', 'open', element, str(workspace)] + result = cli.run(project=project, args=args) + result.assert_success() + + assert cli.get_element_state(project, element) == 'buildable' + assert os.path.exists(str(workspace.join('hello.txt'))) + + return project, workspace + + +def test_open_cross_junction(cli, tmpdir): + open_cross_junction(cli, tmpdir) + + +def test_list_cross_junction(cli, tmpdir): + project, workspace = open_cross_junction(cli, tmpdir) + + element = 'sub.bst:data.bst' + + args = ['workspace', 'list'] + result = cli.run(project=project, args=args) + result.assert_success() + + loaded = _yaml.load_data(result.output) + assert isinstance(loaded.get('workspaces'), list) + workspaces = loaded['workspaces'] + assert len(workspaces) == 1 + assert 'element' in workspaces[0] + assert workspaces[0]['element'] == element + + +def test_close_cross_junction(cli, tmpdir): + project, workspace = open_cross_junction(cli, tmpdir) + + element = 'sub.bst:data.bst' + args = ['workspace', 'close', '--remove-dir', element] + result = cli.run(project=project, args=args) + result.assert_success() + + assert not os.path.exists(str(workspace)) + + args = ['workspace', 'list'] + result = cli.run(project=project, args=args) + result.assert_success() + + loaded = _yaml.load_data(result.output) + assert isinstance(loaded.get('workspaces'), list) + workspaces = loaded['workspaces'] + assert len(workspaces) == 0 + + +def test_close_all_cross_junction(cli, tmpdir): + project, workspace = open_cross_junction(cli, tmpdir) + + args = ['workspace', 'close', '--remove-dir', '--all'] + result = cli.run(project=project, args=args) + result.assert_success() + + assert not os.path.exists(str(workspace)) + + args = ['workspace', 'list'] + result = cli.run(project=project, args=args) + result.assert_success() + + loaded = _yaml.load_data(result.output) + assert isinstance(loaded.get('workspaces'), list) + workspaces = loaded['workspaces'] + assert len(workspaces) == 0 diff --git a/tests/frontend/fetch.py b/tests/frontend/fetch.py index e074dadae..ee3a3c3d5 100644 --- a/tests/frontend/fetch.py +++ b/tests/frontend/fetch.py @@ -157,3 +157,41 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage): # informing the user to track the junction first result = cli.run(project=project, args=['fetch', 'junction-dep.bst']) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) + + +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) +@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) +def test_fetch_cross_junction(cli, tmpdir, datafiles, ref_storage, kind): + project = str(datafiles) + subproject_path = os.path.join(project, 'files', 'sub-project') + junction_path = os.path.join(project, 'elements', 'junction.bst') + + import_etc_path = os.path.join(subproject_path, 'elements', 'import-etc-repo.bst') + etc_files_path = os.path.join(subproject_path, 'files', 'etc-files') + + repo = create_repo(kind, str(tmpdir.join('import-etc'))) + ref = repo.create(etc_files_path) + + element = { + 'kind': 'import', + 'sources': [ + repo.source_config(ref=(ref if ref_storage == 'inline' else None)) + ] + } + _yaml.dump(element, import_etc_path) + + configure_project(project, { + 'ref-storage': ref_storage + }) + + generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == 'inline')) + + if ref_storage == 'project.refs': + result = cli.run(project=project, args=['track', 'junction.bst']) + result.assert_success() + result = cli.run(project=project, args=['track', 'junction.bst:import-etc.bst']) + result.assert_success() + + result = cli.run(project=project, args=['fetch', 'junction.bst:import-etc.bst']) + result.assert_success() diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py index 0d3890993..411ac1b31 100644 --- a/tests/frontend/pull.py +++ b/tests/frontend/pull.py @@ -3,6 +3,8 @@ import shutil import pytest from tests.testutils import cli, create_artifact_share +from . import generate_junction + # Project directory DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), @@ -226,41 +228,73 @@ def test_push_pull_non_strict(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) def test_push_pull_track_non_strict(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) + share = create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) + + # First build the target element and push to the remote. + cli.configure({ + 'artifacts': {'url': share.repo, 'push': True}, + 'projects': { + 'test': {'strict': False} + } + }) + result = cli.run(project=project, args=['build', 'target.bst']) + result.assert_success() + assert cli.get_element_state(project, 'target.bst') == 'cached' + + # Assert that everything is now cached in the remote. + share.update_summary() + all_elements = {'target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst'} + for element_name in all_elements: + assert_shared(cli, share, project, element_name) + + # Now we've pushed, delete the user's local artifact cache + # directory and try to redownload it from the share + # + artifacts = os.path.join(cli.directory, 'artifacts') + shutil.rmtree(artifacts) - with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + # Assert that nothing is cached locally anymore + for element_name in all_elements: + assert cli.get_element_state(project, element_name) != 'cached' - # First build the target element and push to the remote. - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True}, - 'projects': { - 'test': {'strict': False} - } - }) - result = cli.run(project=project, args=['build', 'target.bst']) - result.assert_success() - assert cli.get_element_state(project, 'target.bst') == 'cached' + # Now try bst build with tracking and pulling. + # Tracking will be skipped for target.bst as it doesn't have any sources. + # With the non-strict build plan target.bst immediately enters the pull queue. + # However, pulling has to be deferred until the dependencies have been + # tracked as the strict cache key needs to be calculated before querying + # the caches. + result = cli.run(project=project, args=['build', '--track-all', '--all', 'target.bst']) + result.assert_success() + assert set(result.get_pulled_elements()) == all_elements - # Assert that everything is now cached in the remote. - all_elements = {'target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst'} - for element_name in all_elements: - assert_shared(cli, share, project, element_name) - # Now we've pushed, delete the user's local artifact cache - # directory and try to redownload it from the share - # - artifacts = os.path.join(cli.directory, 'artifacts') - shutil.rmtree(artifacts) +@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux') +@pytest.mark.datafiles(DATA_DIR) +def test_push_pull_cross_junction(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + share = create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) + subproject_path = os.path.join(project, 'files', 'sub-project') + junction_path = os.path.join(project, 'elements', 'junction.bst') - # Assert that nothing is cached locally anymore - for element_name in all_elements: - assert cli.get_element_state(project, element_name) != 'cached' + generate_junction(tmpdir, subproject_path, junction_path, store_ref=True) - # Now try bst build with tracking and pulling. - # Tracking will be skipped for target.bst as it doesn't have any sources. - # With the non-strict build plan target.bst immediately enters the pull queue. - # However, pulling has to be deferred until the dependencies have been - # tracked as the strict cache key needs to be calculated before querying - # the caches. - result = cli.run(project=project, args=['build', '--track-all', '--all', 'target.bst']) - result.assert_success() - assert set(result.get_pulled_elements()) == all_elements + # First build the target element and push to the remote. + cli.configure({ + 'artifacts': {'url': share.repo, 'push': True} + }) + result = cli.run(project=project, args=['build', 'junction.bst:import-etc.bst']) + result.assert_success() + assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'cached' + + cache_dir = os.path.join(project, 'cache', 'artifacts') + shutil.rmtree(cache_dir) + + share.update_summary() + assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'buildable' + + # Now try bst pull + result = cli.run(project=project, args=['pull', 'junction.bst:import-etc.bst']) + result.assert_success() + + # And assert that it's again in the local cache, without having built + assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'cached' diff --git a/tests/frontend/push.py b/tests/frontend/push.py index 459c340bc..076324ce1 100644 --- a/tests/frontend/push.py +++ b/tests/frontend/push.py @@ -1,7 +1,12 @@ import os import pytest +from collections import namedtuple +from unittest.mock import MagicMock + from buildstream._exceptions import ErrorDomain -from tests.testutils import cli, create_artifact_share +from tests.testutils import cli, create_artifact_share, create_element_size +from tests.testutils.site import IS_LINUX +from . import configure_project, generate_junction # Project directory DATA_DIR = os.path.join( diff --git a/tests/frontend/show.py b/tests/frontend/show.py index 719dadbf4..0276961ab 100644 --- a/tests/frontend/show.py +++ b/tests/frontend/show.py @@ -111,7 +111,8 @@ def test_target_is_dependency(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) -def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage): +@pytest.mark.parametrize("element_name", ['junction-dep.bst', 'junction.bst:import-etc.bst']) +def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage, element_name): project = os.path.join(datafiles.dirname, datafiles.basename) subproject_path = os.path.join(project, 'files', 'sub-project') junction_path = os.path.join(project, 'elements', 'junction.bst') @@ -155,14 +156,15 @@ def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage): # Assert the correct error when trying to show the pipeline result = cli.run(project=project, silent=True, args=[ - 'show', 'junction-dep.bst']) + 'show', element_name]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_FETCH_NEEDED) @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) -def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage): +@pytest.mark.parametrize("element_name", ['junction-dep.bst', 'junction.bst:import-etc.bst']) +def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage, element_name): project = os.path.join(datafiles.dirname, datafiles.basename) subproject_path = os.path.join(project, 'files', 'sub-project') junction_path = os.path.join(project, 'elements', 'junction.bst') @@ -190,6 +192,43 @@ def test_inconsistent_junction(cli, tmpdir, datafiles, ref_storage): # Assert the correct error when trying to show the pipeline result = cli.run(project=project, silent=True, args=[ - 'show', 'junction-dep.bst']) + 'show', element_name]) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.SUBPROJECT_INCONSISTENT) + + +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.parametrize("element_name", ['junction-dep.bst', 'junction.bst:import-etc.bst']) +def test_fetched_junction(cli, tmpdir, datafiles, element_name): + project = os.path.join(datafiles.dirname, datafiles.basename) + subproject_path = os.path.join(project, 'files', 'sub-project') + junction_path = os.path.join(project, 'elements', 'junction.bst') + element_path = os.path.join(project, 'elements', 'junction-dep.bst') + + # Create a repo to hold the subproject and generate a junction element for it + generate_junction(tmpdir, subproject_path, junction_path, store_ref=True) + + # Create a stack element to depend on a cross junction element + # + element = { + 'kind': 'stack', + 'depends': [ + { + 'junction': 'junction.bst', + 'filename': 'import-etc.bst' + } + ] + } + _yaml.dump(element, element_path) + + result = cli.run(project=project, silent=True, args=[ + 'fetch', 'junction.bst']) + + result.assert_success() + + # Assert the correct error when trying to show the pipeline + result = cli.run(project=project, silent=True, args=[ + 'show', '--format', '%{name}-%{state}', element_name]) + + results = result.output.strip().splitlines() + assert 'junction.bst:import-etc.bst-buildable' in results diff --git a/tests/frontend/track.py b/tests/frontend/track.py index 2defc2349..51768d650 100644 --- a/tests/frontend/track.py +++ b/tests/frontend/track.py @@ -437,3 +437,46 @@ def test_junction_element(cli, tmpdir, datafiles, ref_storage): # Now assert element state (via bst show under the hood) of the dep again assert cli.get_element_state(project, 'junction-dep.bst') == 'waiting' + + +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) +@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) +def test_cross_junction(cli, tmpdir, datafiles, ref_storage, kind): + project = os.path.join(datafiles.dirname, datafiles.basename) + subproject_path = os.path.join(project, 'files', 'sub-project') + junction_path = os.path.join(project, 'elements', 'junction.bst') + etc_files = os.path.join(subproject_path, 'files', 'etc-files') + repo_element_path = os.path.join(subproject_path, 'elements', + 'import-etc-repo.bst') + + configure_project(project, { + 'ref-storage': ref_storage + }) + + repo = create_repo(kind, str(tmpdir.join('element_repo'))) + ref = repo.create(etc_files) + + generate_element(repo, repo_element_path) + + generate_junction(str(tmpdir.join('junction_repo')), + subproject_path, junction_path, store_ref=False) + + # Track the junction itself first. + result = cli.run(project=project, args=['track', 'junction.bst']) + result.assert_success() + + assert cli.get_element_state(project, 'junction.bst:import-etc-repo.bst') == 'no reference' + + # Track the cross junction element. -J is not given, it is implied. + result = cli.run(project=project, args=['track', 'junction.bst:import-etc-repo.bst']) + + if ref_storage == 'inline': + # This is not allowed to track cross junction without project.refs. + result.assert_main_error(ErrorDomain.PIPELINE, 'untrackable-sources') + else: + result.assert_success() + + assert cli.get_element_state(project, 'junction.bst:import-etc-repo.bst') == 'buildable' + + assert os.path.exists(os.path.join(project, 'project.refs')) diff --git a/tests/frontend/track_cross_junction.py b/tests/frontend/track_cross_junction.py new file mode 100644 index 000000000..34c39ddd2 --- /dev/null +++ b/tests/frontend/track_cross_junction.py @@ -0,0 +1,156 @@ +import os +import pytest +from tests.testutils import cli, create_repo, ALL_REPO_KINDS +from buildstream import _yaml + +from . import generate_junction + + +def generate_element(repo, element_path, dep_name=None): + element = { + 'kind': 'import', + 'sources': [ + repo.source_config() + ] + } + if dep_name: + element['depends'] = [dep_name] + + _yaml.dump(element, element_path) + + +def generate_import_element(tmpdir, kind, project, name): + element_name = 'import-{}.bst'.format(name) + repo_element_path = os.path.join(project, 'elements', element_name) + files = str(tmpdir.join("imported_files_{}".format(name))) + os.makedirs(files) + + with open(os.path.join(files, '{}.txt'.format(name)), 'w') as f: + f.write(name) + + subproject_path = os.path.join(str(tmpdir.join('sub-project-{}'.format(name)))) + + repo = create_repo(kind, str(tmpdir.join('element_{}_repo'.format(name)))) + ref = repo.create(files) + + generate_element(repo, repo_element_path) + + return element_name + + +def generate_project(tmpdir, name, config={}): + project_name = 'project-{}'.format(name) + subproject_path = os.path.join(str(tmpdir.join(project_name))) + os.makedirs(os.path.join(subproject_path, 'elements')) + + project_conf = { + 'name': name, + 'element-path': 'elements' + } + project_conf.update(config) + _yaml.dump(project_conf, os.path.join(subproject_path, 'project.conf')) + + return project_name, subproject_path + + +def generate_simple_stack(project, name, dependencies): + element_name = '{}.bst'.format(name) + element_path = os.path.join(project, 'elements', element_name) + element = { + 'kind': 'stack', + 'depends': dependencies + } + _yaml.dump(element, element_path) + + return element_name + + +def generate_cross_element(project, subproject_name, import_name): + basename, _ = os.path.splitext(import_name) + return generate_simple_stack(project, 'import-{}-{}'.format(subproject_name, basename), + [{ + 'junction': '{}.bst'.format(subproject_name), + 'filename': import_name + }]) + + +@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) +def test_cross_junction_multiple_projects(cli, tmpdir, datafiles, kind): + tmpdir = tmpdir.join(kind) + + # Generate 3 projects: main, a, b + _, project = generate_project(tmpdir, 'main', {'ref-storage': 'project.refs'}) + project_a, project_a_path = generate_project(tmpdir, 'a') + project_b, project_b_path = generate_project(tmpdir, 'b') + + # Generate an element with a trackable source for each project + element_a = generate_import_element(tmpdir, kind, project_a_path, 'a') + element_b = generate_import_element(tmpdir, kind, project_b_path, 'b') + element_c = generate_import_element(tmpdir, kind, project, 'c') + + # Create some indirections to the elements with dependencies to test --deps + stack_a = generate_simple_stack(project_a_path, 'stack-a', [element_a]) + stack_b = generate_simple_stack(project_b_path, 'stack-b', [element_b]) + + # Create junctions for projects a and b in main. + junction_a = '{}.bst'.format(project_a) + junction_a_path = os.path.join(project, 'elements', junction_a) + generate_junction(tmpdir.join('repo_a'), project_a_path, junction_a_path, store_ref=False) + + junction_b = '{}.bst'.format(project_b) + junction_b_path = os.path.join(project, 'elements', junction_b) + generate_junction(tmpdir.join('repo_b'), project_b_path, junction_b_path, store_ref=False) + + # Track the junctions. + result = cli.run(project=project, args=['track', junction_a, junction_b]) + result.assert_success() + + # Import elements from a and b in to main. + imported_a = generate_cross_element(project, project_a, stack_a) + imported_b = generate_cross_element(project, project_b, stack_b) + + # Generate a top level stack depending on everything + all_bst = generate_simple_stack(project, 'all', [imported_a, imported_b, element_c]) + + # Track without following junctions. But explicitly also track the elements in project a. + result = cli.run(project=project, args=['track', '--deps', 'all', all_bst, '{}:{}'.format(junction_a, stack_a)]) + result.assert_success() + + # Elements in project b should not be tracked. But elements in project a and main should. + expected = [element_c, + '{}:{}'.format(junction_a, element_a)] + assert set(result.get_tracked_elements()) == set(expected) + + +@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) +def test_track_exceptions(cli, tmpdir, datafiles, kind): + tmpdir = tmpdir.join(kind) + + _, project = generate_project(tmpdir, 'main', {'ref-storage': 'project.refs'}) + project_a, project_a_path = generate_project(tmpdir, 'a') + + element_a = generate_import_element(tmpdir, kind, project_a_path, 'a') + element_b = generate_import_element(tmpdir, kind, project_a_path, 'b') + + all_bst = generate_simple_stack(project_a_path, 'all', [element_a, + element_b]) + + junction_a = '{}.bst'.format(project_a) + junction_a_path = os.path.join(project, 'elements', junction_a) + generate_junction(tmpdir.join('repo_a'), project_a_path, junction_a_path, store_ref=False) + + result = cli.run(project=project, args=['track', junction_a]) + result.assert_success() + + imported_b = generate_cross_element(project, project_a, element_b) + indirection = generate_simple_stack(project, 'indirection', [imported_b]) + + result = cli.run(project=project, + args=['track', '--deps', 'all', + '--except', indirection, + '{}:{}'.format(junction_a, all_bst), imported_b]) + result.assert_success() + + expected = ['{}:{}'.format(junction_a, element_a), + '{}:{}'.format(junction_a, element_b)] + assert set(result.get_tracked_elements()) == set(expected) |