diff options
Diffstat (limited to 'tests/frontend/pull.py')
| -rw-r--r-- | tests/frontend/pull.py | 377 |
1 files changed, 182 insertions, 195 deletions
diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py index c43cc83e3..a41c3498a 100644 --- a/tests/frontend/pull.py +++ b/tests/frontend/pull.py @@ -2,7 +2,6 @@ import os import shutil import pytest from tests.testutils import cli, create_artifact_share -from tests.testutils.site import IS_LINUX from . import generate_junction @@ -42,43 +41,42 @@ def assert_not_shared(cli, share, project, element_name): # * `bst build` pushes all build elements to configured 'push' cache # * `bst pull --deps all` downloads everything from cache after local deletion # -@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux') @pytest.mark.datafiles(DATA_DIR) def test_push_pull_all(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) - share = create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) - - # First build the target element and push to the remote. - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True} - }) - result = cli.run(project=project, args=['build', 'target.bst']) - result.assert_success() - assert cli.get_element_state(project, 'target.bst') == 'cached' - - # Assert that everything is now cached in the remote. - share.update_summary() - all_elements = ['target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst'] - for element_name in all_elements: - assert_shared(cli, share, project, element_name) - - # Now we've pushed, delete the user's local artifact cache - # directory and try to redownload it from the share - # - artifacts = os.path.join(cli.directory, 'artifacts') - shutil.rmtree(artifacts) - # Assert that nothing is cached locally anymore - for element_name in all_elements: - assert cli.get_element_state(project, element_name) != 'cached' + with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + + # First build the target element and push to the remote. + cli.configure({ + 'artifacts': {'url': share.repo, 'push': True} + }) + result = cli.run(project=project, args=['build', 'target.bst']) + result.assert_success() + assert cli.get_element_state(project, 'target.bst') == 'cached' + + # Assert that everything is now cached in the remote. + all_elements = ['target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst'] + for element_name in all_elements: + assert_shared(cli, share, project, element_name) + + # Now we've pushed, delete the user's local artifact cache + # directory and try to redownload it from the share + # + artifacts = os.path.join(cli.directory, 'artifacts') + shutil.rmtree(artifacts) - # Now try bst pull - result = cli.run(project=project, args=['pull', '--deps', 'all', 'target.bst']) - result.assert_success() + # Assert that nothing is cached locally anymore + for element_name in all_elements: + assert cli.get_element_state(project, element_name) != 'cached' - # And assert that it's again in the local cache, without having built - for element_name in all_elements: - assert cli.get_element_state(project, element_name) == 'cached' + # Now try bst pull + result = cli.run(project=project, args=['pull', '--deps', 'all', 'target.bst']) + result.assert_success() + + # And assert that it's again in the local cache, without having built + for element_name in all_elements: + assert cli.get_element_state(project, element_name) == 'cached' # Tests that: @@ -86,44 +84,40 @@ def test_push_pull_all(cli, tmpdir, datafiles): # * `bst build` pushes all build elements ONLY to configured 'push' cache # * `bst pull` finds artifacts that are available only in the secondary cache # -@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux') @pytest.mark.datafiles(DATA_DIR) def test_pull_secondary_cache(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) - share1 = create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) - share2 = create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) - - # Build the target and push it to share2 only. - cli.configure({ - 'artifacts': [ - {'url': share1.repo, 'push': False}, - {'url': share2.repo, 'push': True}, - ] - }) - result = cli.run(project=project, args=['build', 'target.bst']) - result.assert_success() + with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as share1,\ + create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as share2: - share1.update_summary() - share2.update_summary() + # Build the target and push it to share2 only. + cli.configure({ + 'artifacts': [ + {'url': share1.repo, 'push': False}, + {'url': share2.repo, 'push': True}, + ] + }) + result = cli.run(project=project, args=['build', 'target.bst']) + result.assert_success() - assert_not_shared(cli, share1, project, 'target.bst') - assert_shared(cli, share2, project, 'target.bst') + assert_not_shared(cli, share1, project, 'target.bst') + assert_shared(cli, share2, project, 'target.bst') - # Delete the user's local artifact cache. - artifacts = os.path.join(cli.directory, 'artifacts') - shutil.rmtree(artifacts) + # Delete the user's local artifact cache. + artifacts = os.path.join(cli.directory, 'artifacts') + shutil.rmtree(artifacts) - # Assert that the element is not cached anymore. - assert cli.get_element_state(project, 'target.bst') != 'cached' + # Assert that the element is not cached anymore. + assert cli.get_element_state(project, 'target.bst') != 'cached' - # Now try bst pull - result = cli.run(project=project, args=['pull', 'target.bst']) - result.assert_success() + # Now try bst pull + result = cli.run(project=project, args=['pull', 'target.bst']) + result.assert_success() - # And assert that it's again in the local cache, without having built, - # i.e. we found it in share2. - assert cli.get_element_state(project, 'target.bst') == 'cached' + # And assert that it's again in the local cache, without having built, + # i.e. we found it in share2. + assert cli.get_element_state(project, 'target.bst') == 'cached' # Tests that: @@ -131,182 +125,175 @@ def test_pull_secondary_cache(cli, tmpdir, datafiles): # * `bst push --remote` pushes to the given remote, not one from the config # * `bst pull --remote` pulls from the given remote # -@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux') @pytest.mark.datafiles(DATA_DIR) def test_push_pull_specific_remote(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) - good_share = create_artifact_share(os.path.join(str(tmpdir), 'goodartifactshare')) - bad_share = create_artifact_share(os.path.join(str(tmpdir), 'badartifactshare')) + with create_artifact_share(os.path.join(str(tmpdir), 'goodartifactshare')) as good_share,\ + create_artifact_share(os.path.join(str(tmpdir), 'badartifactshare')) as bad_share: - # Build the target so we have it cached locally only. - result = cli.run(project=project, args=['build', 'target.bst']) - result.assert_success() + # Build the target so we have it cached locally only. + result = cli.run(project=project, args=['build', 'target.bst']) + result.assert_success() - state = cli.get_element_state(project, 'target.bst') - assert state == 'cached' + state = cli.get_element_state(project, 'target.bst') + assert state == 'cached' - # Configure the default push location to be bad_share; we will assert that - # nothing actually gets pushed there. - cli.configure({ - 'artifacts': {'url': bad_share.repo, 'push': True}, - }) + # Configure the default push location to be bad_share; we will assert that + # nothing actually gets pushed there. + cli.configure({ + 'artifacts': {'url': bad_share.repo, 'push': True}, + }) - # Now try `bst push` to the good_share. - result = cli.run(project=project, args=[ - 'push', 'target.bst', '--remote', good_share.repo - ]) - result.assert_success() + # Now try `bst push` to the good_share. + result = cli.run(project=project, args=[ + 'push', 'target.bst', '--remote', good_share.repo + ]) + result.assert_success() - good_share.update_summary() - bad_share.update_summary() + # Assert that all the artifacts are in the share we pushed + # to, and not the other. + assert_shared(cli, good_share, project, 'target.bst') + assert_not_shared(cli, bad_share, project, 'target.bst') - # Assert that all the artifacts are in the share we pushed - # to, and not the other. - assert_shared(cli, good_share, project, 'target.bst') - assert_not_shared(cli, bad_share, project, 'target.bst') - - # Now we've pushed, delete the user's local artifact cache - # directory and try to redownload it from the good_share. - # - artifacts = os.path.join(cli.directory, 'artifacts') - shutil.rmtree(artifacts) + # Now we've pushed, delete the user's local artifact cache + # directory and try to redownload it from the good_share. + # + artifacts = os.path.join(cli.directory, 'artifacts') + shutil.rmtree(artifacts) - result = cli.run(project=project, args=['pull', 'target.bst', '--remote', - good_share.repo]) - result.assert_success() + result = cli.run(project=project, args=['pull', 'target.bst', '--remote', + good_share.repo]) + result.assert_success() - # And assert that it's again in the local cache, without having built - assert cli.get_element_state(project, 'target.bst') == 'cached' + # And assert that it's again in the local cache, without having built + assert cli.get_element_state(project, 'target.bst') == 'cached' # Tests that: # # * In non-strict mode, dependency changes don't block artifact reuse # -@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux') @pytest.mark.datafiles(DATA_DIR) def test_push_pull_non_strict(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) - share = create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) - workspace = os.path.join(str(tmpdir), 'workspace') - - # First build the target element and push to the remote. - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True}, - 'projects': { - 'test': {'strict': False} - } - }) - result = cli.run(project=project, args=['build', 'target.bst']) - result.assert_success() - assert cli.get_element_state(project, 'target.bst') == 'cached' - - # Assert that everything is now cached in the remote. - share.update_summary() - all_elements = ['target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst'] - for element_name in all_elements: - assert_shared(cli, share, project, element_name) - - # Now we've pushed, delete the user's local artifact cache - # directory and try to redownload it from the share - # - artifacts = os.path.join(cli.directory, 'artifacts') - shutil.rmtree(artifacts) - # Assert that nothing is cached locally anymore - for element_name in all_elements: - assert cli.get_element_state(project, element_name) != 'cached' + with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + workspace = os.path.join(str(tmpdir), 'workspace') + + # First build the target element and push to the remote. + cli.configure({ + 'artifacts': {'url': share.repo, 'push': True}, + 'projects': { + 'test': {'strict': False} + } + }) + result = cli.run(project=project, args=['build', 'target.bst']) + result.assert_success() + assert cli.get_element_state(project, 'target.bst') == 'cached' + + # Assert that everything is now cached in the remote. + all_elements = ['target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst'] + for element_name in all_elements: + assert_shared(cli, share, project, element_name) + + # Now we've pushed, delete the user's local artifact cache + # directory and try to redownload it from the share + # + artifacts = os.path.join(cli.directory, 'artifacts') + shutil.rmtree(artifacts) - # Add a file to force change in strict cache key of import-bin.bst - with open(os.path.join(str(project), 'files', 'bin-files', 'usr', 'bin', 'world'), 'w') as f: - f.write('world') + # Assert that nothing is cached locally anymore + for element_name in all_elements: + assert cli.get_element_state(project, element_name) != 'cached' - # Assert that the workspaced element requires a rebuild - assert cli.get_element_state(project, 'import-bin.bst') == 'buildable' - # Assert that the target is still waiting due to --no-strict - assert cli.get_element_state(project, 'target.bst') == 'waiting' + # Add a file to force change in strict cache key of import-bin.bst + with open(os.path.join(str(project), 'files', 'bin-files', 'usr', 'bin', 'world'), 'w') as f: + f.write('world') - # Now try bst pull - result = cli.run(project=project, args=['pull', '--deps', 'all', 'target.bst']) - result.assert_success() + # Assert that the workspaced element requires a rebuild + assert cli.get_element_state(project, 'import-bin.bst') == 'buildable' + # Assert that the target is still waiting due to --no-strict + assert cli.get_element_state(project, 'target.bst') == 'waiting' - # And assert that the target is again in the local cache, without having built - assert cli.get_element_state(project, 'target.bst') == 'cached' + # Now try bst pull + result = cli.run(project=project, args=['pull', '--deps', 'all', 'target.bst']) + result.assert_success() + + # And assert that the target is again in the local cache, without having built + assert cli.get_element_state(project, 'target.bst') == 'cached' # Regression test for https://gitlab.com/BuildStream/buildstream/issues/202 -@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux') @pytest.mark.datafiles(DATA_DIR) def test_push_pull_track_non_strict(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) - share = create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) - - # First build the target element and push to the remote. - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True}, - 'projects': { - 'test': {'strict': False} - } - }) - result = cli.run(project=project, args=['build', 'target.bst']) - result.assert_success() - assert cli.get_element_state(project, 'target.bst') == 'cached' - - # Assert that everything is now cached in the remote. - share.update_summary() - all_elements = {'target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst'} - for element_name in all_elements: - assert_shared(cli, share, project, element_name) - - # Now we've pushed, delete the user's local artifact cache - # directory and try to redownload it from the share - # - artifacts = os.path.join(cli.directory, 'artifacts') - shutil.rmtree(artifacts) - # Assert that nothing is cached locally anymore - for element_name in all_elements: - assert cli.get_element_state(project, element_name) != 'cached' + with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + + # First build the target element and push to the remote. + cli.configure({ + 'artifacts': {'url': share.repo, 'push': True}, + 'projects': { + 'test': {'strict': False} + } + }) + result = cli.run(project=project, args=['build', 'target.bst']) + result.assert_success() + assert cli.get_element_state(project, 'target.bst') == 'cached' + + # Assert that everything is now cached in the remote. + all_elements = {'target.bst', 'import-bin.bst', 'import-dev.bst', 'compose-all.bst'} + for element_name in all_elements: + assert_shared(cli, share, project, element_name) + + # Now we've pushed, delete the user's local artifact cache + # directory and try to redownload it from the share + # + artifacts = os.path.join(cli.directory, 'artifacts') + shutil.rmtree(artifacts) + + # Assert that nothing is cached locally anymore + for element_name in all_elements: + assert cli.get_element_state(project, element_name) != 'cached' + + # Now try bst build with tracking and pulling. + # Tracking will be skipped for target.bst as it doesn't have any sources. + # With the non-strict build plan target.bst immediately enters the pull queue. + # However, pulling has to be deferred until the dependencies have been + # tracked as the strict cache key needs to be calculated before querying + # the caches. + result = cli.run(project=project, args=['build', '--track-all', '--all', 'target.bst']) + result.assert_success() + assert set(result.get_pulled_elements()) == all_elements - # Now try bst build with tracking and pulling. - # Tracking will be skipped for target.bst as it doesn't have any sources. - # With the non-strict build plan target.bst immediately enters the pull queue. - # However, pulling has to be deferred until the dependencies have been - # tracked as the strict cache key needs to be calculated before querying - # the caches. - result = cli.run(project=project, args=['build', '--track-all', '--all', 'target.bst']) - result.assert_success() - assert set(result.get_pulled_elements()) == all_elements - -@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux') @pytest.mark.datafiles(DATA_DIR) def test_push_pull_cross_junction(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) - share = create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - generate_junction(tmpdir, subproject_path, junction_path, store_ref=True) + with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + subproject_path = os.path.join(project, 'files', 'sub-project') + junction_path = os.path.join(project, 'elements', 'junction.bst') + + generate_junction(tmpdir, subproject_path, junction_path, store_ref=True) - # First build the target element and push to the remote. - cli.configure({ - 'artifacts': {'url': share.repo, 'push': True} - }) - result = cli.run(project=project, args=['build', 'junction.bst:import-etc.bst']) - result.assert_success() - assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'cached' + # First build the target element and push to the remote. + cli.configure({ + 'artifacts': {'url': share.repo, 'push': True} + }) + result = cli.run(project=project, args=['build', 'junction.bst:import-etc.bst']) + result.assert_success() + assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'cached' - cache_dir = os.path.join(project, 'cache', 'artifacts') - shutil.rmtree(cache_dir) + cache_dir = os.path.join(project, 'cache', 'artifacts') + shutil.rmtree(cache_dir) - share.update_summary() - assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'buildable' + assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'buildable' - # Now try bst pull - result = cli.run(project=project, args=['pull', 'junction.bst:import-etc.bst']) - result.assert_success() + # Now try bst pull + result = cli.run(project=project, args=['pull', 'junction.bst:import-etc.bst']) + result.assert_success() - # And assert that it's again in the local cache, without having built - assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'cached' + # And assert that it's again in the local cache, without having built + assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'cached' |