diff options
author | Tristan Van Berkom <tristan.van.berkom@gmail.com> | 2018-08-15 09:28:35 +0000 |
---|---|---|
committer | Tristan Van Berkom <tristan.van.berkom@gmail.com> | 2018-08-15 09:28:35 +0000 |
commit | c52f6f9a619ecc3e5aa5993cdc3ccde72a649a40 (patch) | |
tree | e6fcf307b5a00ede8aa2ee8646b15593976587c6 | |
parent | 9d074d7f08b7dad81a126969645701cfbd3a0807 (diff) | |
parent | 1a3c481461a5116edf346c1935a8272e8e4e3ae1 (diff) | |
download | buildstream-c52f6f9a619ecc3e5aa5993cdc3ccde72a649a40.tar.gz |
Merge branch 'valentindavid/fallback_mirror_git' into 'master'
Delay refreshing git submodule until we have a clone
Closes #537
See merge request BuildStream/buildstream!656
-rw-r--r-- | buildstream/plugins/sources/git.py | 4 | ||||
-rw-r--r-- | tests/frontend/mirror.py | 338 | ||||
-rw-r--r-- | tests/frontend/project/files/bar | 0 | ||||
-rw-r--r-- | tests/frontend/project/files/foo | 0 |
4 files changed, 337 insertions, 5 deletions
diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py index a7d3ad6e3..cf2f649db 100644 --- a/buildstream/plugins/sources/git.py +++ b/buildstream/plugins/sources/git.py @@ -414,8 +414,10 @@ class GitSource(Source): mirror.stage(directory) def get_source_fetchers(self): + yield self.mirror self.refresh_submodules() - return [self.mirror] + self.submodules + for submodule in self.submodules: + yield submodule ########################################################### # Local Functions # diff --git a/tests/frontend/mirror.py b/tests/frontend/mirror.py index f37cc18af..93e270b4d 100644 --- a/tests/frontend/mirror.py +++ b/tests/frontend/mirror.py @@ -140,6 +140,67 @@ def test_mirror_fetch(cli, tmpdir, datafiles, kind): @pytest.mark.datafiles(DATA_DIR) +@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) +def test_mirror_fetch_upstream_absent(cli, tmpdir, datafiles, kind): + if kind == 'ostree': + # FIXME: Mirroring fallback fails with ostree + pytest.skip("Bug #538 - ostree mirror fallback breaks assertion") + + bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr') + dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr') + upstream_repodir = os.path.join(str(tmpdir), 'upstream') + mirror_repodir = os.path.join(str(tmpdir), 'mirror') + project_dir = os.path.join(str(tmpdir), 'project') + os.makedirs(project_dir) + element_dir = os.path.join(project_dir, 'elements') + + # Create repo objects of the upstream and mirror + upstream_repo = create_repo(kind, upstream_repodir) + ref = upstream_repo.create(dev_files_path) + mirror_repo = upstream_repo.copy(mirror_repodir) + + element = { + 'kind': 'import', + 'sources': [ + upstream_repo.source_config(ref=ref) + ] + } + + element_name = 'test.bst' + element_path = os.path.join(element_dir, element_name) + full_repo = element['sources'][0]['url'] + upstream_map, repo_name = os.path.split(full_repo) + alias = 'foo-' + kind + aliased_repo = alias + ':' + repo_name + element['sources'][0]['url'] = aliased_repo + full_mirror = mirror_repo.source_config()['url'] + mirror_map, _ = os.path.split(full_mirror) + os.makedirs(element_dir) + _yaml.dump(element, element_path) + + project = { + 'name': 'test', + 'element-path': 'elements', + 'aliases': { + alias: 'http://www.example.com/' + }, + 'mirrors': [ + { + 'name': 'middle-earth', + 'aliases': { + alias: [mirror_map + "/"], + }, + }, + ] + } + project_file = os.path.join(project_dir, 'project.conf') + _yaml.dump(project, project_file) + + result = cli.run(project=project_dir, args=['fetch', element_name]) + result.assert_success() + + +@pytest.mark.datafiles(DATA_DIR) def test_mirror_fetch_multi(cli, tmpdir, datafiles): output_file = os.path.join(str(tmpdir), "output.txt") project_dir = str(tmpdir) @@ -405,10 +466,6 @@ def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind): @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) def test_mirror_from_includes(cli, tmpdir, datafiles, kind): - if kind == 'git': - # FIXME: Mirroring fallback does not work with git because it tries to - # fetch submodules on upstream. - pytest.skip("Bug #537 - Mirror fallback does not work for git") if kind == 'ostree': # FIXME: Mirroring fallback fails with ostree pytest.skip("Bug #538 - ostree mirror fallback breaks assertion") @@ -559,3 +616,276 @@ def test_mirror_junction_from_includes(cli, tmpdir, datafiles, kind): os.rename('{}.bak'.format(upstream_repo.repo), upstream_repo.repo) result = cli.run(project=project_dir, args=['fetch', element_name]) result.assert_success() + + +@pytest.mark.datafiles(DATA_DIR) +def test_mirror_git_submodule_fetch(cli, tmpdir, datafiles): + # Test that it behaves as expected with submodules, both defined in config + # and discovered when fetching. + foo_file = os.path.join(str(datafiles), 'files', 'foo') + bar_file = os.path.join(str(datafiles), 'files', 'bar') + bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr') + dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr') + mirror_dir = os.path.join(str(datafiles), 'mirror') + + defined_subrepo = create_repo('git', str(tmpdir), 'defined_subrepo') + defined_mirror_ref = defined_subrepo.create(bin_files_path) + defined_mirror = defined_subrepo.copy(mirror_dir) + defined_subref = defined_subrepo.add_file(foo_file) + + found_subrepo = create_repo('git', str(tmpdir), 'found_subrepo') + found_subref = found_subrepo.create(dev_files_path) + + main_repo = create_repo('git', str(tmpdir)) + main_mirror_ref = main_repo.create(bin_files_path) + main_repo.add_submodule('defined', 'file://' + defined_subrepo.repo) + main_repo.add_submodule('found', 'file://' + found_subrepo.repo) + main_mirror = main_repo.copy(mirror_dir) + main_ref = main_repo.add_file(bar_file) + + project_dir = os.path.join(str(tmpdir), 'project') + os.makedirs(project_dir) + element_dir = os.path.join(project_dir, 'elements') + os.makedirs(element_dir) + element = { + 'kind': 'import', + 'sources': [ + main_repo.source_config(ref=main_mirror_ref) + ] + } + element_name = 'test.bst' + element_path = os.path.join(element_dir, element_name) + + # Alias the main repo + full_repo = element['sources'][0]['url'] + _, repo_name = os.path.split(full_repo) + alias = 'foo' + aliased_repo = alias + ':' + repo_name + element['sources'][0]['url'] = aliased_repo + + # Hide the found subrepo + del element['sources'][0]['submodules']['found'] + + # Alias the defined subrepo + subrepo = element['sources'][0]['submodules']['defined']['url'] + _, repo_name = os.path.split(subrepo) + aliased_repo = alias + ':' + repo_name + element['sources'][0]['submodules']['defined']['url'] = aliased_repo + + _yaml.dump(element, element_path) + + full_mirror = main_mirror.source_config()['url'] + mirror_map, _ = os.path.split(full_mirror) + project = { + 'name': 'test', + 'element-path': 'elements', + 'aliases': { + alias: 'http://www.example.com/' + }, + 'mirrors': [ + { + 'name': 'middle-earth', + 'aliases': { + alias: [mirror_map + "/"], + }, + }, + ] + } + project_file = os.path.join(project_dir, 'project.conf') + _yaml.dump(project, project_file) + + result = cli.run(project=project_dir, args=['fetch', element_name]) + result.assert_success() + + +@pytest.mark.datafiles(DATA_DIR) +def test_mirror_fallback_git_only_submodules(cli, tmpdir, datafiles): + # Main repo has no mirror or alias. + # One submodule is overridden to use a mirror. + # There is another submodules not overriden. + # Upstream for overriden submodule is down. + # + # We expect: + # - overriden submodule is fetched from mirror. + # - other submodule is fetched. + + bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr') + dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr') + + upstream_bin_repodir = os.path.join(str(tmpdir), 'bin-upstream') + mirror_bin_repodir = os.path.join(str(tmpdir), 'bin-mirror') + upstream_bin_repo = create_repo('git', upstream_bin_repodir) + upstream_bin_repo.create(bin_files_path) + mirror_bin_repo = upstream_bin_repo.copy(mirror_bin_repodir) + + dev_repodir = os.path.join(str(tmpdir), 'dev-upstream') + dev_repo = create_repo('git', dev_repodir) + dev_repo.create(dev_files_path) + + main_files = os.path.join(str(tmpdir), 'main-files') + os.makedirs(main_files) + with open(os.path.join(main_files, 'README'), 'w') as f: + f.write("TEST\n") + main_repodir = os.path.join(str(tmpdir), 'main-upstream') + main_repo = create_repo('git', main_repodir) + main_repo.create(main_files) + + upstream_url = 'file://{}'.format(upstream_bin_repo.repo) + main_repo.add_submodule('bin', url=upstream_url) + main_repo.add_submodule('dev', url='file://{}'.format(dev_repo.repo)) + # Unlist 'dev'. + del main_repo.submodules['dev'] + + main_ref = main_repo.latest_commit() + + upstream_map, repo_name = os.path.split(upstream_url) + alias = 'foo' + aliased_repo = '{}:{}'.format(alias, repo_name) + main_repo.submodules['bin']['url'] = aliased_repo + + full_mirror = mirror_bin_repo.source_config()['url'] + mirror_map, _ = os.path.split(full_mirror) + + project_dir = os.path.join(str(tmpdir), 'project') + os.makedirs(project_dir) + element_dir = os.path.join(project_dir, 'elements') + + element = { + 'kind': 'import', + 'sources': [ + main_repo.source_config(ref=main_ref, checkout_submodules=True) + ] + } + element_name = 'test.bst' + element_path = os.path.join(element_dir, element_name) + os.makedirs(element_dir) + _yaml.dump(element, element_path) + + project = { + 'name': 'test', + 'element-path': 'elements', + 'aliases': { + alias: upstream_map + "/" + }, + 'mirrors': [ + { + 'name': 'middle-earth', + 'aliases': { + alias: [mirror_map + "/"], + } + } + ] + } + project_file = os.path.join(project_dir, 'project.conf') + _yaml.dump(project, project_file) + + # Now make the upstream unavailable. + os.rename(upstream_bin_repo.repo, '{}.bak'.format(upstream_bin_repo.repo)) + result = cli.run(project=project_dir, args=['fetch', element_name]) + result.assert_success() + + result = cli.run(project=project_dir, args=['build', element_name]) + result.assert_success() + + checkout = os.path.join(str(tmpdir), 'checkout') + result = cli.run(project=project_dir, args=['checkout', element_name, checkout]) + result.assert_success() + + assert os.path.exists(os.path.join(checkout, 'bin', 'bin', 'hello')) + assert os.path.exists(os.path.join(checkout, 'dev', 'include', 'pony.h')) + + +@pytest.mark.datafiles(DATA_DIR) +def test_mirror_fallback_git_with_submodules(cli, tmpdir, datafiles): + # Main repo has mirror. But does not list submodules. + # + # We expect: + # - we will fetch submodules anyway + + bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr') + dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr') + + bin_repodir = os.path.join(str(tmpdir), 'bin-repo') + bin_repo = create_repo('git', bin_repodir) + bin_repo.create(bin_files_path) + + dev_repodir = os.path.join(str(tmpdir), 'dev-repo') + dev_repo = create_repo('git', dev_repodir) + dev_repo.create(dev_files_path) + + main_files = os.path.join(str(tmpdir), 'main-files') + os.makedirs(main_files) + with open(os.path.join(main_files, 'README'), 'w') as f: + f.write("TEST\n") + upstream_main_repodir = os.path.join(str(tmpdir), 'main-upstream') + upstream_main_repo = create_repo('git', upstream_main_repodir) + upstream_main_repo.create(main_files) + + upstream_main_repo.add_submodule('bin', url='file://{}'.format(bin_repo.repo)) + upstream_main_repo.add_submodule('dev', url='file://{}'.format(dev_repo.repo)) + # Unlist submodules. + del upstream_main_repo.submodules['bin'] + del upstream_main_repo.submodules['dev'] + + upstream_main_ref = upstream_main_repo.latest_commit() + + mirror_main_repodir = os.path.join(str(tmpdir), 'main-mirror') + mirror_main_repo = upstream_main_repo.copy(mirror_main_repodir) + + upstream_url = mirror_main_repo.source_config()['url'] + + upstream_map, repo_name = os.path.split(upstream_url) + alias = 'foo' + aliased_repo = '{}:{}'.format(alias, repo_name) + + full_mirror = mirror_main_repo.source_config()['url'] + mirror_map, _ = os.path.split(full_mirror) + + project_dir = os.path.join(str(tmpdir), 'project') + os.makedirs(project_dir) + element_dir = os.path.join(project_dir, 'elements') + + element = { + 'kind': 'import', + 'sources': [ + upstream_main_repo.source_config(ref=upstream_main_ref, checkout_submodules=True) + ] + } + element['sources'][0]['url'] = aliased_repo + element_name = 'test.bst' + element_path = os.path.join(element_dir, element_name) + os.makedirs(element_dir) + _yaml.dump(element, element_path) + + project = { + 'name': 'test', + 'element-path': 'elements', + 'aliases': { + alias: upstream_map + "/" + }, + 'mirrors': [ + { + 'name': 'middle-earth', + 'aliases': { + alias: [mirror_map + "/"], + } + } + ] + } + project_file = os.path.join(project_dir, 'project.conf') + _yaml.dump(project, project_file) + + # Now make the upstream unavailable. + os.rename(upstream_main_repo.repo, '{}.bak'.format(upstream_main_repo.repo)) + result = cli.run(project=project_dir, args=['fetch', element_name]) + result.assert_success() + + result = cli.run(project=project_dir, args=['build', element_name]) + result.assert_success() + + checkout = os.path.join(str(tmpdir), 'checkout') + result = cli.run(project=project_dir, args=['checkout', element_name, checkout]) + result.assert_success() + + assert os.path.exists(os.path.join(checkout, 'bin', 'bin', 'hello')) + assert os.path.exists(os.path.join(checkout, 'dev', 'include', 'pony.h')) diff --git a/tests/frontend/project/files/bar b/tests/frontend/project/files/bar new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/frontend/project/files/bar diff --git a/tests/frontend/project/files/foo b/tests/frontend/project/files/foo new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/frontend/project/files/foo |