diff options
author | Sam Thursfield <sam.thursfield@codethink.co.uk> | 2018-02-23 17:37:58 +0000 |
---|---|---|
committer | Sam Thursfield <sam.thursfield@codethink.co.uk> | 2018-02-23 17:37:58 +0000 |
commit | f50042eff26fe7e2d21e7aa04db267f6d53e07f5 (patch) | |
tree | dc0984578ce86cffec2642e741e5e91a5be997f2 | |
parent | f508685a3f5f77ee551cb27554186c679bd7911f (diff) | |
download | buildstream-sam/compose-log-splits.tar.gz |
-rw-r--r-- | buildstream/element.py | 8 | ||||
-rw-r--r-- | buildstream/plugins/elements/compose.py | 48 |
2 files changed, 24 insertions, 32 deletions
diff --git a/buildstream/element.py b/buildstream/element.py index f92b2415f..ac76010b9 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -535,13 +535,6 @@ class Element(Plugin): raise ElementError("Non-whitelisted overlaps detected and fail-on-overlaps is set", detail=error_detail, reason="overlap-error") - if overwrites: - detail = "Staged files overwrite existing files in staging area:\n" - for key, value in overwrites.items(): - detail += "\nFrom {}:\n".format(key) - detail += " " + " ".join(["/" + f + "\n" for f in value]) - self.warn("Overlapping files", detail=detail) - if ignored: detail = "Not staging files which would replace non-empty directories:\n" for key, value in ignored.items(): @@ -1858,6 +1851,7 @@ class Element(Plugin): include_file = True included_by_domains.append(domain) if domain in exclude: + print("Exclude {} due to {}".format(filename, domain)) exclude_file = True if orphans and not claimed_file: diff --git a/buildstream/plugins/elements/compose.py b/buildstream/plugins/elements/compose.py index 7b6f99cae..9b42bd5a9 100644 --- a/buildstream/plugins/elements/compose.py +++ b/buildstream/plugins/elements/compose.py @@ -98,14 +98,16 @@ class ComposeElement(Element): with self.timed_activity("Staging dependencies", silent_nested=True): self.stage_dependency_artifacts(sandbox, Scope.BUILD) - manifest = set() + file_list = set() + artifact_map = dict() if require_split: with self.timed_activity("Computing split", silent_nested=True): for dep in self.dependencies(Scope.BUILD): - files = dep.compute_manifest(include=self.include, + manifest = dep.compute_manifest(include=self.include, exclude=self.exclude, orphans=self.include_orphans) - manifest.update(files) + file_list.update(manifest.keys()) + artifact_map.update(manifest) # Make a snapshot of all the files. basedir = sandbox.get_directory() @@ -128,8 +130,10 @@ class ComposeElement(Element): if require_split: seen = set() + print("\n\n\nsnapshot: {}\n\n\n".format(snapshot)) # Calculate added modified files for path in utils.list_relative_paths(basedir): + print("Got: {}".format(path)) seen.add(path) if snapshot.get(path) is None: added_files.append(path) @@ -138,7 +142,7 @@ class ComposeElement(Element): # Calculate removed files removed_files = [ - path for path in manifest + path for path in file_list if path not in seen ] self.info("Integration modified {}, added {} and removed {} files" @@ -152,8 +156,10 @@ class ComposeElement(Element): # Do we want to force include files which were modified by # the integration commands, even if they were not added ? # - manifest.update(added_files) - manifest.difference_update(removed_files) + file_list.update(added_files) + file_list.difference_update(removed_files) + + print("Explicitly removeD: {}".format(removed_files)) # XXX We should be moving things outside of the build sandbox # instead of into a subdir. The element assemble() method should @@ -182,23 +188,13 @@ class ComposeElement(Element): detail = "\n".join(lines) + total_files = len([f for f in file_list if f != '.']) + with self.timed_activity("Creating composition", detail=detail, silent_nested=True): - manifest = self.stage_dependency_artifacts(sandbox, Scope.BUILD, - path=stagedir, - include=self.include, - exclude=self.exclude, - orphans=self.include_orphans) - - if self.integration: - self.status("Moving {} integration files".format(len(integration_files))) - utils.move_files(basedir, installdir, integration_files) - - for filename in integration_files: - manifest[filename] = manifest.get(filename, {}) - manifest[filename]['integration'] = True - - total_files = len(manifest) - detail = self._readable_manifest(manifest) + self.info("Composing {} files".format(total_files)) + utils.link_files(basedir, installdir, files=file_list) + + detail = self._readable_manifest(file_list, artifact_map) self.log("Composed {} files".format(total_files), detail=detail) # And we're done @@ -206,15 +202,17 @@ class ComposeElement(Element): # Show a list of files that made it into the artifact, grouped by the # artifact and split-rules domains that resulted in each one being there. - def _readable_manifest(self, manifest): + def _readable_manifest(self, file_list, artifact_map): domains = collections.defaultdict(list) # Convert the filename->domain mapping into a domain->filename mapping. - for filename, entry in manifest.items(): + for filename in file_list: + print("filename: {}, map: {}".format(filename, artifact_map.get(filename))) if filename == '.': continue - if 'artifact' in entry: + if filename in artifact_map: + entry = artifact_map[filename] domains_for_file = entry.get('domains') or ["(no domain)"] for domain in domains_for_file: full_domain_name = entry['artifact'].name + " " + domain |