diff options
author | James Ennis <james.ennis@codethink.co.uk> | 2019-04-23 16:24:39 +0100 |
---|---|---|
committer | bst-marge-bot <marge-bot@buildstream.build> | 2019-04-24 10:02:37 +0000 |
commit | be2db853a5a5748d38131c2f1c44754ff8f495e4 (patch) | |
tree | 770a0568c7d93e90d11c04186eaf4d64bfdabd1c | |
parent | b44833516dc7b66b2423dc973a2a01748e833c3d (diff) | |
download | buildstream-be2db853a5a5748d38131c2f1c44754ff8f495e4.tar.gz |
Revert "_stream.py, _project.py: Manage GC during pipeline load"jennis/revert_gc_management
This change was initialy introduced as it provided a minor gain in the
load time of the elements, pre-scheduler.
For the base-files subset of the debian-stack, this gain was ~2 seconds.
However, with it, came a fairly significant cost to peak memory usage (~100M).
Additionally, the benchmarks have highlighted that this patch has introduced
some cost during build time, where the build of the base-files subset of the
debian-like project has increased by ~90 seconds and peak memory by ~100M
This reverts commit 746aa7a63c4d06b1caa012a35c970b43e0166faa and
commit d55039c33685d267fd8834ecc5d16030c1385325.
-rw-r--r-- | buildstream/_project.py | 4 | ||||
-rw-r--r-- | buildstream/_stream.py | 1 | ||||
-rw-r--r-- | buildstream/utils.py | 31 |
3 files changed, 0 insertions, 36 deletions
diff --git a/buildstream/_project.py b/buildstream/_project.py index 93ed9fc3e..c40321c66 100644 --- a/buildstream/_project.py +++ b/buildstream/_project.py @@ -18,7 +18,6 @@ # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> # Tiago Gomes <tiago.gomes@codethink.co.uk> -import gc import os import sys from collections import OrderedDict @@ -455,9 +454,6 @@ class Project(): ticker=None, fetch_subprojects=fetch_subprojects) - # Loading elements generates a lot of garbage, clear it now - gc.collect() - with self._context.timed_activity("Resolving elements"): elements = [ Element._new_from_meta(meta) diff --git a/buildstream/_stream.py b/buildstream/_stream.py index fa57f4468..d4f26e443 100644 --- a/buildstream/_stream.py +++ b/buildstream/_stream.py @@ -1027,7 +1027,6 @@ class Stream(): # (list of Element): The primary element selection # (list of Element): The tracking element selection # - @utils._with_gc_disabled def _load(self, targets, track_targets, *, selection=PipelineSelection.NONE, track_selection=PipelineSelection.NONE, diff --git a/buildstream/utils.py b/buildstream/utils.py index e730dc2a9..ade593750 100644 --- a/buildstream/utils.py +++ b/buildstream/utils.py @@ -23,8 +23,6 @@ Utilities import calendar import errno -import functools -import gc import hashlib import os import re @@ -1280,35 +1278,6 @@ def _search_upward_for_files(directory, filenames): directory = parent_dir -# _with_gc_disabled() -# -# Decorate a function to disable the garbage collector across its execution. -# -# In general, disabling the garbage collector should be considered to be an -# extreme action. Only use this in carefully selected subsets of the code -# where we generally create a lot more objects than we throw away. For example -# in loading the stream. -# -# Args: -# func (callable): The callable to disable the GC for -# -# Returns: -# (callable): The decorated callable -# -def _with_gc_disabled(func): - @functools.wraps(func) - def _gc_disabled(*args, **kwargs): - try: - gc.disable() - return func(*args, **kwargs) - finally: - gc.enable() - # Clean up to ensure we don't grow any more, freeing up room to be - # used by other objects during the course of running BuildStream. - gc.collect() - return _gc_disabled - - # _deterministic_umask() # # Context managed to apply a umask to a section that may be affected by a users |