From 58c51dc7cef80b0b215d33536cd124e9eda2e269 Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Tue, 19 Feb 2019 09:46:40 +0000 Subject: _stream.py, _project.py: Manage GC during pipeline load Since during pipeline load we are generating a lot of objects which need to live for the duration of the BuildStream instance there is benefit to disabling GC for the duration of the load since that drastically reduces wasted CPU cycles iterating data which will remain around always. In order to limit the increase in the peak memory consumption though, we do an explicit gc.collect() after loading the YAML in, since without that, we use 60% more memory at peak, and with it, only 20%. Signed-off-by: Daniel Silverstone # Tiago Gomes +import gc import os from collections import OrderedDict from collections.abc import Mapping @@ -352,6 +353,9 @@ class Project(): ticker=None, fetch_subprojects=fetch_subprojects) + # Loading elements generates a lot of garbage, clear it now + gc.collect() + with self._context.timed_activity("Resolving elements"): elements = [ Element._new_from_meta(meta) diff --git a/buildstream/_stream.py b/buildstream/_stream.py index caaa48908..b0fce3817 100644 --- a/buildstream/_stream.py +++ b/buildstream/_stream.py @@ -941,6 +941,7 @@ class Stream(): # (list of Element): The primary element selection # (list of Element): The tracking element selection # + @utils._with_gc_disabled def _load(self, targets, track_targets, *, selection=PipelineSelection.NONE, track_selection=PipelineSelection.NONE, -- cgit v1.2.1