#!/usr/bin/env python3 # # Copyright (C) 2016 Codethink Limited # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see . # # Authors: # Tristan Van Berkom import os import hashlib import pickle from collections import deque, Mapping from . import _site from . import _yaml from . import utils from ._exceptions import LoadError, LoadErrorReason from ._profile import Topics, profile_start, profile_end # Context() # # The Context object holds all of the user preferences # and context for a given invocation of BuildStream. # # This is a collection of data from configuration files and command # line arguments and consists of information such as where to store # logs and artifacts, where to perform builds and cache downloaded sources, # verbosity levels and basically anything pertaining to the context # in which BuildStream was invoked. # class Context(): def __init__(self, cli_options): # Filename indicating which configuration file was used, or None for the defaults self.config_origin = None # Whether elements must be rebuilt when their dependencies have changed self.strict_build_plan = None # The directory where various sources are stored self.sourcedir = None # The directory where build sandboxes will be created self.builddir = None # The local binary artifact cache directory self.artifactdir = None # The URL from which to download prebuilt artifacts self.artifact_pull = None # The URL to upload built artifacts to self.artifact_push = None # The port number for pushing artifacts over ssh self.artifact_push_port = 22 # The directory to store build logs self.logdir = None # The abbreviated cache key length to display in the UI self.log_key_length = 0 # Whether debug mode is enabled self.log_debug = False # Whether verbose mode is enabled self.log_verbose = False # Maximum number of lines to print from build logs self.log_error_lines = 0 # Maximum number of lines to print in the master log for a detailed message self.log_message_lines = 0 # Format string for printing the pipeline at startup time self.log_element_format = None # Maximum number of fetch or refresh tasks self.sched_fetchers = 4 # Maximum number of build tasks self.sched_builders = 4 # Maximum number of push tasks self.sched_pushers = 4 # Maximum number of retries for network tasks self.sched_network_retries = 2 # What to do when a build fails in non interactive mode self.sched_error_action = 'continue' # Make sure the XDG vars are set in the environment before loading anything self._init_xdg() # Private variables self._cache_key = None self._message_handler = None self._message_depth = deque() self._platform = None self._project_overrides = {} self._cli_options = cli_options # load() # # Loads the configuration files # # Args: # config (filename): The user specified configuration file, if any # # Raises: # LoadError # # This will first load the BuildStream default configuration and then # override that configuration with the configuration file indicated # by *config*, if any was specified. # def load(self, config=None): profile_start(Topics.LOAD_CONTEXT, 'load') # If a specific config file is not specified, default to trying # a $XDG_CONFIG_HOME/buildstream.conf file # if not config: default_config = os.path.join(os.environ['XDG_CONFIG_HOME'], 'buildstream.conf') if os.path.exists(default_config): config = default_config # Load default config # defaults = _yaml.load(_site.default_user_config) if config: self.config_origin = os.path.abspath(config) user_config = _yaml.load(config) _yaml.composite(defaults, user_config) _yaml.node_validate(defaults, [ 'sourcedir', 'builddir', 'artifactdir', 'logdir', 'scheduler', 'artifacts', 'logging', 'projects', ]) for dir in ['sourcedir', 'builddir', 'artifactdir', 'logdir']: # Allow the ~ tilde expansion and any environment variables in # path specification in the config files. # path = _yaml.node_get(defaults, str, dir) path = os.path.expanduser(path) path = os.path.expandvars(path) setattr(self, dir, path) # Load artifact share configuration artifacts = _yaml.node_get(defaults, Mapping, 'artifacts') _yaml.node_validate(artifacts, ['pull-url', 'push-url', 'push-port']) self.artifact_pull = _yaml.node_get(artifacts, str, 'pull-url', default_value='') or None self.artifact_push = _yaml.node_get(artifacts, str, 'push-url', default_value='') or None self.artifact_push_port = _yaml.node_get(artifacts, int, 'push-port', default_value=22) # Load logging config logging = _yaml.node_get(defaults, Mapping, 'logging') _yaml.node_validate(logging, [ 'key-length', 'verbose', 'error-lines', 'message-lines', 'debug', 'element-format' ]) self.log_key_length = _yaml.node_get(logging, int, 'key-length') self.log_debug = _yaml.node_get(logging, bool, 'debug') self.log_verbose = _yaml.node_get(logging, bool, 'verbose') self.log_error_lines = _yaml.node_get(logging, int, 'error-lines') self.log_message_lines = _yaml.node_get(logging, int, 'message-lines') self.log_element_format = _yaml.node_get(logging, str, 'element-format') # Load scheduler config scheduler = _yaml.node_get(defaults, Mapping, 'scheduler') _yaml.node_validate(scheduler, [ 'on-error', 'fetchers', 'builders', 'pushers', 'network-retries' ]) self.sched_error_action = _yaml.node_get(scheduler, str, 'on-error') self.sched_fetchers = _yaml.node_get(scheduler, int, 'fetchers') self.sched_builders = _yaml.node_get(scheduler, int, 'builders') self.sched_pushers = _yaml.node_get(scheduler, int, 'pushers') self.sched_network_retries = _yaml.node_get(scheduler, int, 'network-retries') # Load per-projects overrides self._project_overrides = _yaml.node_get(defaults, Mapping, 'projects', default_value={}) # Shallow validation of overrides, parts of buildstream which rely # on the overrides are expected to validate elsewhere. for project_name, overrides in _yaml.node_items(self._project_overrides): _yaml.node_validate(overrides, ['artifacts', 'options', 'strict']) profile_end(Topics.LOAD_CONTEXT, 'load') valid_actions = ['continue', 'quit'] if self.sched_error_action not in valid_actions: provenance = _yaml.node_get_provenance(scheduler, 'on-error') raise LoadError(LoadErrorReason.INVALID_DATA, "{}: on-error should be one of: {}".format( provenance, ", ".join(valid_actions))) # _get_overrides(): # # Fetch the override dictionary for the active project. This returns # a node loaded from YAML and as such, values loaded from the returned # node should be loaded using the _yaml.node_get() family of functions. # # Args: # project_name (str): The project name # # Returns: # (Mapping): The overrides dictionary for the specified project # def _get_overrides(self, project_name): return _yaml.node_get(self._project_overrides, Mapping, project_name, default_value={}) # _get_strict(): # # Fetch whether we are strict or not # # Args: # project_name (str): The project name # # Returns: # (bool): Whether or not to use strict build plan # def _get_strict(self, project_name): # If it was set by the CLI, it overrides any config if self.strict_build_plan is not None: return self.strict_build_plan overrides = self._get_overrides(project_name) return _yaml.node_get(overrides, bool, 'strict', default_value=True) # _get_cache_key(): # # Returns the cache key, calculating it if necessary # # Returns: # (str): A hex digest cache key for the Context # def _get_cache_key(self): if self._cache_key is None: # Anything that alters the build goes into the unique key self._cache_key = utils._generate_key({}) return self._cache_key # _set_message_handler() # # Sets the handler for any status messages propagated through # the context. # # The message handler should have the same signature as # the _message() method def _set_message_handler(self, handler): self._message_handler = handler # _push_message_depth() / _pop_message_depth() # # For status messages, send the depth of timed # activities inside a given task through the message # def _push_message_depth(self, silent_nested): self._message_depth.appendleft(silent_nested) def _pop_message_depth(self): assert(self._message_depth) self._message_depth.popleft() def _silent_messages(self): for silent in self._message_depth: if silent: return True return False # _message(): # # Proxies a message back to the caller, this is the central # point through which all messages pass. # # Args: # message: A Message object # def _message(self, message): # Tag message only once if message.depth is None: message.depth = len(list(self._message_depth)) # Send it off to the log handler (can be the frontend, # or it can be the child task which will log and propagate # to the frontend) assert(self._message_handler) self._message_handler(message, context=self) return # Force the resolved XDG variables into the environment, # this is so that they can be used directly to specify # preferred locations of things from user configuration # files. def _init_xdg(self): if not os.environ.get('XDG_CACHE_HOME'): os.environ['XDG_CACHE_HOME'] = os.path.expanduser('~/.cache') if not os.environ.get('XDG_CONFIG_HOME'): os.environ['XDG_CONFIG_HOME'] = os.path.expanduser('~/.config') if not os.environ.get('XDG_DATA_HOME'): os.environ['XDG_DATA_HOME'] = os.path.expanduser('~/.local/share')