#
# Copyright (C) 2020 Codethink Limited
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see .
#
# Authors:
# Tristan Van Berkom
from functools import cmp_to_key
from pyroaring import BitMap, FrozenBitMap # pylint: disable=no-name-in-module
from .._exceptions import LoadError
from ..exceptions import LoadErrorReason
from ..element import Element
from ..node cimport MappingNode, Node, ProvenanceInformation, ScalarNode, SequenceNode
from .types import Symbol
# Counter to get ids to LoadElements
cdef int _counter = 0
cdef int _next_synthetic_counter():
global _counter
_counter += 1
return _counter
# Dependency():
#
# Early stage data model for dependencies objects, the LoadElement has
# Dependency objects which in turn refer to other LoadElements in the data
# model.
#
# The constructor is incomplete, normally dependencies are loaded
# via the Dependency.load() API below. The constructor arguments are
# only used as a convenience to create the dummy Dependency objects
# at the toplevel of the load sequence in the Loader.
#
# Args:
# element (LoadElement): a LoadElement on which there is a dependency
# dep_type (str): the type of dependency this dependency link is
#
cdef class Dependency:
cdef readonly LoadElement element # The resolved LoadElement
cdef readonly str dep_type # The dependency type (runtime or build or both)
cdef readonly str name # The project local dependency name
cdef readonly str junction # The junction path of the dependency name, if any
cdef readonly bint strict # Whether this is a strict dependency
cdef Node _node # The original node of the dependency
def __cinit__(self, element=None, dep_type=None):
self.element = element
self.dep_type = dep_type
self.name = None
self.junction = None
self.strict = False
self._node = None
# provenance
#
# A property to return the ProvenanceInformation for this
# dependency.
#
@property
def provenance(self):
return self._node.get_provenance()
# set_element()
#
# Sets the resolved LoadElement
#
# When Dependencies are initially loaded, the `element` member
# will be None until later on when the Loader loads the LoadElement
# objects based on the Dependency `name` and `junction`, the Loader
# will then call this to resolve the `element` member.
#
# Args:
# element (LoadElement): The resolved LoadElement
#
cpdef set_element(self, element: LoadElement):
self.element = element
# load()
#
# Load the dependency from a Node
#
# Args:
# dep (Node): A node to load the dependency from
# default_dep_type (str): The default dependency type
#
cdef load(self, Node dep, str default_dep_type):
cdef str dep_type
self._node = dep
self.element = None
if type(dep) is ScalarNode:
self.name = dep.as_str()
self.dep_type = default_dep_type
self.junction = None
self.strict = False
elif type(dep) is MappingNode:
if default_dep_type:
( dep).validate_keys(['filename', 'junction', 'strict'])
dep_type = default_dep_type
else:
( dep).validate_keys(['filename', 'type', 'junction', 'strict'])
# Make type optional, for this we set it to None
dep_type = ( dep).get_str( Symbol.TYPE, None)
if dep_type is None or dep_type == Symbol.ALL:
dep_type = None
elif dep_type not in [Symbol.BUILD, Symbol.RUNTIME]:
provenance = dep.get_scalar(Symbol.TYPE).get_provenance()
raise LoadError("{}: Dependency type '{}' is not 'build', 'runtime' or 'all'"
.format(provenance, dep_type), LoadErrorReason.INVALID_DATA)
self.name = ( dep).get_str( Symbol.FILENAME)
self.dep_type = dep_type
self.junction = ( dep).get_str( Symbol.JUNCTION, None)
self.strict = ( dep).get_bool( Symbol.STRICT, False)
# Here we disallow explicitly setting 'strict' to False.
#
# This is in order to keep the door open to allowing the project.conf
# set the default of dependency 'strict'-ness which might be useful
# for projects which use mostly static linking and the like, in which
# case we can later interpret explicitly non-strict dependencies
# as an override of the project default.
#
if self.strict == False and Symbol.STRICT in dep:
provenance = dep.get_scalar(Symbol.STRICT).get_provenance()
raise LoadError("{}: Setting 'strict' to False is unsupported"
.format(provenance), LoadErrorReason.INVALID_DATA)
else:
raise LoadError("{}: Dependency is not specified as a string or a dictionary".format(self.provenance),
LoadErrorReason.INVALID_DATA)
# Only build dependencies are allowed to be strict
#
if self.strict and self.dep_type == Symbol.RUNTIME:
raise LoadError("{}: Runtime dependency {} specified as `strict`.".format(self.provenance, self.name),
LoadErrorReason.INVALID_DATA,
detail="Only dependencies required at build time may be declared `strict`.")
# `:` characters are not allowed in filename if a junction was
# explicitly specified
if self.junction and ':' in self.name:
raise LoadError("{}: Dependency {} contains `:` in its name. "
"`:` characters are not allowed in filename when "
"junction attribute is specified.".format(self.provenance, self.name),
LoadErrorReason.INVALID_DATA)
# Attempt to split name if no junction was specified explicitly
if not self.junction and ':' in self.name:
self.junction, self.name = self.name.rsplit(':', maxsplit=1)
# LoadElement():
#
# A transient object breaking down what is loaded allowing us to
# do complex operations in multiple passes.
#
# Args:
# node (dict): A YAML loaded dictionary
# name (str): The element name
# loader (Loader): The Loader object for this element
#
cdef class LoadElement:
cdef readonly MappingNode node
cdef readonly str name
cdef readonly full_name
cdef public bint meta_done
cdef int node_id
cdef readonly object _loader
cdef readonly str link_target
cdef readonly ProvenanceInformation link_target_provenance
# TODO: if/when pyroaring exports symbols, we could type this statically
cdef object _dep_cache
cdef readonly list dependencies
def __cinit__(self, MappingNode node, str filename, object loader):
#
# Public members
#
self.node = node # The YAML node
self.name = filename # The element name
self.full_name = None # The element full name (with associated junction)
self.meta_done = False # If the MetaElement for this LoadElement is done
self.node_id = _next_synthetic_counter()
self.link_target = None # The target of a link element
self.link_target_provenance = None # The provenance of the link target
#
# Private members
#
self._loader = loader # The Loader object
self._dep_cache = None # The dependency cache, to speed up depends()
#
# Initialization
#
if loader.project.junction:
# dependency is in subproject, qualify name
self.full_name = '{}:{}'.format(loader.project.junction._get_full_name(), self.name)
else:
# dependency is in top-level project
self.full_name = self.name
self.dependencies = []
# Ensure the root node is valid
self.node.validate_keys([
'kind', 'depends', 'sources', 'sandbox',
'variables', 'environment', 'environment-nocache',
'config', 'public', 'description',
'build-depends', 'runtime-depends',
])
#
# If this is a link, resolve it right away and just
# store the link target and provenance
#
if self.node.get_str(Symbol.KIND, default=None) == 'link':
meta_element = self._loader.collect_element_no_deps(self)
element = Element._new_from_meta(meta_element)
element._initialize_state()
# Custom error for link dependencies, since we don't completely
# parse their dependencies we cannot rely on the built-in ElementError.
deps = extract_depends_from_node(self.node)
if deps:
provenance = self.node
raise LoadError(
"{}: Dependencies are forbidden for 'link' elements".format(element),
LoadErrorReason.LINK_FORBIDDEN_DEPENDENCIES
)
self.link_target = element.target
self.link_target_provenance = element.target_provenance
@property
def junction(self):
return self._loader.project.junction
# depends():
#
# Checks if this element depends on another element, directly
# or indirectly.
#
# Args:
# other (LoadElement): Another LoadElement
#
# Returns:
# (bool): True if this LoadElement depends on 'other'
#
def depends(self, LoadElement other not None):
self._ensure_depends_cache()
return other.node_id in self._dep_cache
###########################################
# Private Methods #
###########################################
cdef void _ensure_depends_cache(self):
cdef Dependency dep
if self._dep_cache:
return
self._dep_cache = BitMap()
for dep in self.dependencies:
elt = dep.element
# Ensure the cache of the element we depend on
elt._ensure_depends_cache()
# We depend on this element
self._dep_cache.add(elt.node_id)
# And we depend on everything this element depends on
self._dep_cache.update(elt._dep_cache)
self._dep_cache = FrozenBitMap(self._dep_cache)
def _dependency_cmp(Dependency dep_a, Dependency dep_b):
cdef LoadElement element_a = dep_a.element
cdef LoadElement element_b = dep_b.element
# Sort on inter element dependency first
if element_a.depends(element_b):
return 1
elif element_b.depends(element_a):
return -1
# If there are no inter element dependencies, place
# runtime only dependencies last
if dep_a.dep_type != dep_b.dep_type:
if dep_a.dep_type == Symbol.RUNTIME:
return 1
elif dep_b.dep_type == Symbol.RUNTIME:
return -1
# All things being equal, string comparison.
if element_a.name > element_b.name:
return 1
elif element_a.name < element_b.name:
return -1
# Sort local elements before junction elements
# and use string comparison between junction elements
if element_a.junction and element_b.junction:
if element_a.junction > element_b.junction:
return 1
elif element_a.junction < element_b.junction:
return -1
elif element_a.junction:
return -1
elif element_b.junction:
return 1
# This wont ever happen
return 0
# sort_dependencies():
#
# Sort dependencies of each element by their dependencies,
# so that direct dependencies which depend on other direct
# dependencies (directly or indirectly) appear later in the
# list.
#
# This avoids the need for performing multiple topological
# sorts throughout the build process.
#
# Args:
# element (LoadElement): The element to sort
# visited (set): a list of elements that should not be treated because
# because they already have been treated.
# This is useful when wanting to sort dependencies of
# multiple top level elements that might have a common
# part.
#
def sort_dependencies(LoadElement element, set visited):
cdef list working_elements = [element]
cdef Dependency dep
if element in visited:
return
visited.add(element)
# Now dependency sort, we ensure that if any direct dependency
# directly or indirectly depends on another direct dependency,
# it is found later in the list.
while working_elements:
element = working_elements.pop()
for dep in element.dependencies:
if dep.element not in visited:
visited.add(dep.element)
working_elements.append(dep.element)
element.dependencies.sort(key=cmp_to_key(_dependency_cmp))
# _extract_depends_from_node():
#
# Helper for extract_depends_from_node to get dependencies of a particular type
#
# Adds to an array of Dependency objects from a given dict node 'node',
# allows both strings and dicts for expressing the dependency.
#
# After extracting depends, the symbol is deleted from the node
#
# Args:
# node (Node): A YAML loaded dictionary
# key (str): the key on the Node corresponding to the dependency type
# default_dep_type (str): type to give to the dependency
# acc (list): a list in which to add the loaded dependencies
# rundeps (dict): a dictionary mapping dependency (junction, name) to dependency for runtime deps
# builddeps (dict): a dictionary mapping dependency (junction, name) to dependency for build deps
#
cdef void _extract_depends_from_node(Node node, str key, str default_dep_type, list acc, dict rundeps, dict builddeps) except *:
cdef SequenceNode depends = node.get_sequence(key, [])
cdef Node dep_node
cdef tuple deptup
for dep_node in depends:
dependency = Dependency()
dependency.load(dep_node, default_dep_type)
deptup = (dependency.junction, dependency.name)
if dependency.dep_type in [Symbol.BUILD, None]:
if deptup in builddeps:
raise LoadError("{}: Duplicate build dependency found at {}."
.format(dependency.provenance, builddeps[deptup].provenance),
LoadErrorReason.DUPLICATE_DEPENDENCY)
else:
builddeps[deptup] = dependency
if dependency.dep_type in [Symbol.RUNTIME, None]:
if deptup in rundeps:
raise LoadError("{}: Duplicate runtime dependency found at {}."
.format(dependency.provenance, rundeps[deptup].provenance),
LoadErrorReason.DUPLICATE_DEPENDENCY)
else:
rundeps[deptup] = dependency
acc.append(dependency)
# Now delete the field, we dont want it anymore
node.safe_del(key)
# extract_depends_from_node():
#
# Creates an array of Dependency objects from a given dict node 'node',
# allows both strings and dicts for expressing the dependency and
# throws a comprehensive LoadError in the case that the node is malformed.
#
# After extracting depends, the symbol is deleted from the node
#
# Args:
# node (Node): A YAML loaded dictionary
#
# Returns:
# (list): a list of Dependency objects
#
def extract_depends_from_node(Node node):
cdef list acc = []
cdef dict rundeps = {}
cdef dict builddeps = {}
_extract_depends_from_node(node, Symbol.BUILD_DEPENDS, Symbol.BUILD, acc, rundeps, builddeps)
_extract_depends_from_node(node, Symbol.RUNTIME_DEPENDS, Symbol.RUNTIME, acc, rundeps, builddeps)
_extract_depends_from_node(node, Symbol.DEPENDS, None, acc, rundeps, builddeps)
return acc