Add support for dependency loading from plugin

Now element plugins can provide a dependency loader that can provide
extra dependencies from the `config` node.

7 files changed, 187 insertions, 23 deletions

diff --git a/buildstream/_elementfactory.py b/buildstream/_elementfactory.py
index d6591bf4c..e0bb7c870 100644
--- a/buildstream/_elementfactory.py
+++ b/buildstream/_elementfactory.py
@@ -20,6 +20,7 @@
 from . import _site
 from ._plugincontext import PluginContext
 from .element import Element
+from ._exceptions import PluginError
 
 
 # A ElementFactory creates Element instances
@@ -61,3 +62,26 @@ class ElementFactory(PluginContext):
         version = self._format_versions.get(meta.kind, 0)
         self._assert_plugin_format(element, version)
         return element
+
+    # get_dependency_loader()
+    #
+    # Get the dependency loader for an Element plugin
+    #
+    # Args:
+    #    context (object): The Context object for processing
+    #    project (object): The project object
+    #    kind (str): The kind of the Element plugin
+    #
+    # Returns:
+    #    (DependencyLoader): An instance of the dependency loader for
+    #          the requested kind or None if not available.
+
+    def get_dependency_loader(self, context, project, kind):
+        try:
+            element_type, default_config = self.lookup(kind)
+        except PluginError:
+            return None
+
+        if not hasattr(element_type, 'DEPENDENCY_LOADER'):
+            return None
+        return element_type.DEPENDENCY_LOADER(kind, context, project, default_config)
diff --git a/buildstream/_loader/loadelement.py b/buildstream/_loader/loadelement.py
index 1c520f6fa..0b87e35fd 100644
--- a/buildstream/_loader/loadelement.py
+++ b/buildstream/_loader/loadelement.py
@@ -23,9 +23,10 @@ from collections.abc import Mapping
 # BuildStream toplevel imports
 from .._exceptions import LoadError, LoadErrorReason
 from .. import _yaml
+from ..dependency_loader import Dependency
 
 # Local package imports
-from .types import Symbol, Dependency
+from .types import Symbol
 
 
 # LoadElement():
diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py
index 16e880388..3384bde5d 100644
--- a/buildstream/_loader/loader.py
+++ b/buildstream/_loader/loader.py
@@ -31,8 +31,9 @@ from ..element import Element
 from .._profile import Topics, profile_start, profile_end
 from .._includes import Includes
 from .._yamlcache import YamlCache
+from ..dependency_loader import Dependency
 
-from .types import Symbol, Dependency
+from .types import Symbol
 from .loadelement import LoadElement
 from . import MetaElement
 from . import MetaSource
@@ -270,6 +271,13 @@ class Loader():
 
         element = LoadElement(node, filename, self)
 
+        if kind != "junction":
+            config = _yaml.node_get(node, Mapping, Symbol.CONFIG, default_value={})
+            dependency_loader = self.project.get_dependency_loader(kind)
+
+            if dependency_loader is not None:
+                element.deps.extend(dependency_loader._get_dependencies(config))
+
         self._elements[filename] = element
 
         # Load all dependency files for the new LoadElement
diff --git a/buildstream/_loader/types.py b/buildstream/_loader/types.py
index 25b785532..6affc0b3d 100644
--- a/buildstream/_loader/types.py
+++ b/buildstream/_loader/types.py
@@ -41,24 +41,3 @@ class Symbol():
     DIRECTORY = "directory"
     JUNCTION = "junction"
     SANDBOX = "sandbox"
-
-
-# Dependency()
-#
-# A simple object describing a dependency
-#
-# Args:
-#    name (str): The element name
-#    dep_type (str): The type of dependency, can be
-#                    Symbol.ALL, Symbol.BUILD, or Symbol.RUNTIME
-#    junction (str): The element name of the junction, or None
-#    provenance (Provenance): The YAML node provenance of where this
-#                             dependency was declared
-#
-class Dependency():
-    def __init__(self, name,
-                 dep_type=None, junction=None, provenance=None):
-        self.name = name
-        self.dep_type = dep_type
-        self.junction = junction
-        self.provenance = provenance
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 1492fde77..6c9616ee1 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -137,6 +137,8 @@ class Project():
         self._sandbox = None
         self._splits = None
 
+        self.__dependency_loaders = {}
+
         self._context.add_project(self)
 
         self._partially_loaded = False
@@ -269,6 +271,22 @@ class Project():
         else:
             return self.config.source_factory.create(self._context, self, meta)
 
+    # get_dependency_loader()
+    #
+    # Get an instance of dependency loader for the requested kind
+    #
+    # Args:
+    #    kind (str): The requested kind
+    #
+    # Returns:
+    #    (DependencyLoader): An instance of the dependency loader for
+    #          the requested kind or None if not available.
+    def get_dependency_loader(self, kind):
+        if kind not in self.__dependency_loaders:
+            loader = self.config.element_factory.get_dependency_loader(self._context, self, kind)
+            self.__dependency_loaders[kind] = loader
+        return self.__dependency_loaders[kind]
+
     # get_alias_uri()
     #
     # Returns the URI for a given alias, if it exists
diff --git a/buildstream/dependency_loader.py b/buildstream/dependency_loader.py
new file mode 100644
index 000000000..b96732e4a
--- /dev/null
+++ b/buildstream/dependency_loader.py
@@ -0,0 +1,133 @@
+#
+#  Copyright (C) 2019 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Valentin David <valentin.david@codethink.co.uk>
+"""
+Dependency Loader
+=================
+
+A dependency loader can extract extra dependencies from an element
+configuration. In this case, An :class:`Element
+<buildstream.element.Element>` will define a class member
+`DEPENDENCY_LOADER` to a class implementing :class:`DependencyLoader
+<buildstream.dependency_loader.DependencyLoader>`.
+
+This :class:`DependencyLoader
+<buildstream.dependency_loader.DependencyLoader>` will provide through
+implementation of :func:`DependencyLoader.get_dependencies
+<buildstream.dependency_loader.DependencyLoader.get_dependencies>` a
+list of :class:`Dependency
+<buildstream.dependency_loader.Dependency>`.
+
+Class Reference
+---------------
+
+"""
+
+import os
+from collections.abc import Mapping
+
+from . import _yaml
+
+from .plugin import Plugin
+from ._exceptions import LoadError, LoadErrorReason, ImplError
+
+
+class Dependency():
+
+    """Dependency()
+
+    A simple object describing a dependency
+
+    Args:
+       name (str): The element name
+       dep_type (str): The type of dependency, can be
+                       Symbol.ALL, Symbol.BUILD, or Symbol.RUNTIME
+       junction (str): The element name of the junction, or None
+       provenance (Provenance): The YAML node provenance of where this
+                                dependency was declared
+    """
+
+    def __init__(self, name,
+                 dep_type=None, junction=None, provenance=None):
+        self.name = name
+        assert dep_type in (None, 'build', 'runtime', 'all')
+        self.dep_type = dep_type
+        self.junction = junction
+        self.provenance = provenance
+
+
+class DependencyLoader(Plugin):
+
+    """DependencyLoader()
+
+    Base class for element dependency loaders.
+    """
+
+    def __init__(self, name, context, project, default_conf):
+
+        super().__init__(name, context, project, None, "dependency_loader")
+
+        defaults = {}
+        try:
+            defaults = _yaml.load(default_conf, os.path.basename(default_conf))
+        except LoadError as e:
+            if e.reason != LoadErrorReason.MISSING_FILE:
+                raise e
+
+        elements = project.element_overrides
+        overrides = elements.get(self.get_kind())
+        if overrides:
+            _yaml.composite(defaults, overrides)
+
+        self.__defaults = defaults
+
+    def get_dependencies(self, node):
+        """Return the list of extra dependencies given a configuration node
+
+        Args:
+           node (dict): The configuration node
+
+        Returns:
+           (list of Dependency): The extra dependencies found
+
+        This is an abstract method. Implementations of
+        DependencyLoader are required to implement this method.
+        """
+        raise ImplError("DependencyLoader plugin for element '{kind}' does not implement get_dependencies()".format(
+            kind=self.get_kind()))
+
+    # _get_dependencies()
+    #
+    # Get dependencies given a raw config node
+    #
+    # Args:
+    #    node (dict): The configuration node
+    #
+    # Returns:
+    #    (list of Dependency): The extra dependencies found
+    #
+    # This method will call DependencyLoader.get_dependencies()
+    # after applying default configuration for the plugin. The
+    # defaults come from the corresponding element plugin.
+    def _get_dependencies(self, node):
+
+        default_config = _yaml.node_get(self.__defaults, Mapping, 'config', default_value={})
+        default_config = _yaml.node_chain_copy(default_config)
+        _yaml.composite(default_config, node)
+
+        return self.get_dependencies(default_config)
diff --git a/doc/source/core_framework.rst b/doc/source/core_framework.rst
index a66f3640f..cbfeea748 100644
--- a/doc/source/core_framework.rst
+++ b/doc/source/core_framework.rst
@@ -20,3 +20,4 @@ useful for working on BuildStream itself.
    buildstream.scriptelement
    buildstream.sandbox.sandbox
    buildstream.utils
+   buildstream.dependency_loader