Implement strict dependency semantics

This patch allows specifying a dependency as `strict`, e.g.:

  build-depends:
  - filename: element.bst
    strict: true

This allows finer tuning of projects which want to leverage
the non-strict build mode; dependencies which are statically
linked to, or who's content is otherwise included verbatim in
the resulting output, should be marked `strict` to ensure these
bits get reassembled if necessary when building in non-strict
mode.

This fixes #254

Change summary:

  o _loader/loadelement.pyx: Added 'strict' attribute to Dependency

  o _loader/types.pyx: Added 'strict' attribute to Dependency
    do the parsing work.

  o _loader/metaelement.py: Added 'strict_dependencies' list

  o _loader/loader.py: Resolve the 'strict_dependencies' list

  o element.py: Added __strict_dependencies list, and use this
    to conditionally use weak cache keys in place of names for
    the purpose of building the weak cache key (in the case of
    dependencies which are marked as strict).

5 files changed, 43 insertions, 16 deletions

diff --git a/src/buildstream/_loader/loadelement.pyx b/src/buildstream/_loader/loadelement.pyx
index 867de6f29..fb1dd1d15 100644
--- a/src/buildstream/_loader/loadelement.pyx
+++ b/src/buildstream/_loader/loadelement.pyx
@@ -39,18 +39,21 @@ cdef int _next_synthetic_counter():
 # A link from a LoadElement to its dependencies.
 #
 # Keeps a link to one of the current Element's dependencies, together with
-# its dependency type.
+# its dependency attributes.
 #
 # Args:
 #    element (LoadElement): a LoadElement on which there is a dependency
 #    dep_type (str): the type of dependency this dependency link is
+#    strict (bint): whether the dependency is strict
 cdef class Dependency:
     cdef readonly LoadElement element
     cdef readonly str dep_type
+    cdef readonly bint strict
 
-    def __cinit__(self, LoadElement element, str dep_type):
+    def __cinit__(self, LoadElement element, str dep_type, bint strict):
         self.element = element
         self.dep_type = dep_type
+        self.strict = strict
 
 
 # LoadElement():
diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py
index 061d28bf0..89458d40c 100644
--- a/src/buildstream/_loader/loader.py
+++ b/src/buildstream/_loader/loader.py
@@ -124,7 +124,7 @@ class Loader():
         dummy_target = LoadElement(Node.from_dict({}), "", self)
         # Pylint is not very happy with Cython and can't understand 'dependencies' is a list
         dummy_target.dependencies.extend(  # pylint: disable=no-member
-            Dependency(element, Symbol.RUNTIME)
+            Dependency(element, Symbol.RUNTIME, False)
             for element in target_elements
         )
 
@@ -344,7 +344,7 @@ class Loader():
                 # All is well, push the dependency onto the LoadElement
                 # Pylint is not very happy with Cython and can't understand 'dependencies' is a list
                 current_element[0].dependencies.append(  # pylint: disable=no-member
-                    Dependency(dep_element, dep.dep_type))
+                    Dependency(dep_element, dep.dep_type, dep.strict))
             else:
                 # We do not have any more dependencies to load for this
                 # element on the queue, report any invalid dep names
@@ -499,6 +499,8 @@ class Loader():
                     meta_element.build_dependencies.append(meta_dep)
                 if dep.dep_type != 'build':
                     meta_element.dependencies.append(meta_dep)
+                if dep.strict:
+                    meta_element.strict_dependencies.append(meta_dep)
 
             element.meta_done = True
 
diff --git a/src/buildstream/_loader/metaelement.py b/src/buildstream/_loader/metaelement.py
index 67d2ec771..00d8560f8 100644
--- a/src/buildstream/_loader/metaelement.py
+++ b/src/buildstream/_loader/metaelement.py
@@ -56,5 +56,6 @@ class MetaElement():
         self.sandbox = sandbox or Node.from_dict({})
         self.build_dependencies = []
         self.dependencies = []
+        self.strict_dependencies = []
         self.first_pass = first_pass
         self.is_junction = kind == "junction"
diff --git a/src/buildstream/_loader/types.pyx b/src/buildstream/_loader/types.pyx
index 0a223f9a9..cd1302b45 100644
--- a/src/buildstream/_loader/types.pyx
+++ b/src/buildstream/_loader/types.pyx
@@ -44,6 +44,7 @@ class Symbol():
     DIRECTORY = "directory"
     JUNCTION = "junction"
     SANDBOX = "sandbox"
+    STRICT = "strict"
 
 
 # Dependency()
@@ -63,6 +64,7 @@ cdef class Dependency:
     cdef public str name
     cdef public str dep_type
     cdef public str junction
+    cdef public bint strict
 
     def __init__(self,
                  Node dep,
@@ -75,13 +77,14 @@ cdef class Dependency:
             self.name = dep.as_str()
             self.dep_type = default_dep_type
             self.junction = None
+            self.strict = False
 
         elif type(dep) is MappingNode:
             if default_dep_type:
-                (<MappingNode> dep).validate_keys(['filename', 'junction'])
+                (<MappingNode> dep).validate_keys(['filename', 'junction', 'strict'])
                 dep_type = default_dep_type
             else:
-                (<MappingNode> dep).validate_keys(['filename', 'type', 'junction'])
+                (<MappingNode> dep).validate_keys(['filename', 'type', 'junction', 'strict'])
 
                 # Make type optional, for this we set it to None
                 dep_type = (<MappingNode> dep).get_str(<str> Symbol.TYPE, None)
@@ -95,11 +98,19 @@ cdef class Dependency:
             self.name = (<MappingNode> dep).get_str(<str> Symbol.FILENAME)
             self.dep_type = dep_type
             self.junction = (<MappingNode> dep).get_str(<str> Symbol.JUNCTION, None)
+            self.strict = (<MappingNode> dep).get_bool(<str> Symbol.STRICT, False)
 
         else:
             raise LoadError("{}: Dependency is not specified as a string or a dictionary".format(self.provenance),
                             LoadErrorReason.INVALID_DATA)
 
+        # Only build dependencies are allowed to be strict
+        #
+        if self.strict and self.dep_type == Symbol.RUNTIME:
+            raise LoadError("{}: Runtime dependency {} specified as `strict`.".format(self.provenance, self.name),
+                            LoadErrorReason.INVALID_DATA,
+                            detail="Only dependencies required at build time may be declared `strict`.")
+
         # `:` characters are not allowed in filename if a junction was
         # explicitly specified
         if self.junction and ':' in self.name:
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 837ad2fd6..633e29c88 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -208,6 +208,7 @@ class Element(Plugin):
 
         self.__runtime_dependencies = []        # Direct runtime dependency Elements
         self.__build_dependencies = []          # Direct build dependency Elements
+        self.__strict_dependencies = []         # Direct build dependency subset which require strict rebuilds
         self.__reverse_build_deps = set()       # Direct reverse build dependency Elements
         self.__reverse_runtime_deps = set()     # Direct reverse runtime dependency Elements
         self.__build_deps_without_strict_cache_key = None    # Number of build dependencies without a strict key
@@ -989,6 +990,10 @@ class Element(Plugin):
             dependency = Element._new_from_meta(meta_dep, task)
             element.__build_dependencies.append(dependency)
             dependency.__reverse_build_deps.add(element)
+
+            if meta_dep in meta.strict_dependencies:
+                element.__strict_dependencies.append(dependency)
+
         no_of_build_deps = len(element.__build_dependencies)
         element.__build_deps_without_strict_cache_key = no_of_build_deps
         element.__build_deps_without_cache_key = no_of_build_deps
@@ -3107,17 +3112,22 @@ class Element(Plugin):
 
         if self.__weak_cache_key is None:
             # Calculate weak cache key
+            #
             # Weak cache key includes names of direct build dependencies
-            # but does not include keys of dependencies.
-            if self.BST_STRICT_REBUILD:
-                dependencies = [
-                    e._get_cache_key(strength=_KeyStrength.WEAK)
-                    for e in self.dependencies(Scope.BUILD)
-                ]
-            else:
-                dependencies = [
-                    [e.project_name, e.name] for e in self.dependencies(Scope.BUILD)
-                ]
+            # so as to only trigger rebuilds when the shape of the
+            # dependencies change.
+            #
+            # Some conditions cause dependencies to be strict, such
+            # that this element will be rebuilt anyway if the dependency
+            # changes even in non strict mode, for these cases we just
+            # encode the dependency's weak cache key instead of it's name.
+            #
+            dependencies = [
+                e._get_cache_key(strength=_KeyStrength.WEAK)
+                if self.BST_STRICT_REBUILD or e in self.__strict_dependencies
+                else [e.project_name, e.name]
+                for e in self.dependencies(Scope.BUILD)
+            ]
 
             self.__weak_cache_key = self._calculate_cache_key(dependencies)