Implement strict dependency semantics

This patch allows specifying a dependency as `strict`, e.g.:

  build-depends:
  - filename: element.bst
    strict: true

This allows finer tuning of projects which want to leverage
the non-strict build mode; dependencies which are statically
linked to, or who's content is otherwise included verbatim in
the resulting output, should be marked `strict` to ensure these
bits get reassembled if necessary when building in non-strict
mode.

This fixes #254

Change summary:

    o _loader/types.pyx: Added 'strict' attribute to Dependency
      and do the parsing work.

    o _loader/metaelement.py: Added 'strict_dependencies' list

    o _loader/loader.py: Resolve the 'strict_dependencies' list

    o element.py: Added __strict_dependencies list, and use this
      to conditionally use weak cache keys in place of names for
      the purpose of building the weak cache key (in the case of
      dependencies which are marked as strict).

4 files changed, 33 insertions, 12 deletions

diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py
index b861e86cd..ec929eadb 100644
--- a/buildstream/_loader/loader.py
+++ b/buildstream/_loader/loader.py
@@ -474,6 +474,8 @@ class Loader():
                 meta_element.build_dependencies.append(meta_dep)
             if dep.dep_type != 'build':
                 meta_element.dependencies.append(meta_dep)
+            if dep.strict:
+                meta_element.strict_dependencies.append(meta_dep)
 
         return meta_element
 
diff --git a/buildstream/_loader/metaelement.py b/buildstream/_loader/metaelement.py
index c13d5591e..370ce553b 100644
--- a/buildstream/_loader/metaelement.py
+++ b/buildstream/_loader/metaelement.py
@@ -54,4 +54,5 @@ class MetaElement():
         self.sandbox = sandbox
         self.build_dependencies = []
         self.dependencies = []
+        self.strict_dependencies = []
         self.first_pass = first_pass
diff --git a/buildstream/_loader/types.py b/buildstream/_loader/types.py
index eb6932b0b..5b4d0121e 100644
--- a/buildstream/_loader/types.py
+++ b/buildstream/_loader/types.py
@@ -46,6 +46,7 @@ class Symbol():
     DIRECTORY = "directory"
     JUNCTION = "junction"
     SANDBOX = "sandbox"
+    STRICT = "strict"
 
 
 # Dependency()
@@ -68,13 +69,14 @@ class Dependency():
             self.name = dep
             self.dep_type = default_dep_type
             self.junction = None
+            self.strict = False
 
         elif isinstance(dep, Mapping):
             if default_dep_type:
-                _yaml.node_validate(dep, ['filename', 'junction'])
+                _yaml.node_validate(dep, ['filename', 'junction', 'strict'])
                 dep_type = default_dep_type
             else:
-                _yaml.node_validate(dep, ['filename', 'type', 'junction'])
+                _yaml.node_validate(dep, ['filename', 'type', 'junction', 'strict'])
 
                 # Make type optional, for this we set it to None
                 dep_type = _yaml.node_get(dep, str, Symbol.TYPE, default_value=None)
@@ -89,11 +91,19 @@ class Dependency():
             self.name = _yaml.node_get(dep, str, Symbol.FILENAME)
             self.dep_type = dep_type
             self.junction = _yaml.node_get(dep, str, Symbol.JUNCTION, default_value=None)
+            self.strict = _yaml.node_get(dep, bool, Symbol.STRICT, default_value=False)
 
         else:
             raise LoadError(LoadErrorReason.INVALID_DATA,
                             "{}: Dependency is not specified as a string or a dictionary".format(provenance))
 
+        # Only build dependencies are allowed to be strict
+        #
+        if self.strict and self.dep_type == Symbol.RUNTIME:
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            "{}: Runtime dependency {} specified as `strict`.".format(self.provenance, self.name),
+                            detail="Only dependencies required at build time may be declared `strict`.")
+
         # `:` characters are not allowed in filename if a junction was
         # explicitly specified
         if self.junction and ':' in self.name:
diff --git a/buildstream/element.py b/buildstream/element.py
index ff0fcd8d9..703f062da 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -201,6 +201,7 @@ class Element(Plugin):
 
         self.__runtime_dependencies = []        # Direct runtime dependency Elements
         self.__build_dependencies = []          # Direct build dependency Elements
+        self.__strict_dependencies = []         # Direct build dependency subset which require strict rebuilds
         self.__reverse_dependencies = set()     # Direct reverse dependency Elements
         self.__ready_for_runtime = False        # Wether the element has all its dependencies ready and has a cache key
         self.__sources = []                     # List of Sources
@@ -939,6 +940,9 @@ class Element(Plugin):
             element.__build_dependencies.append(dependency)
             dependency.__reverse_dependencies.add(element)
 
+            if meta_dep in meta.strict_dependencies:
+                element.__strict_dependencies.append(dependency)
+
         return element
 
     # _get_redundant_source_refs()
@@ -1069,16 +1073,20 @@ class Element(Plugin):
         if self.__weak_cache_key is None:
             # Calculate weak cache key
             # Weak cache key includes names of direct build dependencies
-            # but does not include keys of dependencies.
-            if self.BST_STRICT_REBUILD:
-                dependencies = [
-                    e._get_cache_key(strength=_KeyStrength.WEAK)
-                    for e in self.dependencies(Scope.BUILD)
-                ]
-            else:
-                dependencies = [
-                    e.name for e in self.dependencies(Scope.BUILD)
-                ]
+            # so as to only trigger rebuilds when the shape of the
+            # dependencies change.
+            #
+            # Some conditions cause dependencies to be strict, such
+            # that this element will be rebuilt anyway if the dependency
+            # changes even in non strict mode, for these cases we just
+            # encode the dependency's weak cache key instead of it's name.
+            #
+            dependencies = [
+                e._get_cache_key(strength=_KeyStrength.WEAK)
+                if self.BST_STRICT_REBUILD or e in self.__strict_dependencies
+                else e.name
+                for e in self.dependencies(Scope.BUILD)
+            ]
 
             self.__weak_cache_key = self.__calculate_cache_key(dependencies)