_project.py: Add fatal-warnings configuration item

This allows for users to configure fatal-warnings to be either a list of
warnings.

This commit deprecates the use of fail-on-overlap within project.conf,
this will now use the fatal-warnings configuration item.

element.py: Cache key calculation now takes into account all of the
fatal-warnings

tests: This modifys the tests/frontend/overlaps.py tests to support the
new fatal-warnings configuration. Backwards compatibility is also
tested for `fail-on-overlap`

_versions.py: BST_FORMAT_VERSION bumped to 15 for fatal-warnings
	      BST_CORE_ARTIFACT_VERSION bumpted to 5 for fatal-warnings

Fixes: #526

33 files changed, 152 insertions, 69 deletions

diff --git a/buildstream/_project.py b/buildstream/_project.py
index 951662e2b..702fd81f4 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -31,6 +31,7 @@ from ._options import OptionPool
 from ._artifactcache import ArtifactCache
 from ._elementfactory import ElementFactory
 from ._sourcefactory import SourceFactory
+from .plugin import CoreWarnings
 from ._projectrefs import ProjectRefs, ProjectRefStorage
 from ._versions import BST_FORMAT_VERSION
 from ._loader import Loader
@@ -105,7 +106,7 @@ class Project():
         self.first_pass_config = ProjectConfig()
 
         self.junction = junction                 # The junction Element object, if this is a subproject
-        self.fail_on_overlap = False             # Whether overlaps are treated as errors
+
         self.ref_storage = None                  # ProjectRefStorage setting
         self.base_environment = {}               # The base set of environment variables
         self.base_env_nocache = None             # The base nocache mask (list) for the environment
@@ -120,6 +121,8 @@ class Project():
         self._cli_options = cli_options
         self._cache_key = None
 
+        self._fatal_warnings = []             # A list of warnings which should trigger an error
+
         self._shell_command = []      # The default interactive shell command
         self._shell_environment = {}  # Statically set environment vars
         self._shell_host_files = []   # A list of HostMount objects
@@ -456,7 +459,7 @@ class Project():
             'split-rules', 'elements', 'plugins',
             'aliases', 'name',
             'artifacts', 'options',
-            'fail-on-overlap', 'shell',
+            'fail-on-overlap', 'shell', 'fatal-warnings',
             'ref-storage', 'sandbox', 'mirrors'
         ])
 
@@ -478,8 +481,25 @@ class Project():
         # Load project split rules
         self._splits = _yaml.node_get(config, Mapping, 'split-rules')
 
-        # Fail on overlap
-        self.fail_on_overlap = _yaml.node_get(config, bool, 'fail-on-overlap')
+        # Fatal warnings
+        self._fatal_warnings = _yaml.node_get(config, list, 'fatal-warnings', default_value=[])
+
+        # Support backwards compatibility for fail-on-overlap
+        fail_on_overlap = _yaml.node_get(config, bool, 'fail-on-overlap', default_value=None)
+
+        if (CoreWarnings.OVERLAPS not in self._fatal_warnings) and fail_on_overlap:
+            self._fatal_warnings.append(CoreWarnings.OVERLAPS)
+
+        # Deprecation check
+        if fail_on_overlap is not None:
+            self._context.message(
+                Message(
+                    None,
+                    MessageType.WARN,
+                    "Use of fail-on-overlap within project.conf " +
+                    "is deprecated. Consider using fatal-warnings instead."
+                )
+            )
 
         # Load project.refs if it exists, this may be ignored.
         if self.ref_storage == ProjectRefStorage.PROJECT_REFS:
@@ -712,3 +732,17 @@ class Project():
                 # paths are passed in relative to the project, but must be absolute
                 origin_dict['path'] = os.path.join(self.directory, path)
             destination.append(origin_dict)
+
+    # _warning_is_fatal():
+    #
+    # Returns true if the warning in question should be considered fatal based on
+    # the project configuration.
+    #
+    # Args:
+    #   warning_str (str): The warning configuration string to check against
+    #
+    # Returns:
+    #    (bool): True if the warning should be considered fatal and cause an error.
+    #
+    def _warning_is_fatal(self, warning_str):
+        return warning_str in self._fatal_warnings
diff --git a/buildstream/_versions.py b/buildstream/_versions.py
index 3f32847ae..a722e11a5 100644
--- a/buildstream/_versions.py
+++ b/buildstream/_versions.py
@@ -23,7 +23,7 @@
 # This version is bumped whenever enhancements are made
 # to the `project.conf` format or the core element format.
 #
-BST_FORMAT_VERSION = 14
+BST_FORMAT_VERSION = 15
 
 
 # The base BuildStream artifact version
@@ -33,4 +33,4 @@ BST_FORMAT_VERSION = 14
 # or if buildstream was changed in a way which can cause
 # the same cache key to produce something that is no longer
 # the same.
-BST_CORE_ARTIFACT_VERSION = 4
+BST_CORE_ARTIFACT_VERSION = 5
diff --git a/buildstream/data/projectconfig.yaml b/buildstream/data/projectconfig.yaml
index b4ad2dcb9..686040ee0 100644
--- a/buildstream/data/projectconfig.yaml
+++ b/buildstream/data/projectconfig.yaml
@@ -13,10 +13,6 @@ element-path: .
 # Store source references in element files
 ref-storage: inline
 
-# Overlaps are just warnings
-fail-on-overlap: False
-
-
 # Variable Configuration
 #
 variables:
diff --git a/buildstream/element.py b/buildstream/element.py
index a099e934b..bb205c777 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -94,6 +94,7 @@ from . import _cachekey
 from . import _signals
 from . import _site
 from ._platform import Platform
+from .plugin import CoreWarnings
 from .sandbox._config import SandboxConfig
 
 from .storage.directory import Directory
@@ -746,32 +747,23 @@ class Element(Plugin):
                 ignored[dep.name] = result.ignored
 
         if overlaps:
-            overlap_error = overlap_warning = False
-            error_detail = warning_detail = "Staged files overwrite existing files in staging area:\n"
+            overlap_warning = False
+            warning_detail = "Staged files overwrite existing files in staging area:\n"
             for f, elements in overlaps.items():
-                overlap_error_elements = []
                 overlap_warning_elements = []
                 # The bottom item overlaps nothing
                 overlapping_elements = elements[1:]
                 for elm in overlapping_elements:
                     element = self.search(scope, elm)
-                    element_project = element._get_project()
                     if not element.__file_is_whitelisted(f):
-                        if element_project.fail_on_overlap:
-                            overlap_error_elements.append(elm)
-                            overlap_error = True
-                        else:
-                            overlap_warning_elements.append(elm)
-                            overlap_warning = True
+                        overlap_warning_elements.append(elm)
+                        overlap_warning = True
 
                 warning_detail += _overlap_error_detail(f, overlap_warning_elements, elements)
-                error_detail += _overlap_error_detail(f, overlap_error_elements, elements)
 
             if overlap_warning:
-                self.warn("Non-whitelisted overlaps detected", detail=warning_detail)
-            if overlap_error:
-                raise ElementError("Non-whitelisted overlaps detected and fail-on-overlaps is set",
-                                   detail=error_detail, reason="overlap-error")
+                self.warn("Non-whitelisted overlaps detected", detail=warning_detail,
+                          warning_token=CoreWarnings.OVERLAPS)
 
         if ignored:
             detail = "Not staging files which would replace non-empty directories:\n"
@@ -2054,9 +2046,7 @@ class Element(Plugin):
                 'cache': type(self.__artifacts).__name__
             }
 
-            # fail-on-overlap setting cannot affect elements without dependencies
-            if project.fail_on_overlap and dependencies:
-                self.__cache_key_dict['fail-on-overlap'] = True
+            self.__cache_key_dict['fatal-warnings'] = sorted(project._fatal_warnings)
 
         cache_key_dict = self.__cache_key_dict.copy()
         cache_key_dict['dependencies'] = dependencies
diff --git a/buildstream/plugin.py b/buildstream/plugin.py
index 836b60834..64b38ff99 100644
--- a/buildstream/plugin.py
+++ b/buildstream/plugin.py
@@ -166,7 +166,6 @@ class Plugin():
         # Infer the kind identifier
         modulename = type(self).__module__
         self.__kind = modulename.split('.')[-1]
-
         self.debug("Created: {}".format(self))
 
     def __del__(self):
@@ -473,14 +472,28 @@ class Plugin():
         """
         self.__message(MessageType.INFO, brief, detail=detail)
 
-    def warn(self, brief, *, detail=None):
-        """Print a warning message
+    def warn(self, brief, *, detail=None, warning_token=None):
+        """Print a warning message, checks warning_token against project configuration
 
         Args:
            brief (str): The brief message
            detail (str): An optional detailed message, can be multiline output
+           warning_token (str): An optional configurable warning assosciated with this warning,
+                                this will cause PluginError to be raised if this warning is configured as fatal.
+                                (*Since 1.4*)
+
+        Raises:
+           (:class:`.PluginError`): When warning_token is considered fatal by the project configuration
         """
-        self.__message(MessageType.WARN, brief, detail=detail)
+        if warning_token:
+            warning_token = _prefix_warning(self, warning_token)
+            brief = "[{}]: {}".format(warning_token, brief)
+            project = self._get_project()
+
+            if project._warning_is_fatal(warning_token):
+                raise PluginError(message="{}\n{}".format(brief, detail), reason=warning_token)
+
+        self.__message(MessageType.WARN, brief=brief, detail=detail)
 
     def log(self, brief, *, detail=None):
         """Log a message into the plugin's log file
@@ -709,6 +722,32 @@ class Plugin():
             return self.name
 
 
+class CoreWarnings():
+    """CoreWarnings()
+
+    Some common warnings which are raised by core functionalities within BuildStream are found in this class.
+    """
+
+    OVERLAPS = "overlaps"
+    """
+    This warning will be produced when buildstream detects an overlap on an element
+        which is not whitelisted.
+    """
+
+    REF_NOT_IN_TRACK = "ref-not-in-track"
+    """
+    This warning will be produced when a source is configured with a reference
+    which is found to be invalid based on the configured track
+    """
+
+
+__CORE_WARNINGS = [
+    value
+    for name, value in CoreWarnings.__dict__.items()
+    if not name.startswith("__")
+]
+
+
 # Hold on to a lookup table by counter of all instantiated plugins.
 # We use this to send the id back from child processes so we can lookup
 # corresponding element/source in the master process.
@@ -739,6 +778,23 @@ def _plugin_lookup(unique_id):
     return __PLUGINS_TABLE[unique_id]
 
 
+# _prefix_warning():
+#
+# Prefix a warning with the plugin kind. CoreWarnings are not prefixed.
+#
+# Args:
+#   plugin (Plugin): The plugin which raised the warning
+#   warning (str): The warning to prefix
+#
+# Returns:
+#    (str): A prefixed warning
+#
+def _prefix_warning(plugin, warning):
+    if any((warning is core_warning for core_warning in __CORE_WARNINGS)):
+        return warning
+    return "{}:{}".format(plugin.get_kind(), warning)
+
+
 # No need for unregister, WeakValueDictionary() will remove entries
 # in itself when the referenced plugins are garbage collected.
 def _plugin_register(plugin):
diff --git a/tests/cachekey/project/elements/build1.expected b/tests/cachekey/project/elements/build1.expected
index fe515806d..1b41b5e06 100644
--- a/tests/cachekey/project/elements/build1.expected
+++ b/tests/cachekey/project/elements/build1.expected
@@ -1 +1 @@
-594be3eb2a211f706557b156ec4b0ffb3ca256af35bcd0116b97fdb8c942d1c5
\ No newline at end of file
+90fa9c1b5334aac3bfd8956d92af0a91e2f2bbcbcac73861957165e577bc8768
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/build2.expected b/tests/cachekey/project/elements/build2.expected
index f28fe802c..83c2bb1b3 100644
--- a/tests/cachekey/project/elements/build2.expected
+++ b/tests/cachekey/project/elements/build2.expected
@@ -1 +1 @@
-5038d37bf1714180d160271e688ec1715d69666ca266ed5b1d880abdee36b310
\ No newline at end of file
+a62c29fe4e05820412e391430f61aa88a1a82e138de8cac726a9dc4fcd7ed8b9
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose1.expected b/tests/cachekey/project/elements/compose1.expected
index ec0b94943..ee82ebb6a 100644
--- a/tests/cachekey/project/elements/compose1.expected
+++ b/tests/cachekey/project/elements/compose1.expected
@@ -1 +1 @@
-e050172c2d445849a2716720814501f0e294b81d28be8cd911ee294291ec60d0
\ No newline at end of file
+4e4c719242aa45fed398cc2fb8936195a1fcae9326d808de7fee340ae48862ea
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose2.expected b/tests/cachekey/project/elements/compose2.expected
index fd738fb74..f7b2d396b 100644
--- a/tests/cachekey/project/elements/compose2.expected
+++ b/tests/cachekey/project/elements/compose2.expected
@@ -1 +1 @@
-ad985f243163aab22576fea63a0b89f7560e361326cb041872c568c5feaabf5c
\ No newline at end of file
+2fc1dd398f6c6f6e1d7ca48d88557e133d2130278882e14cd1105b15a600cd7a
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose3.expected b/tests/cachekey/project/elements/compose3.expected
index 24899fb04..e6339b175 100644
--- a/tests/cachekey/project/elements/compose3.expected
+++ b/tests/cachekey/project/elements/compose3.expected
@@ -1 +1 @@
-ec0fec2c821eb34286e9799a6e8d1f5587f161d1d653bd1dbe385602340d86ae
\ No newline at end of file
+4d7c9e2e1e8cfcc4b300a11693767f41f22c7829db9063dec10856328d03ccc3
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose4.expected b/tests/cachekey/project/elements/compose4.expected
index 8f7730a28..de299e892 100644
--- a/tests/cachekey/project/elements/compose4.expected
+++ b/tests/cachekey/project/elements/compose4.expected
@@ -1 +1 @@
-7cfe8f6161c00d8cf10114e7458f0b97eb003a41504ae301b24f45d48d42155b
\ No newline at end of file
+cad8f3b622f4a906f9fc3f5187a7703e2b17dfc550dd5a07479ca3ebffbd5c86
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose5.expected b/tests/cachekey/project/elements/compose5.expected
index 9653745ae..d7c4c3ef8 100644
--- a/tests/cachekey/project/elements/compose5.expected
+++ b/tests/cachekey/project/elements/compose5.expected
@@ -1 +1 @@
-7f016c3165f2de9161312b74f29513dff7dfdcba5ff8c6897beb5b123eaafd3d
\ No newline at end of file
+4fd21699827aa16da8d7a1525020f9fd45422f0431749510947ff472d76c1802
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import1.expected b/tests/cachekey/project/elements/import1.expected
index f28d7c036..bc9c1c93f 100644
--- a/tests/cachekey/project/elements/import1.expected
+++ b/tests/cachekey/project/elements/import1.expected
@@ -1 +1 @@
-41ce5a640fdfd7b6ce8a2c3fa1dde7983bc4df0e4c3ca926670118bae3c051fe
\ No newline at end of file
+aa443ea4607d7dd5a0c99646a1b827e3165862772fc1b26e20195aadd2ab8885
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import2.expected b/tests/cachekey/project/elements/import2.expected
index 3ec64e941..1013f5826 100644
--- a/tests/cachekey/project/elements/import2.expected
+++ b/tests/cachekey/project/elements/import2.expected
@@ -1 +1 @@
-d657db503460486a6de80d87f15c6b1fa84b0c4dacabed374acdc70172d4761d
\ No newline at end of file
+18ea6bbb968ca6945c8c2941650f447b8df58673be7270c967c8152730eff036
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import3.expected b/tests/cachekey/project/elements/import3.expected
index 09c525c75..9a8517728 100644
--- a/tests/cachekey/project/elements/import3.expected
+++ b/tests/cachekey/project/elements/import3.expected
@@ -1 +1 @@
-8de0293a6231dc661cf7229aa5a2f25abdf9a6d38ff70bd6f2562dae51ff05d3
\ No newline at end of file
+34ce4816b0307f0691302460367ab24b4e1f86e61c0e307e68bcd6833946c1f1
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/script1.expected b/tests/cachekey/project/elements/script1.expected
index b81856cd6..e36a3df70 100644
--- a/tests/cachekey/project/elements/script1.expected
+++ b/tests/cachekey/project/elements/script1.expected
@@ -1 +1 @@
-d0a6b7d29226b083c404d76d9551a0eee98753058580cd62901f8dfac06ca08d
\ No newline at end of file
+c48922b3d80d36e6d30bed7581aa1473a5e463321b3a19606b603c055d2b4be4
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/bzr1.expected b/tests/cachekey/project/sources/bzr1.expected
index 9b29c17fe..fecc86c74 100644
--- a/tests/cachekey/project/sources/bzr1.expected
+++ b/tests/cachekey/project/sources/bzr1.expected
@@ -1 +1 @@
-51415ebc7d72315c5c7704759025d6a9237e786bfe9c2bda8c51e15840c3470a
\ No newline at end of file
+ee271c8f469cd33330229d8dcc44e26f3480a9f47b55db46f42d1a396a94609e
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git1.expected b/tests/cachekey/project/sources/git1.expected
index 858cc2b7d..eab44b627 100644
--- a/tests/cachekey/project/sources/git1.expected
+++ b/tests/cachekey/project/sources/git1.expected
@@ -1 +1 @@
-ef9bd728a328301e0b819be7109761aacfb4c87092904306d4117c86f30478a4
\ No newline at end of file
+39fdc83c2760589c2577fb859cc617a8fdd7ac4cf113f9d4e5c723d70cae3c09
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git2.expected b/tests/cachekey/project/sources/git2.expected
index 31d291b6c..868439c9b 100644
--- a/tests/cachekey/project/sources/git2.expected
+++ b/tests/cachekey/project/sources/git2.expected
@@ -1 +1 @@
-a818930895e164bd342ab786061f4d521b27a4470791f55cc28732fdf92794de
\ No newline at end of file
+8bac8c7d3b8bbd264083db8e6f3aa8894625af5396bbe62589d1ab726a87cccd
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local1.expected b/tests/cachekey/project/sources/local1.expected
index f28d7c036..bc9c1c93f 100644
--- a/tests/cachekey/project/sources/local1.expected
+++ b/tests/cachekey/project/sources/local1.expected
@@ -1 +1 @@
-41ce5a640fdfd7b6ce8a2c3fa1dde7983bc4df0e4c3ca926670118bae3c051fe
\ No newline at end of file
+aa443ea4607d7dd5a0c99646a1b827e3165862772fc1b26e20195aadd2ab8885
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local2.expected b/tests/cachekey/project/sources/local2.expected
index f1954ac6b..610d3fe5d 100644
--- a/tests/cachekey/project/sources/local2.expected
+++ b/tests/cachekey/project/sources/local2.expected
@@ -1 +1 @@
-bf3ceaa62d472c10ce1e991e5b07a81ddb5206e043d39e60167292910e6bd31e
\ No newline at end of file
+51296c83a0d5989a67f40391afcbf420cbbd76c1e6c07aa43fe2aef2e88941e3
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/ostree1.expected b/tests/cachekey/project/sources/ostree1.expected
index 6e0aa34ff..891bea41b 100644
--- a/tests/cachekey/project/sources/ostree1.expected
+++ b/tests/cachekey/project/sources/ostree1.expected
@@ -1 +1 @@
-c9dcee5ad0822df19984ba68e2a2068266427ee583e3dd3265f85f0515cf7510
\ No newline at end of file
+cd8b506c38c116d6bea8999720a82afb8844453d5ad05385302eabc7d858859c
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch1.expected b/tests/cachekey/project/sources/patch1.expected
index ec306edfd..76c2a2a91 100644
--- a/tests/cachekey/project/sources/patch1.expected
+++ b/tests/cachekey/project/sources/patch1.expected
@@ -1 +1 @@
-96f2cf27bb2145290fc85b8cbeb7193738a6d22e1328c384bca7ba1e8754d0fd
\ No newline at end of file
+bf2fe787df6f263cfd7dbd4aa91909af4186e252da722c3d2e2383533fbc7057
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch2.expected b/tests/cachekey/project/sources/patch2.expected
index 57ee7f0a4..ea97daf87 100644
--- a/tests/cachekey/project/sources/patch2.expected
+++ b/tests/cachekey/project/sources/patch2.expected
@@ -1 +1 @@
-bc7cca687951fbaecb365f39fc9d9d7b0fd509b861c77de71f3639e6f49e25fd
\ No newline at end of file
+8f040542ebb9b1c690af99d2db4ffc0a54cb13868a364af4771d19615b9e3b02
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch3.expected b/tests/cachekey/project/sources/patch3.expected
index 62b7b3bd7..96e7d4a13 100644
--- a/tests/cachekey/project/sources/patch3.expected
+++ b/tests/cachekey/project/sources/patch3.expected
@@ -1 +1 @@
-411a041ef82eb9021b308dbc873328e4cc8774c9b8aa0901ff865764bdf82c51
\ No newline at end of file
+fb3985a6527f2498974ffa418cc6832b716d9862c713d7b8dc1c22df45857ee5
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote1.expected b/tests/cachekey/project/sources/remote1.expected
index 0dfd062aa..84b807b42 100644
--- a/tests/cachekey/project/sources/remote1.expected
+++ b/tests/cachekey/project/sources/remote1.expected
@@ -1 +1 @@
-f51945f0a4fae358c4587669a725417d969f04ee2c3e5c3440e07492b1c6de77
\ No newline at end of file
+1fe04362ce6b1e65a0907749a8b11dd2838b2505d2f0c7fee01c005bd43cd63a
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote2.expected b/tests/cachekey/project/sources/remote2.expected
index 071c443d6..31e849a82 100644
--- a/tests/cachekey/project/sources/remote2.expected
+++ b/tests/cachekey/project/sources/remote2.expected
@@ -1 +1 @@
-f8e6e83e39edc14225e8680852fa1fdb1fc093bdd552e99911006b4329f43293
\ No newline at end of file
+105c814f5c88c72e2681a39e1b01a0f2009342afa2b1c1a21c1cc4a664eced29
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar1.expected b/tests/cachekey/project/sources/tar1.expected
index 07f541a7f..7df0789f7 100644
--- a/tests/cachekey/project/sources/tar1.expected
+++ b/tests/cachekey/project/sources/tar1.expected
@@ -1 +1 @@
-a907dfad1c12f6303e7ed20896e49ba3fd5ef566777e7b47098116ec03e0e1f4
\ No newline at end of file
+29331729ccb0f67507f9b1a315410d794d62bda6d16ee1fabd927a39808265a7
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar2.expected b/tests/cachekey/project/sources/tar2.expected
index 3da7239f3..044625e7d 100644
--- a/tests/cachekey/project/sources/tar2.expected
+++ b/tests/cachekey/project/sources/tar2.expected
@@ -1 +1 @@
-99865afccb0926ba5bbaa24e0ded7d8353b56fe499511ad6a809580d17abd80e
\ No newline at end of file
+37e135a6a6270245ef0fcfda96cada821095f819b57e701f635e83a6d47b83a9
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip1.expected b/tests/cachekey/project/sources/zip1.expected
index e15ed0ac2..c4aca98c4 100644
--- a/tests/cachekey/project/sources/zip1.expected
+++ b/tests/cachekey/project/sources/zip1.expected
@@ -1 +1 @@
-23ac1cc41c6a72214b3e59664fe0ef85d909013befd8afde13cf8877510579e4
\ No newline at end of file
+5834df0bd373aebd3e74fe57534dfbefbad02a0bfc391ea9b67a6c7c63823ba0
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip2.expected b/tests/cachekey/project/sources/zip2.expected
index 5208381c1..220f321d9 100644
--- a/tests/cachekey/project/sources/zip2.expected
+++ b/tests/cachekey/project/sources/zip2.expected
@@ -1 +1 @@
-be26d9222bf53589686861ce21391548dd5d6284fdd003ff8a7e39601d6e8bef
\ No newline at end of file
+7f8bb32b8fd8526c1909fbb239706abd7d1ab96911f17eb9f6e96a6f55833c04
\ No newline at end of file
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
index 232d71098..d6d99895a 100644
--- a/tests/cachekey/project/target.expected
+++ b/tests/cachekey/project/target.expected
@@ -1 +1 @@
-b11fe7a9883cee2846a6148703b53785dd031482b359ca56d1ff22b08a1b5cd2
\ No newline at end of file
+e045df890262f7b7c663e64ad0bfba428d9d80ec514df3ddb13313d4ef669b73
\ No newline at end of file
diff --git a/tests/frontend/overlaps.py b/tests/frontend/overlaps.py
index 36e15acd7..27be8deac 100644
--- a/tests/frontend/overlaps.py
+++ b/tests/frontend/overlaps.py
@@ -3,6 +3,7 @@ import pytest
 from tests.testutils.runcli import cli
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
+from buildstream.plugin import CoreWarnings
 
 # Project directory
 DATA_DIR = os.path.join(
@@ -16,30 +17,35 @@ project_template = {
 }
 
 
-def gen_project(project_dir, fail_on_overlap):
+def gen_project(project_dir, fail_on_overlap, use_fatal_warnings=True):
     template = dict(project_template)
-    template["fail-on-overlap"] = fail_on_overlap
+    if use_fatal_warnings:
+        template["fatal-warnings"] = [CoreWarnings.OVERLAPS] if fail_on_overlap else []
+    else:
+        template["fail-on-overlap"] = fail_on_overlap
     projectfile = os.path.join(project_dir, "project.conf")
     _yaml.dump(template, projectfile)
 
 
 @pytest.mark.datafiles(DATA_DIR)
-def test_overlaps(cli, datafiles):
+@pytest.mark.parametrize("use_fatal_warnings", [True, False])
+def test_overlaps(cli, datafiles, use_fatal_warnings):
     project_dir = str(datafiles)
-    gen_project(project_dir, False)
+    gen_project(project_dir, False, use_fatal_warnings)
     result = cli.run(project=project_dir, silent=True, args=[
         'build', 'collect.bst'])
     result.assert_success()
 
 
 @pytest.mark.datafiles(DATA_DIR)
-def test_overlaps_error(cli, datafiles):
+@pytest.mark.parametrize("use_fatal_warnings", [True, False])
+def test_overlaps_error(cli, datafiles, use_fatal_warnings):
     project_dir = str(datafiles)
-    gen_project(project_dir, True)
+    gen_project(project_dir, True, use_fatal_warnings)
     result = cli.run(project=project_dir, silent=True, args=[
         'build', 'collect.bst'])
     result.assert_main_error(ErrorDomain.STREAM, None)
-    result.assert_task_error(ErrorDomain.ELEMENT, "overlap-error")
+    result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS)
 
 
 @pytest.mark.datafiles(DATA_DIR)
@@ -70,15 +76,16 @@ def test_overlaps_whitelist_on_overlapper(cli, datafiles):
     result = cli.run(project=project_dir, silent=True, args=[
         'build', 'collect-partially-whitelisted.bst'])
     result.assert_main_error(ErrorDomain.STREAM, None)
-    result.assert_task_error(ErrorDomain.ELEMENT, "overlap-error")
+    result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS)
 
 
 @pytest.mark.datafiles(DATA_DIR)
-def test_overlaps_script(cli, datafiles):
+@pytest.mark.parametrize("use_fatal_warnings", [True, False])
+def test_overlaps_script(cli, datafiles, use_fatal_warnings):
     # Test overlaps with script element to test
     # Element.stage_dependency_artifacts() with Scope.RUN
     project_dir = str(datafiles)
-    gen_project(project_dir, False)
+    gen_project(project_dir, False, use_fatal_warnings)
     result = cli.run(project=project_dir, silent=True, args=[
         'build', 'script.bst'])
     result.assert_success()