Add SourceTransform base class

This adds a new base plugin, an alternative to `Source`, that is more suitable
to implement plugins for languages with package-managers.

6 files changed, 154 insertions, 5 deletions

diff --git a/buildstream/__init__.py b/buildstream/__init__.py
index cf56ecfe1..7748d8891 100644
--- a/buildstream/__init__.py
+++ b/buildstream/__init__.py
@@ -33,3 +33,4 @@ if "_BST_COMPLETION" not in os.environ:
     from .element import Element, ElementError, Scope
     from .buildelement import BuildElement
     from .scriptelement import ScriptElement
+    from .sourcetransform import SourceTransform
diff --git a/buildstream/_scheduler/fetchqueue.py b/buildstream/_scheduler/fetchqueue.py
index 24512bddb..632b89aa2 100644
--- a/buildstream/_scheduler/fetchqueue.py
+++ b/buildstream/_scheduler/fetchqueue.py
@@ -39,8 +39,12 @@ class FetchQueue(Queue):
         self._skip_cached = skip_cached
 
     def process(self, element):
+        previous_sources = []
         for source in element.sources():
-            source._fetch()
+            # `previous_sources` is SourceTransform specific and is swallowed
+            # by `**kwargs` in `Source`
+            source._fetch(previous_sources=previous_sources)
+            previous_sources.append(source)
 
     def status(self, element):
         # state of dependencies may have changed, recalculate element state
diff --git a/buildstream/element.py b/buildstream/element.py
index fc21f80b6..ff1b0ce5f 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -96,6 +96,7 @@ from . import _signals
 from . import _site
 from ._platform import Platform
 from .sandbox._config import SandboxConfig
+from .sourcetransform import SourceTransform
 
 
 # _KeyStrength():
@@ -1180,6 +1181,10 @@ class Element(Plugin):
             # Prepend provenance to the error
             raise ElementError("{}: {}".format(self, e), reason=e.reason) from e
 
+        if self.__sources and isinstance(self.__sources[0], SourceTransform):
+            raise ElementError("{}: A SourceTransform plugin can't be the first source of a build element"
+                               .format(self))
+
         # Preflight the sources
         for source in self.sources():
             source._preflight()
@@ -1223,9 +1228,11 @@ class Element(Plugin):
     #
     def _track(self):
         refs = []
-        for source in self.__sources:
+        for index, source in enumerate(self.__sources):
             old_ref = source.get_ref()
-            new_ref = source._track()
+            # `previous_sources` is SourceTransform specific and is swallowed
+            # by `**kwargs` in `Source`
+            new_ref = source._track(previous_sources=self.__sources[0:index])
             refs.append((source._get_unique_id(), new_ref))
 
             # Complimentary warning that the new ref will be unused.
diff --git a/buildstream/source.py b/buildstream/source.py
index ec38ae8f2..124b05389 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -373,7 +373,7 @@ class Source(Plugin):
 
     # Wrapper function around plugin provided fetch method
     #
-    def _fetch(self):
+    def _fetch(self, **kwargs):
         self.fetch()
 
     # Wrapper for stage() api which gives the source
@@ -580,7 +580,7 @@ class Source(Plugin):
 
     # Wrapper for track()
     #
-    def _track(self):
+    def _track(self, **kwargs):
         new_ref = self.track()
         current_ref = self.get_ref()
 
diff --git a/buildstream/sourcetransform.py b/buildstream/sourcetransform.py
new file mode 100644
index 000000000..ccf9218a3
--- /dev/null
+++ b/buildstream/sourcetransform.py
@@ -0,0 +1,136 @@
+#
+#  Copyright Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Antoine Wacheux <awacheux@bloomberg.net>
+#        Gokcen Nurlu <gnurlu1@bloomberg.net>
+"""
+SourceTransform
+===============
+
+
+.. _core_sourcetransform_abstract_methods:
+
+This plugin is a base for package-manager-like source plugins. It can't be the
+first source in an element and it makes use of previously tracked & fetched
+sources.
+
+Abstract Methods
+----------------
+For loading and configuration purposes, SourceTransform based plugins must
+implement the
+:ref:`Plugin base class abstract methods <core_plugin_abstract_methods>` and
+:ref:`Source base class abstract methods <core_source_abstract_methods>`.
+
+Keep in mind that SourceTransform exposes the following abstract methods that
+have function signature different than
+:ref:`their counterparts in Source <core_source_abstract_methods>` and these
+have to be implemented instead.
+
+* :func:`SourceTransform.track(previous_staging_dir) <buildstream.source.SourceTransform.track>`
+
+  Automatically derive a new ref from previously staged sources.
+
+* :func:`SourceTransform.fetch(previous_staging_dir) <buildstream.source.SourceTransform.fetch>`
+
+  Fetch the actual payload for the currently set ref.
+
+"""
+
+from buildstream import Consistency
+from .source import Source
+from . import utils
+from ._exceptions import ImplError
+
+
+class SourceTransform(Source):
+    def __ensure_previous_sources(self, previous_sources):
+        for src in previous_sources:
+            if src.get_consistency() == Consistency.RESOLVED:
+                src._fetch()
+            elif src.get_consistency() == Consistency.INCONSISTENT:
+                new_ref = src._track()
+                src._save_ref(new_ref)
+                src._fetch()
+
+    # Needs explanation
+    def track(self, previous_staging_dir):
+        """Resolve a new ref from the plugin's track option
+
+        Different than :func:`~buildstream.source.Source.track`, implementors
+        have access to previous sources. This one is also mandatory to
+        implement.
+
+        Args:
+           previous_staging_dir (str): Path to a temporary directory where
+                                       previous sources are staged.
+
+        Returns:
+           (simple object): A new internal source reference, or None
+
+        See :func:`~buildstream.source.Source.get_ref` for a discussion on
+        the *ref* parameter.
+        """
+        raise ImplError("SourceTransform plugin '{}' does not implement track()".format(self.get_kind()))
+
+    # Needs explanation
+    def fetch(self, previous_staging_dir):
+        """Fetch remote sources and mirror them locally, ensuring at least
+        that the specific reference is cached locally.
+
+        Different than :func:`~buildstream.source.Source.fetch`, implementors
+        have access to previous sources.
+
+        Args:
+           previous_staging_dir (str): Path to a temporary directory where
+                                       previous sources are staged.
+
+        Raises:
+           :class:`.SourceError`
+
+        Implementors should raise :class:`.SourceError` if the there is some
+        network error or if the source reference could not be matched.
+        """
+        raise ImplError("SourceTransform plugin '{}' does not implement fetch()".format(self.get_kind()))
+
+    def _track(self, previous_sources):
+        self.__ensure_previous_sources(previous_sources)
+
+        with utils._tempdir(suffix="tracking") as staging_directory:
+            for src in previous_sources:
+                src._stage(staging_directory)
+
+            # Rest is same with Source._track(), but calling a different .track
+            new_ref = self.track(staging_directory)
+            current_ref = self.get_ref()
+
+            if new_ref is None:
+                # No tracking, keep current ref
+                new_ref = current_ref
+
+            if current_ref != new_ref:
+                self.info("Found new revision: {}".format(new_ref))
+
+            return new_ref
+
+    def _fetch(self, previous_sources):
+        self.__ensure_previous_sources(previous_sources)
+
+        with utils._tempdir(suffix="fetch") as staging_directory:
+            for src in previous_sources:
+                src._stage(staging_directory)
+
+            self.fetch(staging_directory)
diff --git a/doc/source/core_framework.rst b/doc/source/core_framework.rst
index c3b84a9b1..098df1e64 100644
--- a/doc/source/core_framework.rst
+++ b/doc/source/core_framework.rst
@@ -14,6 +14,7 @@ useful for working on BuildStream itself.
 
    buildstream.plugin
    buildstream.source
+   buildstream.sourcetransform
    buildstream.element
    buildstream.buildelement
    buildstream.scriptelement