_gitsourcebase.py: Manage submodules recursively

Previously, GitSourceBase would only consider immediate submodules of the
superproject. It now fetches and stages recursively.

To achieve this, this commit somewhat refactors the relationship between
GitMirror and GitSourceBase. Enumerating GitMirrors for the submodules is now
done in GitMirror itself. GitSourceBase recursively iterates these mirror
classes with _recurse_submodules and applies the source configuration with
_configure_submodules.

1 files changed, 65 insertions, 56 deletions

diff --git a/src/buildstream/_gitsourcebase.py b/src/buildstream/_gitsourcebase.py
index 9365137e1..082c49177 100644
--- a/src/buildstream/_gitsourcebase.py
+++ b/src/buildstream/_gitsourcebase.py
@@ -74,7 +74,6 @@ class _GitMirror(SourceFetcher):
         self.tags = tags
         self.primary = primary
         self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url))
-        self.mark_download_url(url)
 
     # Ensures that the mirror exists
     def ensure(self, alias_override=None):
@@ -255,6 +254,19 @@ class _GitMirror(SourceFetcher):
             cwd=fullpath,
         )
 
+    # get_submodule_mirrors():
+    #
+    # Returns:
+    #     An iterator through new instances of this class, one of each submodule
+    #     in the repo
+    #
+    def get_submodule_mirrors(self):
+        for path, url in self.submodule_list():
+            ref = self.submodule_ref(path)
+            if ref is not None:
+                mirror = self.__class__(self.source, os.path.join(self.path, path), url, ref)
+                yield mirror
+
     # List the submodules (path/url tuples) present at the given ref of this repo
     def submodule_list(self):
         modules = "{}:{}".format(self.ref, GIT_MODULES)
@@ -456,7 +468,6 @@ class _GitSourceBase(Source):
             )
 
         self.checkout_submodules = node.get_bool("checkout-submodules", default=True)
-        self.submodules = []
 
         # Parse a dict of submodule overrides, stored in the submodule_overrides
         # and submodule_checkout_overrides dictionaries.
@@ -565,34 +576,27 @@ class _GitSourceBase(Source):
         return ret
 
     def init_workspace(self, directory):
-        # XXX: may wish to refactor this as some code dupe with stage()
-        self._refresh_submodules()
-
         with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True):
             self.mirror.init_workspace(directory)
-            for mirror in self.submodules:
+            for mirror in self._recurse_submodules(configure=True):
                 mirror.init_workspace(directory)
 
     def stage(self, directory):
-
-        # Need to refresh submodule list here again, because
-        # it's possible that we did not load in the main process
-        # with submodules present (source needed fetching) and
-        # we may not know about the submodule yet come time to build.
-        #
-        self._refresh_submodules()
-
         # Stage the main repo in the specified directory
         #
         with self.timed_activity("Staging {}".format(self.mirror.url), silent_nested=True):
             self.mirror.stage(directory)
-            for mirror in self.submodules:
+            for mirror in self._recurse_submodules(configure=True):
                 mirror.stage(directory)
 
     def get_source_fetchers(self):
+        self.mirror.mark_download_url(self.mirror.url)
         yield self.mirror
-        self._refresh_submodules()
-        for submodule in self.submodules:
+        # _recurse_submodules only iterates those which are known at the current
+        # cached state - but fetch is called on each result as we go, so this will
+        # yield all configured submodules
+        for submodule in self._recurse_submodules(configure=True):
+            submodule.mark_download_url(submodule.url)
             yield submodule
 
     def validate_cache(self):
@@ -600,14 +604,13 @@ class _GitSourceBase(Source):
         unlisted_submodules = []
         invalid_submodules = []
 
-        for path, url in self.mirror.submodule_list():
-            discovered_submodules[path] = url
-            if self._ignore_submodule(path):
+        for submodule in self._recurse_submodules(configure=False):
+            discovered_submodules[submodule.path] = submodule.url
+            if self._ignore_submodule(submodule.path):
                 continue
 
-            override_url = self.submodule_overrides.get(path)
-            if not override_url:
-                unlisted_submodules.append((path, url))
+            if submodule.path not in self.submodule_overrides:
+                unlisted_submodules.append((submodule.path, submodule.url))
 
         # Warn about submodules which are explicitly configured but do not exist
         for path, url in self.submodule_overrides.items():
@@ -679,44 +682,50 @@ class _GitSourceBase(Source):
     ###########################################################
 
     def _have_all_refs(self):
-        if not self.mirror.has_ref():
-            return False
-
-        self._refresh_submodules()
-        for mirror in self.submodules:
-            if not os.path.exists(mirror.mirror):
-                return False
-            if not mirror.has_ref():
-                return False
-
-        return True
+        return self.mirror.has_ref() and all(
+            submodule.has_ref() for submodule in self._recurse_submodules(configure=True)
+        )
 
-    # Refreshes the BST_MIRROR_CLASS objects for submodules
+    # _configure_submodules():
     #
-    # Assumes that we have our mirror and we have the ref which we point to
+    # Args:
+    #     submodules: An iterator of _GitMirror (or similar) objects for submodules
     #
-    def _refresh_submodules(self):
-        self.mirror.ensure()
-        submodules = []
-
-        for path, url in self.mirror.submodule_list():
-
-            # Completely ignore submodules which are disabled for checkout
-            if self._ignore_submodule(path):
+    # Returns:
+    #     An iterator through `submodules` but filtered of any ignored submodules
+    #     and modified to use any custom URLs configured in the source
+    #
+    def _configure_submodules(self, submodules):
+        for submodule in submodules:
+            if self._ignore_submodule(submodule.path):
                 continue
+            # Allow configuration to override the upstream location of the submodules.
+            submodule.url = self.submodule_overrides.get(submodule.path, submodule.url)
+            yield submodule
 
-            # Allow configuration to override the upstream
-            # location of the submodules.
-            override_url = self.submodule_overrides.get(path)
-            if override_url:
-                url = override_url
-
-            ref = self.mirror.submodule_ref(path)
-            if ref is not None:
-                mirror = self.BST_MIRROR_CLASS(self, path, url, ref)
-                submodules.append(mirror)
-
-        self.submodules = submodules
+    # _recurse_submodules():
+    #
+    # Recursively iterates through GitMirrors for submodules of the main repo. Only
+    # submodules that are cached are recursed into - but this is decided at
+    # iteration time, so you can fetch in a for loop over this function to fetch
+    # all submodules.
+    #
+    # Args:
+    #     configure (bool): Whether to apply the 'submodule' config while recursing
+    #                       (URL changing and 'checkout' overrides)
+    #
+    def _recurse_submodules(self, configure):
+        def recurse(mirror):
+            submodules = mirror.get_submodule_mirrors()
+            if configure:
+                submodules = self._configure_submodules(submodules)
+
+            for submodule in submodules:
+                yield submodule
+                if submodule.has_ref():
+                    yield from recurse(submodule)
+
+        yield from recurse(self.mirror)
 
     def _load_tags(self, node):
         tags = []