Load and save junctioned source refs from/to junction.refs

  o _projectrefs.py: Additional constructor option to choose the base name

  o _project.py: Load two ProjectRefs objects, one for the junctions

  o source.py: Load and save junctioned source refs with the appropriate ProjectRefs object

  o tests: Updated some tests to expect junctions to be stored in junction.refs

This fixes issue #361

6 files changed, 40 insertions, 13 deletions

diff --git a/buildstream/_project.py b/buildstream/_project.py
index 745289c2b..d4afd7712 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -80,8 +80,11 @@ class Project():
         # Absolute path to where elements are loaded from within the project
         self.element_path = None
 
+        # ProjectRefs for the main refs and also for junctions
+        self.refs = ProjectRefs(self.directory, 'project.refs')
+        self.junction_refs = ProjectRefs(self.directory, 'junction.refs')
+
         self.workspaces = None                   # Workspaces
-        self.refs = ProjectRefs(self.directory)  # ProjectRefs
         self.options = None                      # OptionPool
         self.junction = junction                 # The junction Element object, if this is a subproject
         self.fail_on_overlap = False             # Whether overlaps are treated as errors
@@ -382,6 +385,7 @@ class Project():
         # Load project.refs if it exists, this may be ignored.
         if self.ref_storage == ProjectRefStorage.PROJECT_REFS:
             self.refs.load(self.options)
+            self.junction_refs.load(self.options)
 
         # Parse shell options
         shell_options = _yaml.node_get(config, Mapping, 'shell')
diff --git a/buildstream/_projectrefs.py b/buildstream/_projectrefs.py
index a555615b0..83dd2619b 100644
--- a/buildstream/_projectrefs.py
+++ b/buildstream/_projectrefs.py
@@ -41,11 +41,16 @@ class ProjectRefStorage():
 #
 # The project.refs file management
 #
+# Args:
+#    directory (str): The project directory
+#    base_name (str): The project.refs basename
+#
 class ProjectRefs():
 
-    def __init__(self, directory):
+    def __init__(self, directory, base_name):
         directory = os.path.abspath(directory)
-        self._fullpath = os.path.join(directory, "project.refs")
+        self._fullpath = os.path.join(directory, base_name)
+        self._base_name = base_name
         self._toplevel_node = None
         self._toplevel_save = None
 
@@ -59,7 +64,7 @@ class ProjectRefs():
     def load(self, options):
 
         try:
-            self._toplevel_node = _yaml.load(self._fullpath, shortname='project.refs', copy_tree=True)
+            self._toplevel_node = _yaml.load(self._fullpath, shortname=self._base_name, copy_tree=True)
             provenance = _yaml.node_get_provenance(self._toplevel_node)
             self._toplevel_save = provenance.toplevel
 
diff --git a/buildstream/source.py b/buildstream/source.py
index 9af17210e..fa547d641 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -132,6 +132,7 @@ class Source(Plugin):
 
         self.__element_name = meta.element_name         # The name of the element owning this source
         self.__element_index = meta.element_index       # The index of the source in the owning element's source list
+        self.__element_kind = meta.element_kind         # The kind of the element owning this source
         self.__directory = meta.directory               # Staging relative directory
         self.__consistency = Consistency.INCONSISTENT   # Cached consistency state
 
@@ -425,6 +426,20 @@ class Source(Plugin):
 
         return changed
 
+    # _project_refs():
+    #
+    # Gets the appropriate ProjectRefs object for this source,
+    # which depends on whether the owning element is a junction
+    #
+    # Args:
+    #    project (Project): The project to check
+    #
+    def _project_refs(self, project):
+        element_kind = self.__element_kind
+        if element_kind == 'junction':
+            return project.junction_refs
+        return project.refs
+
     # _load_ref():
     #
     # Loads the ref for the said source.
@@ -462,7 +477,8 @@ class Source(Plugin):
 
         # If the main project overrides the ref, use the override
         if project is not toplevel and toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS:
-            ref_node = toplevel.refs.lookup_ref(project.name, element_name, element_idx)
+            refs = self._project_refs(toplevel)
+            ref_node = refs.lookup_ref(project.name, element_name, element_idx)
             if ref_node is not None:
                 do_load_ref(ref_node)
 
@@ -479,7 +495,8 @@ class Source(Plugin):
                 self.set_ref(None, {})
 
             # Try to load the ref
-            ref_node = project.refs.lookup_ref(project.name, element_name, element_idx)
+            refs = self._project_refs(project)
+            ref_node = refs.lookup_ref(project.name, element_name, element_idx)
             if ref_node is not None:
                 do_load_ref(ref_node)
 
@@ -505,6 +522,7 @@ class Source(Plugin):
         context = self._get_context()
         project = self._get_project()
         toplevel = context.get_toplevel_project()
+        toplevel_refs = self._project_refs(toplevel)
         provenance = self._get_provenance()
 
         element_name = self.__element_name
@@ -515,12 +533,12 @@ class Source(Plugin):
         #
         if project is toplevel:
             if toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS:
-                node = toplevel.refs.lookup_ref(project.name, element_name, element_idx, write=True)
+                node = toplevel_refs.lookup_ref(project.name, element_name, element_idx, write=True)
             else:
                 node = provenance.node
         else:
             if toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS:
-                node = toplevel.refs.lookup_ref(project.name, element_name, element_idx, write=True)
+                node = toplevel_refs.lookup_ref(project.name, element_name, element_idx, write=True)
             else:
                 node = {}
 
@@ -542,7 +560,7 @@ class Source(Plugin):
         #
         if project is toplevel:
             if toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS:
-                do_save_refs(toplevel.refs)
+                do_save_refs(toplevel_refs)
             else:
                 # Save the ref in the originating file
                 #
@@ -555,7 +573,7 @@ class Source(Plugin):
                                       reason="save-ref-error") from e
         else:
             if toplevel.ref_storage == ProjectRefStorage.PROJECT_REFS:
-                do_save_refs(toplevel.refs)
+                do_save_refs(toplevel_refs)
             else:
                 self.warn("{}: Not persisting new reference in junctioned project".format(self))
 
diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py
index ba56fc00c..45ef33bae 100644
--- a/tests/frontend/buildcheckout.py
+++ b/tests/frontend/buildcheckout.py
@@ -272,7 +272,7 @@ def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage):
                 }
             }
         }
-        _yaml.dump(project_refs, os.path.join(project, 'project.refs'))
+        _yaml.dump(project_refs, os.path.join(project, 'junction.refs'))
 
     # Now try to build it, this should automatically result in fetching
     # the junction itself at load time.
diff --git a/tests/frontend/fetch.py b/tests/frontend/fetch.py
index 261e0b22e..e074dadae 100644
--- a/tests/frontend/fetch.py
+++ b/tests/frontend/fetch.py
@@ -117,7 +117,7 @@ def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage):
                 }
             }
         }
-        _yaml.dump(project_refs, os.path.join(project, 'project.refs'))
+        _yaml.dump(project_refs, os.path.join(project, 'junction.refs'))
 
     # Now try to fetch it, this should automatically result in fetching
     # the junction itself.
diff --git a/tests/frontend/show.py b/tests/frontend/show.py
index f515fc44d..719dadbf4 100644
--- a/tests/frontend/show.py
+++ b/tests/frontend/show.py
@@ -151,7 +151,7 @@ def test_unfetched_junction(cli, tmpdir, datafiles, ref_storage):
                 }
             }
         }
-        _yaml.dump(project_refs, os.path.join(project, 'project.refs'))
+        _yaml.dump(project_refs, os.path.join(project, 'junction.refs'))
 
     # Assert the correct error when trying to show the pipeline
     result = cli.run(project=project, silent=True, args=[