Track of git tags and save them to reproduce minimum shallow repository

Instead of tag information being fetched which can change with time,
they are tracked and saved in the projects.refs/.bst. Then we re-tag
automatically the closest tag so that `git describe` works and is
reproducible.

8 files changed, 354 insertions, 15 deletions

diff --git a/buildstream/_versions.py b/buildstream/_versions.py
index 842ac8bf8..42dcb1a31 100644
--- a/buildstream/_versions.py
+++ b/buildstream/_versions.py
@@ -23,7 +23,7 @@
 # This version is bumped whenever enhancements are made
 # to the `project.conf` format or the core element format.
 #
-BST_FORMAT_VERSION = 18
+BST_FORMAT_VERSION = 19
 
 
 # The base BuildStream artifact version
diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py
index 8586ebcf8..bb42a1b98 100644
--- a/buildstream/plugins/sources/git.py
+++ b/buildstream/plugins/sources/git.py
@@ -76,6 +76,24 @@ git - stage files from a git repository
        url: upstream:baz.git
        checkout: False
 
+   # Enable tag tracking. This allows to create a dummy shallow
+   # repository with necessary tag informations required to run 'git
+   # describe'. Default is 'False'.
+   track-tags: True
+
+   # A dummy shallow repository is created with the following tags.
+   # Tags should not be edited by hand but rather be populated by
+   # 'bst track' with 'track-tags' set to True. These tags can
+   # also be stored in 'project.refs'. If you do not want to track
+   # the 'ref' at the same time, set 'track' to the wanted commit.
+   tags:
+   - tag: lightweight-example
+     commit: 04ad0dc656cb7cc6feb781aa13bdbf1d67d0af78
+     annotated: false
+   - tag: annotated-example
+     commit: 10abe77fe8d77385d86f225b503d9185f4ef7f3a
+     annotated: true
+
 See :ref:`built-in functionality doumentation <core_source_builtins>` for
 details on common configuration options for sources.
 
@@ -95,6 +113,7 @@ import re
 import shutil
 from collections.abc import Mapping
 from io import StringIO
+from tempfile import TemporaryFile
 
 from configparser import RawConfigParser
 
@@ -115,13 +134,14 @@ INCONSISTENT_SUBMODULE = "inconsistent-submodules"
 #
 class GitMirror(SourceFetcher):
 
-    def __init__(self, source, path, url, ref, *, primary=False):
+    def __init__(self, source, path, url, ref, *, primary=False, tags=[]):
 
         super().__init__()
         self.source = source
         self.path = path
         self.url = url
         self.ref = ref
+        self.tags = tags
         self.primary = primary
         self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url))
         self.mark_download_url(url)
@@ -214,7 +234,7 @@ class GitMirror(SourceFetcher):
             raise SourceError("{}: expected ref '{}' was not found in git repository: '{}'"
                               .format(self.source, self.ref, self.url))
 
-    def latest_commit(self, tracking):
+    def latest_commit_with_tags(self, tracking, track_tags=False):
         _, output = self.source.check_output(
             [self.source.host_git, 'rev-parse', tracking],
             fail="Unable to find commit for specified branch name '{}'".format(tracking),
@@ -230,7 +250,28 @@ class GitMirror(SourceFetcher):
             if exit_code == 0:
                 ref = output.rstrip('\n')
 
-        return ref
+        if not track_tags:
+            return ref, []
+
+        tags = set()
+        for options in [[], ['--first-parent'], ['--tags'], ['--tags', '--first-parent']]:
+            exit_code, output = self.source.check_output(
+                [self.source.host_git, 'describe', '--abbrev=0', ref] + options,
+                cwd=self.mirror)
+            if exit_code == 0:
+                tag = output.strip()
+                _, commit_ref = self.source.check_output(
+                    [self.source.host_git, 'rev-parse', tag + '^{commit}'],
+                    fail="Unable to resolve tag '{}'".format(tag),
+                    cwd=self.mirror)
+                exit_code = self.source.call(
+                    [self.source.host_git, 'cat-file', 'tag', tag],
+                    cwd=self.mirror)
+                annotated = (exit_code == 0)
+
+                tags.add((tag, commit_ref.strip(), annotated))
+
+        return ref, list(tags)
 
     def stage(self, directory, track=None):
         fullpath = os.path.join(directory, self.path)
@@ -246,13 +287,15 @@ class GitMirror(SourceFetcher):
                          fail="Failed to checkout git ref {}".format(self.ref),
                          cwd=fullpath)
 
+        # Remove .git dir
+        shutil.rmtree(os.path.join(fullpath, ".git"))
+
+        self._rebuild_git(fullpath)
+
         # Check that the user specified ref exists in the track if provided & not already tracked
         if track:
             self.assert_ref_in_track(fullpath, track)
 
-        # Remove .git dir
-        shutil.rmtree(os.path.join(fullpath, ".git"))
-
     def init_workspace(self, directory, track=None):
         fullpath = os.path.join(directory, self.path)
         url = self.source.translate_url(self.url)
@@ -359,6 +402,78 @@ class GitMirror(SourceFetcher):
                          .format(self.source, self.ref, track, self.url),
                          detail=detail, warning_token=CoreWarnings.REF_NOT_IN_TRACK)
 
+    def _rebuild_git(self, fullpath):
+        if not self.tags:
+            return
+
+        with self.source.tempdir() as tmpdir:
+            included = set()
+            shallow = set()
+            for _, commit_ref, _ in self.tags:
+
+                _, out = self.source.check_output([self.source.host_git, 'rev-list',
+                                                   '--boundary', '{}..{}'.format(commit_ref, self.ref)],
+                                                  fail="Failed to get git history {}..{} in directory: {}"
+                                                  .format(commit_ref, self.ref, fullpath),
+                                                  fail_temporarily=True,
+                                                  cwd=self.mirror)
+                for line in out.splitlines():
+                    rev = line.lstrip('-')
+                    if line[0] == '-':
+                        shallow.add(rev)
+                    else:
+                        included.add(rev)
+
+            shallow -= included
+            included |= shallow
+
+            self.source.call([self.source.host_git, 'init'],
+                             fail="Cannot initialize git repository: {}".format(fullpath),
+                             cwd=fullpath)
+
+            for rev in included:
+                with TemporaryFile(dir=tmpdir) as commit_file:
+                    self.source.call([self.source.host_git, 'cat-file', 'commit', rev],
+                                     stdout=commit_file,
+                                     fail="Failed to get commit {}".format(rev),
+                                     cwd=self.mirror)
+                    commit_file.seek(0, 0)
+                    self.source.call([self.source.host_git, 'hash-object', '-w', '-t', 'commit', '--stdin'],
+                                     stdin=commit_file,
+                                     fail="Failed to add commit object {}".format(rev),
+                                     cwd=fullpath)
+
+            with open(os.path.join(fullpath, '.git', 'shallow'), 'w') as shallow_file:
+                for rev in shallow:
+                    shallow_file.write('{}\n'.format(rev))
+
+            for tag, commit_ref, annotated in self.tags:
+                if annotated:
+                    with TemporaryFile(dir=tmpdir) as tag_file:
+                        tag_data = 'object {}\ntype commit\ntag {}\n'.format(commit_ref, tag)
+                        tag_file.write(tag_data.encode('ascii'))
+                        tag_file.seek(0, 0)
+                        _, tag_ref = self.source.check_output(
+                            [self.source.host_git, 'hash-object', '-w', '-t',
+                             'tag', '--stdin'],
+                            stdin=tag_file,
+                            fail="Failed to add tag object {}".format(tag),
+                            cwd=fullpath)
+
+                    self.source.call([self.source.host_git, 'tag', tag, tag_ref.strip()],
+                                     fail="Failed to tag: {}".format(tag),
+                                     cwd=fullpath)
+                else:
+                    self.source.call([self.source.host_git, 'tag', tag, commit_ref],
+                                     fail="Failed to tag: {}".format(tag),
+                                     cwd=fullpath)
+
+            with open(os.path.join(fullpath, '.git', 'HEAD'), 'w') as head:
+                self.source.call([self.source.host_git, 'rev-parse', self.ref],
+                                 stdout=head,
+                                 fail="Failed to parse commit {}".format(self.ref),
+                                 cwd=self.mirror)
+
 
 class GitSource(Source):
     # pylint: disable=attribute-defined-outside-init
@@ -366,11 +481,20 @@ class GitSource(Source):
     def configure(self, node):
         ref = self.node_get_member(node, str, 'ref', None)
 
-        config_keys = ['url', 'track', 'ref', 'submodules', 'checkout-submodules', 'ref-format']
+        config_keys = ['url', 'track', 'ref', 'submodules',
+                       'checkout-submodules', 'ref-format',
+                       'track-tags', 'tags']
         self.node_validate(node, config_keys + Source.COMMON_CONFIG_KEYS)
 
+        tags_node = self.node_get_member(node, list, 'tags', [])
+        for tag_node in tags_node:
+            self.node_validate(tag_node, ['tag', 'commit', 'annotated'])
+
+        tags = self._load_tags(node)
+        self.track_tags = self.node_get_member(node, bool, 'track-tags', False)
+
         self.original_url = self.node_get_member(node, str, 'url')
-        self.mirror = GitMirror(self, '', self.original_url, ref, primary=True)
+        self.mirror = GitMirror(self, '', self.original_url, ref, tags=tags, primary=True)
         self.tracking = self.node_get_member(node, str, 'track', None)
 
         self.ref_format = self.node_get_member(node, str, 'ref-format', 'sha1')
@@ -417,6 +541,9 @@ class GitSource(Source):
         # the ref, if the user changes the alias to fetch the same sources
         # from another location, it should not affect the cache key.
         key = [self.original_url, self.mirror.ref]
+        if self.mirror.tags:
+            tags = {tag: (commit, annotated) for tag, commit, annotated in self.mirror.tags}
+            key.append({'tags': tags})
 
         # Only modify the cache key with checkout_submodules if it's something
         # other than the default behaviour.
@@ -442,12 +569,33 @@ class GitSource(Source):
 
     def load_ref(self, node):
         self.mirror.ref = self.node_get_member(node, str, 'ref', None)
+        self.mirror.tags = self._load_tags(node)
 
     def get_ref(self):
-        return self.mirror.ref
-
-    def set_ref(self, ref, node):
-        node['ref'] = self.mirror.ref = ref
+        return self.mirror.ref, self.mirror.tags
+
+    def set_ref(self, ref_data, node):
+        if not ref_data:
+            self.mirror.ref = None
+            if 'ref' in node:
+                del node['ref']
+            self.mirror.tags = []
+            if 'tags' in node:
+                del node['tags']
+        else:
+            ref, tags = ref_data
+            node['ref'] = self.mirror.ref = ref
+            self.mirror.tags = tags
+            if tags:
+                node['tags'] = []
+                for tag, commit_ref, annotated in tags:
+                    data = {'tag': tag,
+                            'commit': commit_ref,
+                            'annotated': annotated}
+                    node['tags'].append(data)
+            else:
+                if 'tags' in node:
+                    del node['tags']
 
     def track(self):
 
@@ -470,7 +618,7 @@ class GitSource(Source):
             self.mirror._fetch()
 
             # Update self.mirror.ref and node.ref from the self.tracking branch
-            ret = self.mirror.latest_commit(self.tracking)
+            ret = self.mirror.latest_commit_with_tags(self.tracking, self.track_tags)
 
         # Set tracked attribute, parameter for if self.mirror.assert_ref_in_track is needed
         self.tracked = True
@@ -556,6 +704,16 @@ class GitSource(Source):
 
         self.submodules = submodules
 
+    def _load_tags(self, node):
+        tags = []
+        tags_node = self.node_get_member(node, list, 'tags', [])
+        for tag_node in tags_node:
+            tag = self.node_get_member(tag_node, str, 'tag')
+            commit_ref = self.node_get_member(tag_node, str, 'commit')
+            annotated = self.node_get_member(tag_node, bool, 'annotated')
+            tags.append((tag, commit_ref, annotated))
+        return tags
+
 
 # Plugin entry point
 def setup():
diff --git a/tests/cachekey/project/sources/git3.bst b/tests/cachekey/project/sources/git3.bst
new file mode 100644
index 000000000..b331a3af3
--- /dev/null
+++ b/tests/cachekey/project/sources/git3.bst
@@ -0,0 +1,12 @@
+kind: import
+sources:
+- kind: git
+  url: https://example.com/git/repo.git
+  ref: 6ac68af3e80b7b17c23a3c65233043550a7fa685
+  tags:
+  - tag: lightweight
+    commit: 0a3917d57477ee9afe7be49a0e8a76f56d176df1
+    annotated: false
+  - tag: annotated
+    commit: 68c7f0bd386684742c41ec2a54ce2325e3922f6c
+    annotated: true
diff --git a/tests/cachekey/project/sources/git3.expected b/tests/cachekey/project/sources/git3.expected
new file mode 100644
index 000000000..b383ccbfc
--- /dev/null
+++ b/tests/cachekey/project/sources/git3.expected
@@ -0,0 +1 @@
+6a25f539bd8629a36399c58efd2f5c9c117feb845076a37dc321b55d456932b6
\ No newline at end of file
diff --git a/tests/cachekey/project/target.bst b/tests/cachekey/project/target.bst
index d96645da8..8c5c12723 100644
--- a/tests/cachekey/project/target.bst
+++ b/tests/cachekey/project/target.bst
@@ -7,6 +7,7 @@ depends:
 - sources/bzr1.bst
 - sources/git1.bst
 - sources/git2.bst
+- sources/git3.bst
 - sources/local1.bst
 - sources/local2.bst
 - sources/ostree1.bst
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
index 640133e23..0c89af6bb 100644
--- a/tests/cachekey/project/target.expected
+++ b/tests/cachekey/project/target.expected
@@ -1 +1 @@
-125d9e7dcf4f49e5f80d85b7f144b43ed43186064afc2e596e57f26cce679cf5
\ No newline at end of file
+bc99c288f855ac2619787f0067223f7812d2e10a9d2c7f2bf47de7113c0fd25c
\ No newline at end of file
diff --git a/tests/sources/git.py b/tests/sources/git.py
index 7ab32a6b5..462200f97 100644
--- a/tests/sources/git.py
+++ b/tests/sources/git.py
@@ -22,6 +22,7 @@
 
 import os
 import pytest
+import subprocess
 
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
@@ -523,3 +524,155 @@ def test_track_fetch(cli, tmpdir, datafiles, ref_format, tag, extra_commit):
     # Fetch it
     result = cli.run(project=project, args=['fetch', 'target.bst'])
     result.assert_success()
+
+
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
+@pytest.mark.parametrize("tag_type", [('annotated'), ('lightweight')])
+def test_git_describe(cli, tmpdir, datafiles, ref_storage, tag_type):
+    project = str(datafiles)
+
+    project_config = _yaml.load(os.path.join(project, 'project.conf'))
+    project_config['ref-storage'] = ref_storage
+    _yaml.dump(_yaml.node_sanitize(project_config), os.path.join(project, 'project.conf'))
+
+    repofiles = os.path.join(str(tmpdir), 'repofiles')
+    os.makedirs(repofiles, exist_ok=True)
+    file0 = os.path.join(repofiles, 'file0')
+    with open(file0, 'w') as f:
+        f.write('test\n')
+
+    repo = create_repo('git', str(tmpdir))
+
+    def tag(name):
+        if tag_type == 'annotated':
+            repo.add_annotated_tag(name, name)
+        else:
+            repo.add_tag(name)
+
+    ref = repo.create(repofiles)
+    tag('uselesstag')
+
+    file1 = os.path.join(str(tmpdir), 'file1')
+    with open(file1, 'w') as f:
+        f.write('test\n')
+    repo.add_file(file1)
+    tag('tag1')
+
+    file2 = os.path.join(str(tmpdir), 'file2')
+    with open(file2, 'w') as f:
+        f.write('test\n')
+    repo.branch('branch2')
+    repo.add_file(file2)
+    tag('tag2')
+
+    repo.checkout('master')
+    file3 = os.path.join(str(tmpdir), 'file3')
+    with open(file3, 'w') as f:
+        f.write('test\n')
+    repo.add_file(file3)
+
+    repo.merge('branch2')
+
+    config = repo.source_config()
+    config['track'] = repo.latest_commit()
+    config['track-tags'] = True
+
+    # Write out our test target
+    element = {
+        'kind': 'import',
+        'sources': [
+            config
+        ],
+    }
+    element_path = os.path.join(project, 'target.bst')
+    _yaml.dump(element, element_path)
+
+    if ref_storage == 'inline':
+        result = cli.run(project=project, args=['track', 'target.bst'])
+        result.assert_success()
+    else:
+        result = cli.run(project=project, args=['track', 'target.bst', '--deps', 'all'])
+        result.assert_success()
+
+    if ref_storage == 'inline':
+        element = _yaml.load(element_path)
+        tags = _yaml.node_sanitize(element['sources'][0]['tags'])
+        assert len(tags) == 2
+        for tag in tags:
+            assert 'tag' in tag
+            assert 'commit' in tag
+            assert 'annotated' in tag
+            assert tag['annotated'] == (tag_type == 'annotated')
+
+        assert set([(tag['tag'], tag['commit']) for tag in tags]) == set([('tag1', repo.rev_parse('tag1^{commit}')),
+                                                                          ('tag2', repo.rev_parse('tag2^{commit}'))])
+
+    checkout = os.path.join(str(tmpdir), 'checkout')
+
+    result = cli.run(project=project, args=['build', 'target.bst'])
+    result.assert_success()
+    result = cli.run(project=project, args=['checkout', 'target.bst', checkout])
+    result.assert_success()
+
+    if tag_type == 'annotated':
+        options = []
+    else:
+        options = ['--tags']
+    describe = subprocess.check_output(['git', 'describe'] + options,
+                                       cwd=checkout).decode('ascii')
+    assert describe.startswith('tag2-2-')
+
+    describe_fp = subprocess.check_output(['git', 'describe', '--first-parent'] + options,
+                                          cwd=checkout).decode('ascii')
+    assert describe_fp.startswith('tag1-2-')
+
+    tags = subprocess.check_output(['git', 'tag'],
+                                   cwd=checkout).decode('ascii')
+    tags = set(tags.splitlines())
+    assert tags == set(['tag1', 'tag2'])
+
+    p = subprocess.run(['git', 'log', repo.rev_parse('uselesstag')],
+                       cwd=checkout)
+    assert p.returncode != 0
+
+
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+def test_default_do_not_track_tags(cli, tmpdir, datafiles):
+    project = str(datafiles)
+
+    project_config = _yaml.load(os.path.join(project, 'project.conf'))
+    project_config['ref-storage'] = 'inline'
+    _yaml.dump(_yaml.node_sanitize(project_config), os.path.join(project, 'project.conf'))
+
+    repofiles = os.path.join(str(tmpdir), 'repofiles')
+    os.makedirs(repofiles, exist_ok=True)
+    file0 = os.path.join(repofiles, 'file0')
+    with open(file0, 'w') as f:
+        f.write('test\n')
+
+    repo = create_repo('git', str(tmpdir))
+
+    ref = repo.create(repofiles)
+    repo.add_tag('tag')
+
+    config = repo.source_config()
+    config['track'] = repo.latest_commit()
+
+    # Write out our test target
+    element = {
+        'kind': 'import',
+        'sources': [
+            config
+        ],
+    }
+    element_path = os.path.join(project, 'target.bst')
+    _yaml.dump(element, element_path)
+
+    result = cli.run(project=project, args=['track', 'target.bst'])
+    result.assert_success()
+
+    element = _yaml.load(element_path)
+    assert 'tags' not in element['sources'][0]
diff --git a/tests/testutils/repo/git.py b/tests/testutils/repo/git.py
index bc2dae691..fe3ebd547 100644
--- a/tests/testutils/repo/git.py
+++ b/tests/testutils/repo/git.py
@@ -45,6 +45,9 @@ class Git(Repo):
     def add_tag(self, tag):
         self._run_git('tag', tag)
 
+    def add_annotated_tag(self, tag, message):
+        self._run_git('tag', '-a', tag, '-m', message)
+
     def add_commit(self):
         self._run_git('commit', '--allow-empty', '-m', 'Additional commit')
         return self.latest_commit()
@@ -95,3 +98,14 @@ class Git(Repo):
 
     def branch(self, branch_name):
         self._run_git('checkout', '-b', branch_name)
+
+    def checkout(self, commit):
+        self._run_git('checkout', commit)
+
+    def merge(self, commit):
+        self._run_git('merge', '-m', 'Merge', commit)
+        return self.latest_commit()
+
+    def rev_parse(self, rev):
+        output = self._run_git('rev-parse', rev, stdout=subprocess.PIPE).stdout
+        return output.decode('UTF-8').strip()