Merge branch 'benbrown/release-notes' into 'master'

Further release note generation fixes

See merge request !360

5 files changed, 63 insertions, 27 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b9feb5f..d2d4566 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -11,6 +11,7 @@ cache:
 stages:
 - pep8
 - cache_keys
+- release_notes
 - build
 - test
 - deploy
@@ -43,6 +44,17 @@ cache_keys_v6:
   - echo ci.d9dec300a7cb2bc273dc6846e69a11b7f3ad304462f3a4c6ce8c5ab6ead11647 > expected.result
   - diff expected.result ybd.result
 
+check_release_notes:
+  stage: release_notes
+  variables:
+    YBD_release_note: "${CI_PROJECT_DIR}/release-notes.txt"
+    YBD_mode: "keys-only"
+  script:
+    - ./ybd.py definitions/systems/minimal-system-x86_64-generic.morph x86_64
+  artifacts:
+    paths:
+    - release-notes.txt
+
 check_build_debian_jessie:
   image: debian:jessie
   stage: build
diff --git a/ybd/config/ybd.conf b/ybd/config/ybd.conf
index e78681b..a851d69 100644
--- a/ybd/config/ybd.conf
+++ b/ybd/config/ybd.conf
@@ -187,7 +187,7 @@ no-distcc: True
 # release-since: HEAD~1
 
 # release-cmd can be used to configure the git output for the release note
-# release-cmd: ['git', 'log', '--pretty=oneline', '-n=2']
+# release-cmd: ['git', 'log', '--pretty=oneline', '-n', '2']
 
 # Some modes of ybd (eg build-only, keys-only) output a result to a file
 result-file: './ybd.result'
diff --git a/ybd/release_note.py b/ybd/release_note.py
index eeecdf7..7d44b59 100644
--- a/ybd/release_note.py
+++ b/ybd/release_note.py
@@ -15,11 +15,15 @@
 # =*= License: GPL-2 =*=
 
 import os
-from subprocess import check_output, call
+import re
+from subprocess import check_output
 import tempfile
 import app
+import yaml
+import difflib
+import itertools
 from app import chdir, config, log
-from morphs import Morphs
+from pots import Pots
 from repos import (
     explore, get_last_tag, get_repo_name, mirror, mirror_has_ref, update_mirror
 )
@@ -28,18 +32,14 @@ from repos import (
 def do_release_note(release_note):
     tempfile.tempdir = config['tmp']
     tmpdir = tempfile.mkdtemp()
-    if call(['git', 'config', 'user.name']):
-        call(['git', 'config', 'user.name', 'ybd'])
-    if call(['git', 'config', 'user.email']):
-        call(['git', 'config', 'user.email', 'ybd@baserock.org'])
 
     if 'release-since' in config:
         ref = config['release-since']
     else:
         ref = get_last_tag('.')
 
-    with explore(ref):
-        old_defs = Morphs()._data
+    with explore(ref) as defdir:
+        old_defs = Pots()._data
 
     for path in app.defs._data:
         dn = app.defs.get(path)
@@ -59,6 +59,22 @@ def do_release_note(release_note):
         release_note)
 
 
+def represent_str(dumper, data):
+    if data.count('\n') == 0:
+        return yaml.representer.SafeRepresenter.represent_str(dumper, data)
+    return dumper.represent_scalar(u'tag:yaml.org,2002:str', data, style='|')
+
+
+def sanitise_yaml(obj):
+    stripped = re.sub(r' +(?=\\n)', '', yaml.dump(obj), flags=re.MULTILINE)
+    yaml.add_representer(str, represent_str)
+    return yaml.safe_load(stripped)
+
+
+def expand_obj(obj):
+    return [x for x in yaml.dump(obj, default_flow_style=False).splitlines()]
+
+
 def log_changes(dn, tmpdir, old_defs, ref):
     do_git_log = False
     old_def = old_defs.get(dn['path'], {})
@@ -72,31 +88,30 @@ def log_changes(dn, tmpdir, old_defs, ref):
                 if type(dn[key]) in (str, unicode, int):
                     f.write('%s | %s\n' % (old_value, dn[key]))
                 if type(dn[key]) not in (str, unicode, int, float):
-                    if old_value:
-                        for x in old_value:
-                            f.write(repr(x))
-                    f.write('\n                vvv\n')
-                    if dn[key]:
-                        for x in dn[key]:
-                            f.write(repr(x))
+                    before = expand_obj(sanitise_yaml(old_value or ""))
+                    after = expand_obj(sanitise_yaml(dn[key]))
+                    diff = itertools.islice(difflib.unified_diff(
+                        before, after), 2, None)
+                    f.write('\n'.join(diff))
                 f.write('\n\n')
 
         if (dn.get('kind', 'chunk') == 'chunk' and dn.get('repo') and
                 config.get('release-cmd') and old_def):
-            log(dn, 'Logging git change history', tmpdir)
             try:
                 gitdir = os.path.join(config['gits'],
                                       get_repo_name(dn['repo']))
                 cur_ref = dn.get('sha', dn['ref'])
                 old_ref = old_def.get('sha', old_def['ref'])
                 if old_ref != cur_ref:
+                    log(dn, 'Logging git change history', tmpdir)
                     if not os.path.exists(gitdir):
                         mirror(dn['name'], dn['repo'])
                     elif not mirror_has_ref(gitdir, cur_ref) or \
                             not mirror_has_ref(gitdir, old_ref):
                         update_mirror(dn['name'], dn['repo'], gitdir)
                     with chdir(gitdir):
-                        text = cur_ref + '..' + old_ref
+                        text = old_ref + '..' + cur_ref
+                        f.write("[%s] Repository changes:\n" % dn['path'])
                         f.write(check_output(config['release-cmd'] + [text]))
             except Exception:
                 log(dn, 'WARNING: Failed to log git changes')
diff --git a/ybd/repos.py b/ybd/repos.py
index cdf1578..643367e 100644
--- a/ybd/repos.py
+++ b/ybd/repos.py
@@ -330,12 +330,10 @@ def checkout_submodules(dn):
 
 @contextlib.contextmanager
 def explore(ref):
-    try:
-        call(['git', 'stash', '--include-untracked'])
-        head = get_version('.').split(' ')[1]
-        call(['git', 'checkout', ref])
-        yield
-
-    finally:
-        call(['git', 'checkout', head])
-        call(['git', 'stash', 'pop'])
+    with open(os.devnull, "w") as fnull:
+        with utils.tempdir(dir=app.config['tmp']) as temp:
+            check_output(['git', 'clone', app.config['defdir'], temp],
+                         stderr=fnull)
+            check_output(['git', 'checkout', ref], cwd=temp, stderr=fnull)
+            with app.chdir(temp):
+                yield temp
diff --git a/ybd/utils.py b/ybd/utils.py
index ff186c1..a3493e7 100644
--- a/ybd/utils.py
+++ b/ybd/utils.py
@@ -17,6 +17,7 @@
 import re
 import gzip
 import tarfile
+import tempfile
 import contextlib
 import os
 import shutil
@@ -479,3 +480,13 @@ def ref_expects_lfs(gitdir, ref):
                 ['git', 'cat-file', 'blob', blob.split()[2]], stderr=fnull)
             return bool(re.search('filter=lfs.*-text', attributes))
     return False
+
+
+@contextlib.contextmanager
+def tempdir(dir=None):
+    tempfile.tempdir = dir
+    tempdir = tempfile.mkdtemp()
+    try:
+        yield tempdir
+    finally:
+        shutil.rmtree(tempdir, ignore_errors=True)