diff options
author | Ben Brown <ben.brown@codethink.co.uk> | 2017-02-28 15:56:17 +0000 |
---|---|---|
committer | Ben Brown <ben.brown@codethink.co.uk> | 2017-03-01 19:14:16 +0000 |
commit | c59d65cfb08f22c87a4d059319da0262f78953e0 (patch) | |
tree | d2ffe6d64ccd520b0604c6d4730c943d80499811 | |
parent | ea4f9cabcf20c0590afe694b8f2afcc8dd13e8a3 (diff) | |
download | ybd-c59d65cfb08f22c87a4d059319da0262f78953e0.tar.gz |
Add support for Git LFS repositories
-rw-r--r-- | ybd/app.py | 3 | ||||
-rw-r--r-- | ybd/repos.py | 29 | ||||
-rw-r--r-- | ybd/utils.py | 32 |
3 files changed, 64 insertions, 0 deletions
@@ -124,6 +124,9 @@ def warning_handler(message, category, filename, lineno, file=None, line=None): def setup(program, target, arch, mode, original_cwd=""): os.environ['LANG'] = 'en_US.UTF-8' + # Installation of git-lfs on a system can pollute /etc/gitconfig. Setting + # GIT_CONFIG_NOSYSTEM so ybd will ignore the system config. + os.environ['GIT_CONFIG_NOSYSTEM'] = "1" config['start-time'] = datetime.datetime.now() config['program'] = os.path.basename(program) config['my-version'] = get_version(os.path.dirname(__file__)) diff --git a/ybd/repos.py b/ybd/repos.py index cbf53a0..2b32713 100644 --- a/ybd/repos.py +++ b/ybd/repos.py @@ -17,6 +17,7 @@ import contextlib import os import re +import errno import shutil import string from subprocess import call, check_output @@ -195,6 +196,26 @@ def checkout(dn): utils.set_mtime_recursively(dn['checkout']) +def fetch_lfs_binaries(name, checkout): + app.log(name, "Checking out lfs binaries") + try: + with open(os.devnull, 'w') as fnull: + call(['git-lfs', 'version'], stdout=fnull, stderr=fnull) + except OSError as e: + if e.errno == errno.ENOENT: + app.log(name, + 'git-lfs is required to fetch LFS repos', exit=True) + + with app.chdir(checkout), open(os.devnull, 'w') as fnull: + if call(['git', 'lfs', 'install', '--local'], + stdout=fnull, stderr=fnull): + app.log(name, 'lfs install failed for', checkout, exit=True) + if call(['git', 'lfs', 'fetch'], stdout=fnull, stderr=fnull): + app.log(name, 'lfs fetch failed for', checkout, exit=True) + if call(['git', 'lfs', 'checkout'], stdout=fnull, stderr=fnull): + app.log(name, 'lfs checkout failed for', checkout, exit=True) + + def _checkout(name, repo, ref, checkout): gitdir = os.path.join(app.config['gits'], get_repo_name(repo)) if not os.path.exists(gitdir): @@ -203,6 +224,10 @@ def _checkout(name, repo, ref, checkout): update_mirror(name, repo, gitdir) # checkout the required version from git with open(os.devnull, "w") as fnull: + # Ensure lfs filters are not set in .gitconfig: cloning with lfs + # from a local path will error. + call(['git', 'config', '--global', '--remove-section', 'filter.lfs'], + stdout=fnull, stderr=fnull) # We need to pass '--no-hardlinks' because right now there's nothing to # stop the build from overwriting the files in the .git directory # inside the sandbox. If they were hardlinks, it'd be possible for a @@ -219,6 +244,10 @@ def _checkout(name, repo, ref, checkout): app.log(name, 'Git checkout %s in %s' % (repo, checkout)) app.log(name, 'Upstream version %s' % get_version(checkout, ref)) + is_lfs = utils.ref_expects_lfs(gitdir, ref) + if is_lfs: + utils.set_origin_url(gitdir, checkout) + fetch_lfs_binaries(name, checkout) def source_date_epoch(checkout): diff --git a/ybd/utils.py b/ybd/utils.py index aa49f0a..ff186c1 100644 --- a/ybd/utils.py +++ b/ybd/utils.py @@ -14,6 +14,7 @@ # # =*= License: GPL-2 =*= +import re import gzip import tarfile import contextlib @@ -25,6 +26,7 @@ from fs.osfs import OSFS from fs.multifs import MultiFS import calendar import app +from subprocess import check_call, check_output # The magic number for timestamps: 2011-11-11 11:11:11 default_magic_timestamp = calendar.timegm([2011, 11, 11, 11, 11, 11]) @@ -447,3 +449,33 @@ def monkeypatch(obj, attr, new_value): setattr(obj, attr, new_value) yield setattr(obj, attr, old_value) + + +def set_origin_url(gitdir, checkout): + '''Sets the origin url of a checkout to that of its gitdir. + + git-lfs requires a remote server in order to fetch binaries, so we set the + origin url to that of the mirror for lfs enabled checkouts. + ''' + try: + with open(os.devnull, 'w') as fnull, app.chdir(gitdir): + origin_url = check_output( + ['git', 'config', '--get', 'remote.origin.url'], stderr=fnull) + with open(os.devnull, 'w') as fnull, app.chdir(checkout): + check_call(['git', 'config', 'remote.origin.url', origin_url], + stderr=fnull) + except: + app.log('UTILS', 'Setting origin url failed for', checkout, exit=True) + raise + + +def ref_expects_lfs(gitdir, ref): + '''Parses .gitattributes at a ref to determine if git-lfs is required.''' + with open(os.devnull, 'w') as fnull, app.chdir(gitdir): + blob = check_output( + ['git', 'ls-tree', ref, '.gitattributes'], stderr=fnull) + if blob: + attributes = check_output( + ['git', 'cat-file', 'blob', blob.split()[2]], stderr=fnull) + return bool(re.search('filter=lfs.*-text', attributes)) + return False |