diff options
author | Gökçen Nurlu <gnurlu1@bloomberg.net> | 2018-06-18 11:43:07 +0100 |
---|---|---|
committer | Gökçen Nurlu <gnurlu1@bloomberg.net> | 2018-06-27 16:48:56 +0100 |
commit | ddcc06456b7814afe7efcdca37f151c39b5a072e (patch) | |
tree | eb410ba443de9f094fafa447cea2546f600b858d | |
parent | 866c4ca2291db1b66ab6d48f42ae2bff619c917b (diff) | |
download | buildstream-ddcc06456b7814afe7efcdca37f151c39b5a072e.tar.gz |
Add dep_transform example
-rw-r--r-- | buildstream/plugins/sources/dep_transform.py | 192 |
1 files changed, 192 insertions, 0 deletions
diff --git a/buildstream/plugins/sources/dep_transform.py b/buildstream/plugins/sources/dep_transform.py new file mode 100644 index 000000000..f0d02506d --- /dev/null +++ b/buildstream/plugins/sources/dep_transform.py @@ -0,0 +1,192 @@ +# +# Copyright Bloomberg Finance LP +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see <http://www.gnu.org/licenses/>. +# +# Authors: +# Chandan Singh <csingh43@bloomberg.net> +# Gokcen Nurlu <gnurlu1@bloomberg.net> +"""A SourceTransform implementation for staging Go dependencies via `dep` + +**Usage** + +.. code: yaml + # Specify the godep source kind + kind: dep_transform + + # Optionally specify a relative staging directory + # directory: path/to/stage + + # Specify the lock file as the ref. + # If no ref is provided, the new ref will be the contents of the lock file + # generated by running `dep ensure -no-vendor`. + ref: "<lock_file content>" +.. +""" + +import errno +import os +import shutil +from hashlib import md5 + +from buildstream import SourceTransform, Source, SourceError +from buildstream import utils +from buildstream import Consistency + +GOPKG_LOCK_FILE = 'Gopkg.lock' +GOPKG_VENDOR_DIR = 'vendor' + + +class DepTransform(SourceTransform): + # pylint: disable=attribute-defined-outside-init + + def configure(self, node): + self.node_validate(node, ['ref'] + Source.COMMON_CONFIG_KEYS) + self.ref = self.node_get_member(node, str, 'ref', '').strip() + + @property + def mirror(self): + path = os.path.join( + self.get_mirror_directory(), + self.name, + md5(self.ref.encode('utf-8')).hexdigest() + ) + os.makedirs(path, exist_ok=True) + return path + + def preflight(self): + # Check if dep is installed, get the binaries at the same time + self.host_dep = utils.get_host_tool('dep') + + def get_unique_key(self): + # TODO: This plugin's inputs are actually previous source. What should + # we do here? + return (self.ref,) + + def get_consistency(self): + if self.ref == '': + return Consistency.INCONSISTENT + for dest in (GOPKG_LOCK_FILE, GOPKG_VENDOR_DIR): + dest_path = os.path.join(self.mirror, dest) + if not os.path.exists(dest_path): + return Consistency.RESOLVED + lock_file_path = os.path.join(self.mirror, GOPKG_LOCK_FILE) + with open(lock_file_path, encoding='utf-8') as lock_file: + if lock_file.read().strip() == self.ref: + return Consistency.CACHED + return Consistency.RESOLVED + + def get_ref(self): + return self.ref + + def set_ref(self, ref, node): + self.ref = node['ref'] = ref + + def track(self, previous_staging_dir): + with self.timed_activity('Tracking DepTransform source based on previous sources'): + with self.tempdir() as goroot: + # dep refuses to work on sources that are not under + # GOPATH/src so we need to artificially create that directory + # stucture. + go_sources_root = os.path.join(goroot, 'src', 'project') + os.makedirs(os.path.dirname(go_sources_root), exist_ok=True) + + shutil.move(previous_staging_dir, go_sources_root) + gopkg_lock_path = os.path.join(go_sources_root, GOPKG_LOCK_FILE) + # Check if the repo has a lock file already + if not os.path.isfile(gopkg_lock_path): + # If doesn't, let's create a new lock file + self.call([self.host_dep, 'ensure', '-no-vendor'], + cwd=go_sources_root, + env=dict(os.environ, GOPATH=goroot), + fail='Failed to update the go dep lock file with the new references') + + with open(gopkg_lock_path, encoding='utf-8') as lock_file: + return lock_file.read() + + def fetch(self, previous_staging_dir): + with self.timed_activity('Fetching DepTransform source dependencies based on Gopkg.lock'): + with self.tempdir() as goroot: + # dep refuses to work on sources that are not under + # GOPATH/src so we need to artificially create that directory + # stucture. + go_sources_root = os.path.join(goroot, 'src', 'project') + os.makedirs(os.path.dirname(go_sources_root), exist_ok=True) + + shutil.move(previous_staging_dir, go_sources_root) + + gopkg_lock_path = os.path.join(go_sources_root, GOPKG_LOCK_FILE) + + def write_lock_file(): + with open(gopkg_lock_path, mode='w', encoding='utf-8') as lock_file: + lock_file.write(self.get_ref()) + + # Is there a lock file present? + if os.path.isfile(gopkg_lock_path): + # There is! We should ensure it is the same as the ref + with open(gopkg_lock_path, encoding='utf-8') as lock_file: + stripped_lock_file = lock_file.read().strip() + # The current yaml parser strips this already, but let's not rely on it to do so + stripped_track_ref = self.get_ref().strip() + + if stripped_lock_file != stripped_track_ref: + self.warn("Tracking ref and lock file in source differ. Using tracking ref.") + write_lock_file() + else: + write_lock_file() + + self.call([self.host_dep, 'ensure', '-vendor-only'], + cwd=go_sources_root, + env=dict(os.environ, GOPATH=goroot), + fail='Failed to populate the vendor directory from the lock file') + + # Copy the stuff that we really need + for source in (GOPKG_LOCK_FILE, GOPKG_VENDOR_DIR): + source_path = os.path.join(go_sources_root, source) + dest_path = os.path.join(self.mirror, source) + try: + os.replace(source_path, dest_path) + except OSError as e: + # To avoid race condition between two concurrently + # running `fetch()` processes. + if e.errno not in (errno.ENOTEMPTY, errno.EEXIST): + raise + self.warn( + "{} seems already fetched. ".format(dest_path) + + "Continuing with the existing source." + ) + + def stage(self, directory): + with self.timed_activity("Staging DepTransform source from based on Gopkg.lock", silent_nested=True): + shutil.copy(os.path.join(self.mirror, GOPKG_LOCK_FILE), + os.path.join(directory, GOPKG_LOCK_FILE)) + # FIXME: Should we ever need to support one, this may + # break on platforms that don't support symlinks if + # a project contains broken symlinks. + target_vendor_dir = os.path.join(directory, GOPKG_VENDOR_DIR) + try: + shutil.copytree(os.path.join(self.mirror, GOPKG_VENDOR_DIR), + target_vendor_dir, + symlinks=True) + except FileExistsError: + raise SourceError("{}: Unable to stage vendor directory because it already exists here '{}'" + .format(self, target_vendor_dir)) + except shutil.Error as err: + raise SourceError("{}: Unable to stage vendor directory at '{}'" + .format(self, target_vendor_dir)) from err + + +# Plugin entry point +def setup(): + return DepTransform |