diff options
author | Sam Thursfield <sam.thursfield@codethink.co.uk> | 2014-10-21 13:42:03 +0100 |
---|---|---|
committer | Sam Thursfield <sam.thursfield@codethink.co.uk> | 2014-10-21 14:28:13 +0100 |
commit | 44ee17802dc04b4ecfa7ec812a5a0318bf69c62f (patch) | |
tree | 38b6b01cc9af8c62c537009cdab14f808c107daf | |
parent | f9c68e022952307b51efb9fd6badeb6c44d3de0c (diff) | |
download | import-44ee17802dc04b4ecfa7ec812a5a0318bf69c62f.tar.gz |
Reorganise, tidy and document code of main application
-rw-r--r-- | baserock-import.py | 24 | ||||
-rw-r--r-- | baserockimport/__init__.py | 25 | ||||
-rw-r--r-- | baserockimport/app.py | 178 | ||||
-rw-r--r-- | baserockimport/lorryset.py | 196 | ||||
-rw-r--r-- | baserockimport/mainloop.py (renamed from main.py) | 440 | ||||
-rw-r--r-- | baserockimport/morphsetondisk.py | 79 | ||||
-rw-r--r-- | baserockimport/package.py | 68 |
7 files changed, 592 insertions, 418 deletions
diff --git a/baserock-import.py b/baserock-import.py new file mode 100644 index 0000000..b469b74 --- /dev/null +++ b/baserock-import.py @@ -0,0 +1,24 @@ +#!/usr/bin/python +# Import foreign packaging systems into Baserock +# +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import baserockimport + + +app = baserockimport.app.BaserockImportApplication(progname='import') +app.run() diff --git a/baserockimport/__init__.py b/baserockimport/__init__.py new file mode 100644 index 0000000..171f3cf --- /dev/null +++ b/baserockimport/__init__.py @@ -0,0 +1,25 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''Baserock library for importing metadata from foreign packaging systems.''' + + +import lorryset +import morphsetondisk +import package + +import app +import mainloop diff --git a/baserockimport/app.py b/baserockimport/app.py new file mode 100644 index 0000000..c7f0e05 --- /dev/null +++ b/baserockimport/app.py @@ -0,0 +1,178 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import ansicolor +import cliapp + +import logging +import os +import pipes +import sys + +import baserockimport + + +class BaserockImportApplication(cliapp.Application): + def add_settings(self): + self.settings.string(['lorries-dir'], + "location for Lorry files", + metavar="PATH", + default=os.path.abspath('./lorries')) + self.settings.string(['definitions-dir'], + "location for morphology files", + metavar="PATH", + default=os.path.abspath('./definitions')) + self.settings.string(['checkouts-dir'], + "location for Git checkouts", + metavar="PATH", + default=os.path.abspath('./checkouts')) + self.settings.string(['lorry-working-dir'], + "Lorry working directory", + metavar="PATH", + default=os.path.abspath('./lorry-working-dir')) + + self.settings.boolean(['update-existing'], + "update all the checked-out Git trees and " + "generated definitions", + default=False) + self.settings.boolean(['use-local-sources'], + "use file:/// URLs in the stratum 'repo' " + "fields, instead of upstream: URLs", + default=False) + self.settings.boolean(['use-master-if-no-tag'], + "if the correct tag for a version can't be " + "found, use 'master' instead of raising an " + "error", + default=False) + + def _stream_has_colours(self, stream): + # http://blog.mathieu-leplatre.info/colored-output-in-console-with-python.html + if not hasattr(stream, "isatty"): + return False + if not stream.isatty(): + return False # auto color only on TTYs + try: + import curses + curses.setupterm() + return curses.tigetnum("colors") > 2 + except: + # guess false in case of error + return False + + def setup(self): + self.add_subcommand('omnibus', self.import_omnibus, + arg_synopsis='REPO PROJECT_NAME SOFTWARE_NAME') + self.add_subcommand('rubygems', self.import_rubygems, + arg_synopsis='GEM_NAME') + + self.stdout_has_colours = self._stream_has_colours(sys.stdout) + + def setup_logging_formatter_for_file(self): + root_logger = logging.getLogger() + root_logger.name = 'main' + + # You need recent cliapp for this to work, with commit "Split logging + # setup into further overrideable methods". + return logging.Formatter("%(name)s: %(levelname)s: %(message)s") + + def process_args(self, args): + if len(args) == 0: + # Cliapp default is to just say "ERROR: must give subcommand" if + # no args are passed, I prefer this. + args = ['help'] + + super(BaserockImportApplication, self).process_args(args) + + def status(self, msg, *args, **kwargs): + text = msg % args + if kwargs.get('error') == True: + logging.error(text) + if self.stdout_has_colours: + sys.stdout.write(ansicolor.red(text)) + else: + sys.stdout.write(text) + else: + logging.info(text) + sys.stdout.write(text) + sys.stdout.write('\n') + + def import_omnibus(self, args): + '''Import a software component from an Omnibus project. + + Omnibus is a tool for generating application bundles for various + platforms. See <https://github.com/opscode/omnibus> for more + information. + + ''' + if len(args) != 3: + raise cliapp.AppException( + 'Please give the location of the Omnibus definitions repo, ' + 'and the name of the project and the top-level software ' + 'component.') + + def running_inside_bundler(): + return 'BUNDLE_GEMFILE' in os.environ + + def command_to_run_python_in_directory(directory, args): + # Bundler requires that we run it from the Omnibus project + # directory. That messes up any relative paths the user may have + # passed on the commandline, so we do a bit of a hack to change + # back to the original directory inside the `bundle exec` process. + subshell_command = "(cd %s; exec python %s)" % \ + (pipes.quote(directory), ' '.join(map(pipes.quote, args))) + shell_command = "sh -c %s" % pipes.quote(subshell_command) + return shell_command + + def reexecute_self_with_bundler(path): + script = sys.argv[0] + + logging.info('Reexecuting %s within Bundler, so that extensions ' + 'use the correct dependencies for Omnibus and the ' + 'Omnibus project definitions.', script) + command = command_to_run_python_in_directory(os.getcwd(), sys.argv) + + logging.debug('Running: `bundle exec %s` in dir %s', command, path) + os.chdir(path) + os.execvp('bundle', [script, 'exec', command]) + + # Omnibus definitions are spread across multiple repos, and there is + # no stability guarantee for the definition format. The official advice + # is to use Bundler to execute Omnibus, so let's do that. + if not running_inside_bundler(): + reexecute_self_with_bundler(args[0]) + + definitions_dir = args[0] + project_name = args[1] + + loop = baserockimport.mainloop.ImportLoop( + app=self, + goal_kind='omnibus', goal_name=args[2], goal_version='master') + loop.enable_importer('omnibus', + extra_args=[definitions_dir, project_name]) + loop.enable_importer('rubygems') + loop.run() + + def import_rubygems(self, args): + '''Import one or more RubyGems.''' + if len(args) != 1: + raise cliapp.AppException( + 'Please pass the name of a RubyGem on the commandline.') + + loop = baserockimport.mainloop.ImportLoop( + app=self, + goal_kind='rubygems', goal_name=args[0], goal_version='master') + loop.enable_importer('rubygems') + loop.run() diff --git a/baserockimport/lorryset.py b/baserockimport/lorryset.py new file mode 100644 index 0000000..c3d109b --- /dev/null +++ b/baserockimport/lorryset.py @@ -0,0 +1,196 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import morphlib +import six + +import json +import logging +import os + + +class LorrySetError(Exception): + pass + + +class LorrySet(object): + '''Manages a set of .lorry files. + + A LorrySet instance operates on all the .lorry files inside the path given + at construction time. This includes .lorry files in subdirectories of the + path. + + A lorry *entry* describes the upstream repository for a given project, + which is associated to a name such as 'ruby-gems/chef' or 'gcc-tarball'. A + lorry *file* contains one or more *entries*. The filename of a lorry file + is not necessarily linked to the name of any lorry entry that it contains. + + ''' + + def __init__(self, lorries_path): + '''Initialise a LorrySet instance for the given directory. + + This will load and parse all of the .lorry files inside 'lorries_path' + into memory. + + ''' + self.path = lorries_path + + if os.path.exists(lorries_path): + self.data = self._parse_all_lorries() + else: + os.makedirs(lorries_path) + self.data = {} + + def all_lorry_files(self): + '''Return the path of each lorry file in this set.''' + for dirpath, dirnames, filenames in os.walk(self.path): + for filename in filenames: + if filename.endswith('.lorry'): + yield os.path.join(dirpath, filename) + + def _parse_all_lorries(self): + lorry_set = {} + for lorry_file in self.all_lorry_files(): + lorry = self._parse_lorry(lorry_file) + + lorry_items = lorry.items() + + for key, value in lorry_items: + if key in lorry_set: + raise LorrySetError( + '%s: duplicates existing lorry %s' % (lorry_file, key)) + + lorry_set.update(lorry_items) + + return lorry_set + + def _parse_lorry(self, lorry_file): + try: + with open(lorry_file, 'r') as f: + lorry = json.load(f) + return lorry + except ValueError as e: + raise LorrySetError( + "Error parsing %s: %s" % (lorry_file, e)) + + def get_lorry(self, name): + '''Return the lorry entry for the named project.''' + return {name: self.data[name]} + + def find_lorry_for_package(self, kind, package_name): + '''Find the lorry entry for a given foreign package, or return None. + + This makes use of an extension to the .lorry format made by the + Baserock Import tool. Fields follow the form 'x-products-$KIND' + and specify the name of a package in the foreign packaging universe + named $KIND. + + ''' + key = 'x-products-%s' % kind + for name, lorry in self.data.iteritems(): + products = lorry.get(key, []) + for entry in products: + if entry == package_name: + return {name: lorry} + + return None + + def _check_for_conflicts_in_standard_fields(self, existing, new): + '''Ensure that two lorries for the same project do actually match.''' + for field, value in existing.iteritems(): + if field.startswith('x-'): + continue + if field == 'url': + # FIXME: need a much better way of detecting whether the URLs + # are equivalent ... right now HTTP vs. HTTPS will cause an + # error, for example! + matches = (value.rstrip('/') == new[field].rstrip('/')) + else: + matches = (value == new[field]) + if not matches: + raise LorrySetError( + 'Lorry %s conflicts with existing entry %s at field %s' % + (new, existing, field)) + + def _merge_products_fields(self, existing, new): + '''Merge the x-products- fields from new lorry into an existing one.''' + is_product_field = lambda x: x.startswith('x-products-') + + existing_fields = [f for f in existing.iterkeys() if + is_product_field(f)] + new_fields = [f for f in new.iterkeys() if f not in existing_fields and + is_product_field(f)] + + for field in existing_fields: + existing[field].extend(new[field]) + existing[field] = list(set(existing[field])) + + for field in new_fields: + existing[field] = new[field] + + def _add_lorry_entry_to_lorry_file(self, filename, entry): + if os.path.exists(filename): + with open(filename) as f: + contents = json.load(f) + else: + contents = {} + + contents.update(entry) + + with morphlib.savefile.SaveFile(filename, 'w') as f: + json.dump(contents, f, indent=4, separators=(',', ': '), + sort_keys=True) + + def add(self, filename, lorry_entry): + '''Add a lorry entry to the named .lorry file. + + The lorry_entry should follow the on-disk format for a lorry stanza, + which is a dict of one entry mapping the name of the entry to its + contents. + + The .lorry file will be created if it doesn't exist. + + ''' + logging.debug('Adding %s to lorryset', filename) + + filename = os.path.join(self.path, '%s.lorry' % filename) + + assert len(lorry_entry) == 1 + + project_name = lorry_entry.keys()[0] + info = lorry_entry.values()[0] + + if len(project_name) == 0: + raise LorrySetError( + 'Invalid lorry %s: %s' % (filename, lorry_entry)) + + if not isinstance(info.get('url'), six.string_types): + raise LorrySetError( + 'Invalid URL in lorry %s: %s' % (filename, info.get('url'))) + + if project_name in self.data: + stored_lorry = self.get_lorry(project_name) + + self._check_for_conflicts_in_standard_fields( + stored_lorry[project_name], lorry_entry[project_name]) + self._merge_products_fields( + stored_lorry[project_name], lorry_entry[project_name]) + lorry_entry = stored_lorry + else: + self.data[project_name] = info + + self._add_lorry_entry_to_lorry_file(filename, lorry_entry) diff --git a/main.py b/baserockimport/mainloop.py index 7f9a1d6..c081be5 100644 --- a/main.py +++ b/baserockimport/mainloop.py @@ -1,6 +1,3 @@ -#!/usr/bin/python -# Import foreign packaging systems into Baserock -# # Copyright (C) 2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify @@ -17,212 +14,17 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import ansicolor import cliapp import morphlib import networkx -import six -import contextlib -import copy import json import logging import os -import pipes -import sys import tempfile import time -from logging import debug - - -class LorrySet(object): - '''Manages a set of .lorry files. - - The structure of .lorry files makes the code a little more confusing than - I would like. A lorry "entry" is a dict of one entry mapping name to info. - A lorry "file" is a dict of one or more of these entries merged together. - If it were a list of entries with 'name' fields, the code would be neater. - - ''' - def __init__(self, lorries_path): - self.path = lorries_path - - if os.path.exists(lorries_path): - self.data = self.parse_all_lorries() - else: - os.makedirs(lorries_path) - self.data = {} - - def all_lorry_files(self): - for dirpath, dirnames, filenames in os.walk(self.path): - for filename in filenames: - if filename.endswith('.lorry'): - yield os.path.join(dirpath, filename) - - def parse_all_lorries(self): - lorry_set = {} - for lorry_file in self.all_lorry_files(): - lorry = self.parse_lorry(lorry_file) - - lorry_items = lorry.items() - - for key, value in lorry_items: - if key in lorry_set: - raise Exception( - '%s: duplicates existing lorry %s' % (lorry_file, key)) - - lorry_set.update(lorry_items) - - return lorry_set - - def parse_lorry(self, lorry_file): - try: - with open(lorry_file, 'r') as f: - lorry = json.load(f) - return lorry - except ValueError as e: - raise cliapp.AppException( - "Error parsing %s: %s" % (lorry_file, e)) - - def get_lorry(self, name): - return {name: self.data[name]} - - def find_lorry_for_package(self, kind, package_name): - key = 'x-products-%s' % kind - for name, lorry in self.data.iteritems(): - products = lorry.get(key, []) - for entry in products: - if entry == package_name: - return {name: lorry} - - return None - - def _check_for_conflicts_in_standard_fields(self, existing, new): - '''Ensure that two lorries for the same project do actually match.''' - for field, value in existing.iteritems(): - if field.startswith('x-'): - continue - if field == 'url': - # FIXME: need a much better way of detecting whether the URLs - # are equivalent ... right now HTTP vs. HTTPS will cause an - # error, for example! - matches = (value.rstrip('/') == new[field].rstrip('/')) - else: - matches = (value == new[field]) - if not matches: - raise Exception( - 'Lorry %s conflicts with existing entry %s at field %s' % - (new, existing, field)) - - def _merge_products_fields(self, existing, new): - '''Merge the x-products- fields from new lorry into an existing one.''' - is_product_field = lambda x: x.startswith('x-products-') - - existing_fields = [f for f in existing.iterkeys() if - is_product_field(f)] - new_fields = [f for f in new.iterkeys() if f not in existing_fields and - is_product_field(f)] - - for field in existing_fields: - existing[field].extend(new[field]) - existing[field] = list(set(existing[field])) - - for field in new_fields: - existing[field] = new[field] - - def add(self, filename, lorry_entry): - logging.debug('Adding %s to lorryset', filename) - - filename = os.path.join(self.path, '%s.lorry' % filename) - - assert len(lorry_entry) == 1 - - project_name = lorry_entry.keys()[0] - info = lorry_entry.values()[0] - - if len(project_name) == 0: - raise cliapp.AppException( - 'Invalid lorry %s: %s' % (filename, lorry_entry)) - - if not isinstance(info.get('url'), six.string_types): - raise cliapp.AppException( - 'Invalid URL in lorry %s: %s' % (filename, info.get('url'))) - - if project_name in self.data: - stored_lorry = self.get_lorry(project_name) - - self._check_for_conflicts_in_standard_fields( - stored_lorry[project_name], lorry_entry[project_name]) - self._merge_products_fields( - stored_lorry[project_name], lorry_entry[project_name]) - lorry_entry = stored_lorry - else: - self.data[project_name] = info - - self._add_lorry_entry_to_lorry_file(filename, lorry_entry) - - def _add_lorry_entry_to_lorry_file(self, filename, entry): - if os.path.exists(filename): - with open(filename) as f: - contents = json.load(f) - else: - contents = {} - - contents.update(entry) - - with morphlib.savefile.SaveFile(filename, 'w') as f: - json.dump(contents, f, indent=4, separators=(',', ': '), - sort_keys=True) - - -class MorphologySet(morphlib.morphset.MorphologySet): - def __init__(self, path): - super(MorphologySet, self).__init__() - - self.path = path - self.loader = morphlib.morphloader.MorphologyLoader() - - if os.path.exists(path): - self.load_all_morphologies() - else: - os.makedirs(path) - - def load_all_morphologies(self): - logging.info('Loading all .morph files under %s', self.path) - - class FakeGitDir(morphlib.gitdir.GitDirectory): - '''Ugh - - This is here because the default constructor will search up the - directory heirarchy until it finds a '.git' directory, but that - may be totally the wrong place for our purpose: we don't have a - Git directory at all. - - ''' - def __init__(self, path): - self.dirname = path - self._config = {} - - gitdir = FakeGitDir(self.path) - finder = morphlib.morphologyfinder.MorphologyFinder(gitdir) - for filename in (f for f in finder.list_morphologies() - if not gitdir.is_symlink(f)): - text = finder.read_morphology(filename) - morph = self.loader.load_from_string(text, filename=filename) - morph.repo_url = None # self.root_repository_url - morph.ref = None # self.system_branch_name - self.add_morphology(morph) - - def get_morphology(self, repo_url, ref, filename): - return self._get_morphology(repo_url, ref, filename) - - def save_morphology(self, filename, morphology): - self.add_morphology(morphology) - morphology_to_save = copy.copy(morphology) - self.loader.unset_defaults(morphology_to_save) - filename = os.path.join(self.path, filename) - self.loader.save_to_file(filename, morphology_to_save) +import baserockimport class GitDirectory(morphlib.gitdir.GitDirectory): @@ -233,6 +35,10 @@ class GitDirectory(morphlib.gitdir.GitDirectory): # when 'repopath' isn't a Git repo. If 'repopath' is contained # within a Git repo then the GitDirectory will traverse up to the # parent repo, which isn't what we want in this case. + # + # FIXME: this should be a change to the base class, which should take + # a flag at construct time saying 'traverse_upwards_to_find_root' or + # some such. if self.dirname != dirname: logging.error( 'Got git directory %s for %s!', self.dirname, dirname) @@ -251,56 +57,6 @@ class BaserockImportException(cliapp.AppException): pass -class Package(object): - '''A package in the processing queue. - - In order to provide helpful errors, this item keeps track of what - packages depend on it, and hence of why it was added to the queue. - - ''' - def __init__(self, kind, name, version): - self.kind = kind - self.name = name - self.version = version - self.required_by = [] - self.morphology = None - self.dependencies = None - self.is_build_dep = False - self.version_in_use = version - - def __cmp__(self, other): - return cmp(self.name, other.name) - - def __repr__(self): - return '<Package %s-%s>' % (self.name, self.version) - - def __str__(self): - if len(self.required_by) > 0: - required_msg = ', '.join(self.required_by) - required_msg = ', required by: ' + required_msg - else: - required_msg = '' - return '%s-%s%s' % (self.name, self.version, required_msg) - - def add_required_by(self, item): - self.required_by.append('%s-%s' % (item.name, item.version)) - - def match(self, name, version): - return (self.name==name and self.version==version) - - def set_morphology(self, morphology): - self.morphology = morphology - - def set_dependencies(self, dependencies): - self.dependencies = dependencies - - def set_is_build_dep(self, is_build_dep): - self.is_build_dep = is_build_dep - - def set_version_in_use(self, version_in_use): - self.version_in_use = version_in_use - - def find(iterable, match): return next((x for x in iterable if match(x)), None) @@ -327,7 +83,7 @@ def run_extension(filename, args, cwd='.'): ) # There are better ways of doing this, but it works for now. - main_path = os.path.dirname(os.path.realpath(__file__)) + main_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) extension_path = os.path.join(main_path, filename) logging.debug("Running %s %s with cwd %s" % (extension_path, args, cwd)) @@ -359,8 +115,10 @@ class ImportLoop(object): self.goal_version = goal_version self.extra_args = extra_args - self.lorry_set = LorrySet(self.app.settings['lorries-dir']) - self.morph_set = MorphologySet(self.app.settings['definitions-dir']) + self.lorry_set = baserockimport.lorryset.LorrySet( + self.app.settings['lorries-dir']) + self.morph_set = baserockimport.morphsetondisk.MorphologySetOnDisk( + self.app.settings['definitions-dir']) self.morphloader = morphlib.morphloader.MorphologyLoader() @@ -389,7 +147,8 @@ class ImportLoop(object): if not os.path.exists(chunk_dir): os.makedirs(chunk_dir) - goal = Package(self.goal_kind, self.goal_name, self.goal_version) + goal = baserockimport.package.Package( + self.goal_kind, self.goal_name, self.goal_version) to_process = [goal] processed = networkx.DiGraph() @@ -488,7 +247,8 @@ class ImportLoop(object): queue_item = find( to_process, lambda i: i.match(dep_name, dep_version)) if queue_item is None: - queue_item = Package(kind, dep_name, dep_version) + queue_item = baserockimport.package.Package( + kind, dep_name, dep_version) to_process.append(queue_item) dep_package = queue_item @@ -544,7 +304,7 @@ class ImportLoop(object): lorry_text = run_extension(tool, extra_args + [name]) try: lorry = json.loads(lorry_text) - except ValueError as e: + except ValueError: raise cliapp.AppException( 'Invalid output from %s: %s' % (tool, lorry_text)) return lorry @@ -733,7 +493,7 @@ class ImportLoop(object): order = reversed(sorted(graph.nodes())) try: return networkx.topological_sort(graph, nbunch=order) - except networkx.NetworkXUnfeasible as e: + except networkx.NetworkXUnfeasible: # Cycle detected! loop_subgraphs = networkx.strongly_connected_component_subgraphs( graph, copy=False) @@ -751,11 +511,12 @@ class ImportLoop(object): self.app.settings['definitions-dir'], 'strata', '%s.morph' % goal_name) - if os.path.exists(filename) and not self.app.settings['update-existing']: - self.app.status( - msg='Found stratum morph for %s at %s, not overwriting' % - (goal_name, filename)) - return + if os.path.exists(filename): + if not self.app.settings['update-existing']: + self.app.status( + msg='Found stratum morph for %s at %s, not overwriting' % + (goal_name, filename)) + return self.app.status(msg='Generating stratum morph for %s' % goal_name) @@ -800,160 +561,3 @@ class ImportLoop(object): json.dumps(stratum), filename=filename) self.morphloader.unset_defaults(morphology) self.morphloader.save_to_file(filename, morphology) - - -class BaserockImportApplication(cliapp.Application): - def add_settings(self): - self.settings.string(['lorries-dir'], - "location for Lorry files", - metavar="PATH", - default=os.path.abspath('./lorries')) - self.settings.string(['definitions-dir'], - "location for morphology files", - metavar="PATH", - default=os.path.abspath('./definitions')) - self.settings.string(['checkouts-dir'], - "location for Git checkouts", - metavar="PATH", - default=os.path.abspath('./checkouts')) - self.settings.string(['lorry-working-dir'], - "Lorry working directory", - metavar="PATH", - default=os.path.abspath('./lorry-working-dir')) - - self.settings.boolean(['update-existing'], - "update all the checked-out Git trees and " - "generated definitions", - default=False) - self.settings.boolean(['use-local-sources'], - "use file:/// URLs in the stratum 'repo' " - "fields, instead of upstream: URLs", - default=False) - self.settings.boolean(['use-master-if-no-tag'], - "if the correct tag for a version can't be " - "found, use 'master' instead of raising an " - "error", - default=False) - - def _stream_has_colours(self, stream): - # http://blog.mathieu-leplatre.info/colored-output-in-console-with-python.html - if not hasattr(stream, "isatty"): - return False - if not stream.isatty(): - return False # auto color only on TTYs - try: - import curses - curses.setupterm() - return curses.tigetnum("colors") > 2 - except: - # guess false in case of error - return False - - def setup(self): - self.add_subcommand('omnibus', self.import_omnibus, - arg_synopsis='REPO PROJECT_NAME SOFTWARE_NAME') - self.add_subcommand('rubygems', self.import_rubygems, - arg_synopsis='GEM_NAME') - - self.stdout_has_colours = self._stream_has_colours(sys.stdout) - - def setup_logging_formatter_for_file(self): - root_logger = logging.getLogger() - root_logger.name = 'main' - - # You need recent cliapp for this to work, with commit "Split logging - # setup into further overrideable methods". - return logging.Formatter("%(name)s: %(levelname)s: %(message)s") - - def process_args(self, args): - if len(args) == 0: - # Cliapp default is to just say "ERROR: must give subcommand" if - # no args are passed, I prefer this. - args = ['help'] - - super(BaserockImportApplication, self).process_args(args) - - def status(self, msg, *args, **kwargs): - text = msg % args - if kwargs.get('error') == True: - logging.error(text) - if self.stdout_has_colours: - sys.stdout.write(ansicolor.red(text)) - else: - sys.stdout.write(text) - else: - logging.info(text) - sys.stdout.write(text) - sys.stdout.write('\n') - - def import_omnibus(self, args): - '''Import a software component from an Omnibus project. - - Omnibus is a tool for generating application bundles for various - platforms. See <https://github.com/opscode/omnibus> for more - information. - - ''' - if len(args) != 3: - raise cliapp.AppException( - 'Please give the location of the Omnibus definitions repo, ' - 'and the name of the project and the top-level software ' - 'component.') - - def running_inside_bundler(): - return 'BUNDLE_GEMFILE' in os.environ - - def command_to_run_python_in_directory(directory, args): - # Bundler requires that we run it from the Omnibus project - # directory. That messes up any relative paths the user may have - # passed on the commandline, so we do a bit of a hack to change - # back to the original directory inside the `bundle exec` process. - subshell_command = "(cd %s; exec python %s)" % \ - (pipes.quote(directory), ' '.join(map(pipes.quote, args))) - shell_command = "sh -c %s" % pipes.quote(subshell_command) - return shell_command - - def reexecute_self_with_bundler(path): - script = sys.argv[0] - - logging.info('Reexecuting %s within Bundler, so that extensions ' - 'use the correct dependencies for Omnibus and the ' - 'Omnibus project definitions.', script) - command = command_to_run_python_in_directory(os.getcwd(), sys.argv) - - logging.debug('Running: `bundle exec %s` in dir %s', command, path) - os.chdir(path) - os.execvp('bundle', [script, 'exec', command]) - - # Omnibus definitions are spread across multiple repos, and there is - # no stability guarantee for the definition format. The official advice - # is to use Bundler to execute Omnibus, so let's do that. - if not running_inside_bundler(): - reexecute_self_with_bundler(args[0]) - - definitions_dir = args[0] - project_name = args[1] - - loop = ImportLoop( - app=self, - goal_kind='omnibus', goal_name=args[2], goal_version='master') - loop.enable_importer('omnibus', - extra_args=[definitions_dir, project_name]) - loop.enable_importer('rubygems') - loop.run() - - def import_rubygems(self, args): - '''Import one or more RubyGems.''' - if len(args) != 1: - raise cliapp.AppException( - 'Please pass the name of a RubyGem on the commandline.') - - loop = ImportLoop( - app=self, - goal_kind='rubygems', goal_name=args[0], goal_version='master') - loop.enable_importer('rubygems') - loop.run() - - -app = BaserockImportApplication(progname='import') -app.run() diff --git a/baserockimport/morphsetondisk.py b/baserockimport/morphsetondisk.py new file mode 100644 index 0000000..565ad24 --- /dev/null +++ b/baserockimport/morphsetondisk.py @@ -0,0 +1,79 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import morphlib + +import copy +import logging +import os + + +class MorphologySetOnDisk(morphlib.morphset.MorphologySet): + '''Extensions to the morphlib.MorphologySet class. + + The base class deals only with reading morphologies into memory. This class + extends it to support reading and writing them from disk. + + FIXME: this should perhaps be merged into the base class in morphlib. + + ''' + + def __init__(self, path): + super(MorphologySetOnDisk, self).__init__() + + self.path = path + self.loader = morphlib.morphloader.MorphologyLoader() + + if os.path.exists(path): + self.load_all_morphologies() + else: + os.makedirs(path) + + def load_all_morphologies(self): + logging.info('Loading all .morph files under %s', self.path) + + class FakeGitDir(morphlib.gitdir.GitDirectory): + '''FIXME: Ugh + + This is here because the default constructor will search up the + directory heirarchy until it finds a '.git' directory, but that + may be totally the wrong place for our purpose: we don't have a + Git directory at all. + + ''' + def __init__(self, path): + self.dirname = path + self._config = {} + + gitdir = FakeGitDir(self.path) + finder = morphlib.morphologyfinder.MorphologyFinder(gitdir) + for filename in (f for f in finder.list_morphologies() + if not gitdir.is_symlink(f)): + text = finder.read_morphology(filename) + morph = self.loader.load_from_string(text, filename=filename) + morph.repo_url = None # self.root_repository_url + morph.ref = None # self.system_branch_name + self.add_morphology(morph) + + def get_morphology(self, repo_url, ref, filename): + return self._get_morphology(repo_url, ref, filename) + + def save_morphology(self, filename, morphology): + self.add_morphology(morphology) + morphology_to_save = copy.copy(morphology) + self.loader.unset_defaults(morphology_to_save) + filename = os.path.join(self.path, filename) + self.loader.save_to_file(filename, morphology_to_save) diff --git a/baserockimport/package.py b/baserockimport/package.py new file mode 100644 index 0000000..6095b30 --- /dev/null +++ b/baserockimport/package.py @@ -0,0 +1,68 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +class Package(object): + '''A package in the processing queue. + + In order to provide helpful errors, this item keeps track of what + packages depend on it, and hence of why it was added to the queue. + + ''' + def __init__(self, kind, name, version): + self.kind = kind + self.name = name + self.version = version + self.required_by = [] + self.morphology = None + self.dependencies = None + self.is_build_dep = False + self.version_in_use = version + + def __cmp__(self, other): + return cmp(self.name, other.name) + + def __repr__(self): + return '<Package %s-%s>' % (self.name, self.version) + + def __str__(self): + if len(self.required_by) > 0: + required_msg = ', '.join(self.required_by) + required_msg = ', required by: ' + required_msg + else: + required_msg = '' + return '%s-%s%s' % (self.name, self.version, required_msg) + + def add_required_by(self, item): + self.required_by.append('%s-%s' % (item.name, item.version)) + + def match(self, name, version): + return (self.name==name and self.version==version) + + # FIXME: these accessors are useless, but I want there to be some way + # of making it clear that some of the state of the Package object is + # mutable and some of the state is not ... + + def set_morphology(self, morphology): + self.morphology = morphology + + def set_dependencies(self, dependencies): + self.dependencies = dependencies + + def set_is_build_dep(self, is_build_dep): + self.is_build_dep = is_build_dep + + def set_version_in_use(self, version_in_use): + self.version_in_use = version_in_use |