Reorganise, tidy and document code of main application

7 files changed, 592 insertions, 418 deletions

diff --git a/baserock-import.py b/baserock-import.py
new file mode 100644
index 0000000..b469b74
--- /dev/null
+++ b/baserock-import.py
@@ -0,0 +1,24 @@
+#!/usr/bin/python
+# Import foreign packaging systems into Baserock
+#
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import baserockimport
+
+
+app = baserockimport.app.BaserockImportApplication(progname='import')
+app.run()
diff --git a/baserockimport/__init__.py b/baserockimport/__init__.py
new file mode 100644
index 0000000..171f3cf
--- /dev/null
+++ b/baserockimport/__init__.py
@@ -0,0 +1,25 @@
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+'''Baserock library for importing metadata from foreign packaging systems.'''
+
+
+import lorryset
+import morphsetondisk
+import package
+
+import app
+import mainloop
diff --git a/baserockimport/app.py b/baserockimport/app.py
new file mode 100644
index 0000000..c7f0e05
--- /dev/null
+++ b/baserockimport/app.py
@@ -0,0 +1,178 @@
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import ansicolor
+import cliapp
+
+import logging
+import os
+import pipes
+import sys
+
+import baserockimport
+
+
+class BaserockImportApplication(cliapp.Application):
+    def add_settings(self):
+        self.settings.string(['lorries-dir'],
+                             "location for Lorry files",
+                             metavar="PATH",
+                             default=os.path.abspath('./lorries'))
+        self.settings.string(['definitions-dir'],
+                             "location for morphology files",
+                             metavar="PATH",
+                             default=os.path.abspath('./definitions'))
+        self.settings.string(['checkouts-dir'],
+                             "location for Git checkouts",
+                             metavar="PATH",
+                             default=os.path.abspath('./checkouts'))
+        self.settings.string(['lorry-working-dir'],
+                             "Lorry working directory",
+                             metavar="PATH",
+                             default=os.path.abspath('./lorry-working-dir'))
+
+        self.settings.boolean(['update-existing'],
+                              "update all the checked-out Git trees and "
+                              "generated definitions",
+                              default=False)
+        self.settings.boolean(['use-local-sources'],
+                              "use file:/// URLs in the stratum 'repo' "
+                              "fields, instead of upstream: URLs",
+                              default=False)
+        self.settings.boolean(['use-master-if-no-tag'],
+                              "if the correct tag for a version can't be "
+                              "found, use 'master' instead of raising an "
+                              "error",
+                              default=False)
+
+    def _stream_has_colours(self, stream):
+        # http://blog.mathieu-leplatre.info/colored-output-in-console-with-python.html
+        if not hasattr(stream, "isatty"):
+            return False
+        if not stream.isatty():
+            return False # auto color only on TTYs
+        try:
+            import curses
+            curses.setupterm()
+            return curses.tigetnum("colors") > 2
+        except:
+            # guess false in case of error
+            return False
+
+    def setup(self):
+        self.add_subcommand('omnibus', self.import_omnibus,
+                            arg_synopsis='REPO PROJECT_NAME SOFTWARE_NAME')
+        self.add_subcommand('rubygems', self.import_rubygems,
+                            arg_synopsis='GEM_NAME')
+
+        self.stdout_has_colours = self._stream_has_colours(sys.stdout)
+
+    def setup_logging_formatter_for_file(self):
+        root_logger = logging.getLogger()
+        root_logger.name = 'main'
+
+        # You need recent cliapp for this to work, with commit "Split logging
+        # setup into further overrideable methods".
+        return logging.Formatter("%(name)s: %(levelname)s: %(message)s")
+
+    def process_args(self, args):
+        if len(args) == 0:
+            # Cliapp default is to just say "ERROR: must give subcommand" if
+            # no args are passed, I prefer this.
+            args = ['help']
+
+        super(BaserockImportApplication, self).process_args(args)
+
+    def status(self, msg, *args, **kwargs):
+        text = msg % args
+        if kwargs.get('error') == True:
+            logging.error(text)
+            if self.stdout_has_colours:
+                sys.stdout.write(ansicolor.red(text))
+            else:
+                sys.stdout.write(text)
+        else:
+            logging.info(text)
+            sys.stdout.write(text)
+        sys.stdout.write('\n')
+
+    def import_omnibus(self, args):
+        '''Import a software component from an Omnibus project.
+
+        Omnibus is a tool for generating application bundles for various
+        platforms. See <https://github.com/opscode/omnibus> for more
+        information.
+
+        '''
+        if len(args) != 3:
+            raise cliapp.AppException(
+                'Please give the location of the Omnibus definitions repo, '
+                'and the name of the project and the top-level software '
+                'component.')
+
+        def running_inside_bundler():
+            return 'BUNDLE_GEMFILE' in os.environ
+
+        def command_to_run_python_in_directory(directory, args):
+            # Bundler requires that we run it from the Omnibus project
+            # directory. That messes up any relative paths the user may have
+            # passed on the commandline, so we do a bit of a hack to change
+            # back to the original directory inside the `bundle exec` process.
+            subshell_command = "(cd %s; exec python %s)" % \
+                (pipes.quote(directory), ' '.join(map(pipes.quote, args)))
+            shell_command = "sh -c %s" % pipes.quote(subshell_command)
+            return shell_command
+
+        def reexecute_self_with_bundler(path):
+            script = sys.argv[0]
+
+            logging.info('Reexecuting %s within Bundler, so that extensions '
+                         'use the correct dependencies for Omnibus and the '
+                         'Omnibus project definitions.', script)
+            command = command_to_run_python_in_directory(os.getcwd(), sys.argv)
+
+            logging.debug('Running: `bundle exec %s` in dir %s', command, path)
+            os.chdir(path)
+            os.execvp('bundle', [script, 'exec', command])
+
+        # Omnibus definitions are spread across multiple repos, and there is
+        # no stability guarantee for the definition format. The official advice
+        # is to use Bundler to execute Omnibus, so let's do that.
+        if not running_inside_bundler():
+            reexecute_self_with_bundler(args[0])
+
+        definitions_dir = args[0]
+        project_name = args[1]
+
+        loop = baserockimport.mainloop.ImportLoop(
+            app=self,
+            goal_kind='omnibus', goal_name=args[2], goal_version='master')
+        loop.enable_importer('omnibus',
+                             extra_args=[definitions_dir, project_name])
+        loop.enable_importer('rubygems')
+        loop.run()
+
+    def import_rubygems(self, args):
+        '''Import one or more RubyGems.'''
+        if len(args) != 1:
+            raise cliapp.AppException(
+                'Please pass the name of a RubyGem on the commandline.')
+
+        loop = baserockimport.mainloop.ImportLoop(
+            app=self,
+            goal_kind='rubygems', goal_name=args[0], goal_version='master')
+        loop.enable_importer('rubygems')
+        loop.run()
diff --git a/baserockimport/lorryset.py b/baserockimport/lorryset.py
new file mode 100644
index 0000000..c3d109b
--- /dev/null
+++ b/baserockimport/lorryset.py
@@ -0,0 +1,196 @@
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import morphlib
+import six
+
+import json
+import logging
+import os
+
+
+class LorrySetError(Exception):
+    pass
+
+
+class LorrySet(object):
+    '''Manages a set of .lorry files.
+
+    A LorrySet instance operates on all the .lorry files inside the path given
+    at construction time. This includes .lorry files in subdirectories of the
+    path.
+
+    A lorry *entry* describes the upstream repository for a given project,
+    which is associated to a name such as 'ruby-gems/chef' or 'gcc-tarball'. A
+    lorry *file* contains one or more *entries*. The filename of a lorry file
+    is not necessarily linked to the name of any lorry entry that it contains.
+
+    '''
+
+    def __init__(self, lorries_path):
+        '''Initialise a LorrySet instance for the given directory.
+
+        This will load and parse all of the .lorry files inside 'lorries_path'
+        into memory.
+
+        '''
+        self.path = lorries_path
+
+        if os.path.exists(lorries_path):
+            self.data = self._parse_all_lorries()
+        else:
+            os.makedirs(lorries_path)
+            self.data = {}
+
+    def all_lorry_files(self):
+        '''Return the path of each lorry file in this set.'''
+        for dirpath, dirnames, filenames in os.walk(self.path):
+            for filename in filenames:
+                if filename.endswith('.lorry'):
+                    yield os.path.join(dirpath, filename)
+
+    def _parse_all_lorries(self):
+        lorry_set = {}
+        for lorry_file in self.all_lorry_files():
+            lorry = self._parse_lorry(lorry_file)
+
+            lorry_items = lorry.items()
+
+            for key, value in lorry_items:
+                if key in lorry_set:
+                    raise LorrySetError(
+                        '%s: duplicates existing lorry %s' % (lorry_file, key))
+
+            lorry_set.update(lorry_items)
+
+        return lorry_set
+
+    def _parse_lorry(self, lorry_file):
+        try:
+            with open(lorry_file, 'r') as f:
+                lorry = json.load(f)
+            return lorry
+        except ValueError as e:
+            raise LorrySetError(
+                "Error parsing %s: %s" % (lorry_file, e))
+
+    def get_lorry(self, name):
+        '''Return the lorry entry for the named project.'''
+        return {name: self.data[name]}
+
+    def find_lorry_for_package(self, kind, package_name):
+        '''Find the lorry entry for a given foreign package, or return None.
+
+        This makes use of an extension to the .lorry format made by the
+        Baserock Import tool. Fields follow the form 'x-products-$KIND'
+        and specify the name of a package in the foreign packaging universe
+        named $KIND.
+
+        '''
+        key = 'x-products-%s' % kind
+        for name, lorry in self.data.iteritems():
+            products = lorry.get(key, [])
+            for entry in products:
+                if entry == package_name:
+                    return {name: lorry}
+
+        return None
+
+    def _check_for_conflicts_in_standard_fields(self, existing, new):
+        '''Ensure that two lorries for the same project do actually match.'''
+        for field, value in existing.iteritems():
+            if field.startswith('x-'):
+                continue
+            if field == 'url':
+                # FIXME: need a much better way of detecting whether the URLs
+                # are equivalent ... right now HTTP vs. HTTPS will cause an
+                # error, for example!
+                matches = (value.rstrip('/') == new[field].rstrip('/'))
+            else:
+                matches = (value == new[field])
+            if not matches:
+                raise LorrySetError(
+                    'Lorry %s conflicts with existing entry %s at field %s' %
+                    (new, existing, field))
+
+    def _merge_products_fields(self, existing, new):
+        '''Merge the x-products- fields from new lorry into an existing one.'''
+        is_product_field = lambda x: x.startswith('x-products-')
+
+        existing_fields = [f for f in existing.iterkeys() if
+                           is_product_field(f)]
+        new_fields = [f for f in new.iterkeys() if f not in existing_fields and
+                      is_product_field(f)]
+
+        for field in existing_fields:
+            existing[field].extend(new[field])
+            existing[field] = list(set(existing[field]))
+
+        for field in new_fields:
+            existing[field] = new[field]
+
+    def _add_lorry_entry_to_lorry_file(self, filename, entry):
+        if os.path.exists(filename):
+            with open(filename) as f:
+                contents = json.load(f)
+        else:
+            contents = {}
+
+        contents.update(entry)
+
+        with morphlib.savefile.SaveFile(filename, 'w') as f:
+            json.dump(contents, f, indent=4, separators=(',', ': '),
+                      sort_keys=True)
+
+    def add(self, filename, lorry_entry):
+        '''Add a lorry entry to the named .lorry file.
+
+        The lorry_entry should follow the on-disk format for a lorry stanza,
+        which is a dict of one entry mapping the name of the entry to its
+        contents.
+
+        The .lorry file will be created if it doesn't exist.
+
+        '''
+        logging.debug('Adding %s to lorryset', filename)
+
+        filename = os.path.join(self.path, '%s.lorry' % filename)
+
+        assert len(lorry_entry) == 1
+
+        project_name = lorry_entry.keys()[0]
+        info = lorry_entry.values()[0]
+
+        if len(project_name) == 0:
+            raise LorrySetError(
+                'Invalid lorry %s: %s' % (filename, lorry_entry))
+
+        if not isinstance(info.get('url'), six.string_types):
+            raise LorrySetError(
+                'Invalid URL in lorry %s: %s' % (filename, info.get('url')))
+
+        if project_name in self.data:
+            stored_lorry = self.get_lorry(project_name)
+
+            self._check_for_conflicts_in_standard_fields(
+                stored_lorry[project_name], lorry_entry[project_name])
+            self._merge_products_fields(
+                stored_lorry[project_name], lorry_entry[project_name])
+            lorry_entry = stored_lorry
+        else:
+            self.data[project_name] = info
+
+        self._add_lorry_entry_to_lorry_file(filename, lorry_entry)
diff --git a/main.py b/baserockimport/mainloop.py
index 7f9a1d6..c081be5 100644
--- a/main.py
+++ b/baserockimport/mainloop.py
@@ -1,6 +1,3 @@
-#!/usr/bin/python
-# Import foreign packaging systems into Baserock
-#
 # Copyright (C) 2014  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
@@ -17,212 +14,17 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 
-import ansicolor
 import cliapp
 import morphlib
 import networkx
-import six
 
-import contextlib
-import copy
 import json
 import logging
 import os
-import pipes
-import sys
 import tempfile
 import time
 
-from logging import debug
-
-
-class LorrySet(object):
-    '''Manages a set of .lorry files.
-
-    The structure of .lorry files makes the code a little more confusing than
-    I would like. A lorry "entry" is a dict of one entry mapping name to info.
-    A lorry "file" is a dict of one or more of these entries merged together.
-    If it were a list of entries with 'name' fields, the code would be neater.
-
-    '''
-    def __init__(self, lorries_path):
-        self.path = lorries_path
-
-        if os.path.exists(lorries_path):
-            self.data = self.parse_all_lorries()
-        else:
-            os.makedirs(lorries_path)
-            self.data = {}
-
-    def all_lorry_files(self):
-        for dirpath, dirnames, filenames in os.walk(self.path):
-            for filename in filenames:
-                if filename.endswith('.lorry'):
-                    yield os.path.join(dirpath, filename)
-
-    def parse_all_lorries(self):
-        lorry_set = {}
-        for lorry_file in self.all_lorry_files():
-            lorry = self.parse_lorry(lorry_file)
-
-            lorry_items = lorry.items()
-
-            for key, value in lorry_items:
-                if key in lorry_set:
-                    raise Exception(
-                        '%s: duplicates existing lorry %s' % (lorry_file, key))
-
-            lorry_set.update(lorry_items)
-
-        return lorry_set
-
-    def parse_lorry(self, lorry_file):
-        try:
-            with open(lorry_file, 'r') as f:
-                lorry = json.load(f)
-            return lorry
-        except ValueError as e:
-            raise cliapp.AppException(
-                "Error parsing %s: %s" % (lorry_file, e))
-
-    def get_lorry(self, name):
-        return {name: self.data[name]}
-
-    def find_lorry_for_package(self, kind, package_name):
-        key = 'x-products-%s' % kind
-        for name, lorry in self.data.iteritems():
-            products = lorry.get(key, [])
-            for entry in products:
-                if entry == package_name:
-                    return {name: lorry}
-
-        return None
-
-    def _check_for_conflicts_in_standard_fields(self, existing, new):
-        '''Ensure that two lorries for the same project do actually match.'''
-        for field, value in existing.iteritems():
-            if field.startswith('x-'):
-                continue
-            if field == 'url':
-                # FIXME: need a much better way of detecting whether the URLs
-                # are equivalent ... right now HTTP vs. HTTPS will cause an
-                # error, for example!
-                matches = (value.rstrip('/') == new[field].rstrip('/'))
-            else:
-                matches = (value == new[field])
-            if not matches:
-                raise Exception(
-                    'Lorry %s conflicts with existing entry %s at field %s' %
-                    (new, existing, field))
-
-    def _merge_products_fields(self, existing, new):
-        '''Merge the x-products- fields from new lorry into an existing one.'''
-        is_product_field = lambda x: x.startswith('x-products-')
-
-        existing_fields = [f for f in existing.iterkeys() if
-                           is_product_field(f)]
-        new_fields = [f for f in new.iterkeys() if f not in existing_fields and
-                      is_product_field(f)]
-
-        for field in existing_fields:
-            existing[field].extend(new[field])
-            existing[field] = list(set(existing[field]))
-
-        for field in new_fields:
-            existing[field] = new[field]
-
-    def add(self, filename, lorry_entry):
-        logging.debug('Adding %s to lorryset', filename)
-
-        filename = os.path.join(self.path, '%s.lorry' % filename)
-
-        assert len(lorry_entry) == 1
-
-        project_name = lorry_entry.keys()[0]
-        info = lorry_entry.values()[0]
-
-        if len(project_name) == 0:
-            raise cliapp.AppException(
-                'Invalid lorry %s: %s' % (filename, lorry_entry))
-
-        if not isinstance(info.get('url'), six.string_types):
-            raise cliapp.AppException(
-                'Invalid URL in lorry %s: %s' % (filename, info.get('url')))
-
-        if project_name in self.data:
-            stored_lorry = self.get_lorry(project_name)
-
-            self._check_for_conflicts_in_standard_fields(
-                stored_lorry[project_name], lorry_entry[project_name])
-            self._merge_products_fields(
-                stored_lorry[project_name], lorry_entry[project_name])
-            lorry_entry = stored_lorry
-        else:
-            self.data[project_name] = info
-
-        self._add_lorry_entry_to_lorry_file(filename, lorry_entry)
-
-    def _add_lorry_entry_to_lorry_file(self, filename, entry):
-        if os.path.exists(filename):
-            with open(filename) as f:
-                contents = json.load(f)
-        else:
-            contents = {}
-
-        contents.update(entry)
-
-        with morphlib.savefile.SaveFile(filename, 'w') as f:
-            json.dump(contents, f, indent=4, separators=(',', ': '),
-                      sort_keys=True)
-
-
-class MorphologySet(morphlib.morphset.MorphologySet):
-    def __init__(self, path):
-        super(MorphologySet, self).__init__()
-
-        self.path = path
-        self.loader = morphlib.morphloader.MorphologyLoader()
-
-        if os.path.exists(path):
-            self.load_all_morphologies()
-        else:
-            os.makedirs(path)
-
-    def load_all_morphologies(self):
-        logging.info('Loading all .morph files under %s', self.path)
-
-        class FakeGitDir(morphlib.gitdir.GitDirectory):
-            '''Ugh
-
-            This is here because the default constructor will search up the
-            directory heirarchy until it finds a '.git' directory, but that
-            may be totally the wrong place for our purpose: we don't have a
-            Git directory at all.
-
-            '''
-            def __init__(self, path):
-                self.dirname = path
-                self._config = {}
-
-        gitdir = FakeGitDir(self.path)
-        finder = morphlib.morphologyfinder.MorphologyFinder(gitdir)
-        for filename in (f for f in finder.list_morphologies()
-                         if not gitdir.is_symlink(f)):
-            text = finder.read_morphology(filename)
-            morph = self.loader.load_from_string(text, filename=filename)
-            morph.repo_url = None  # self.root_repository_url
-            morph.ref = None  # self.system_branch_name
-            self.add_morphology(morph)
-
-    def get_morphology(self, repo_url, ref, filename):
-        return self._get_morphology(repo_url, ref, filename)
-
-    def save_morphology(self, filename, morphology):
-        self.add_morphology(morphology)
-        morphology_to_save = copy.copy(morphology)
-        self.loader.unset_defaults(morphology_to_save)
-        filename = os.path.join(self.path, filename)
-        self.loader.save_to_file(filename, morphology_to_save)
+import baserockimport
 
 
 class GitDirectory(morphlib.gitdir.GitDirectory):
@@ -233,6 +35,10 @@ class GitDirectory(morphlib.gitdir.GitDirectory):
         # when 'repopath' isn't a Git repo. If 'repopath' is contained
         # within a Git repo then the GitDirectory will traverse up to the
         # parent repo, which isn't what we want in this case.
+        #
+        # FIXME: this should be a change to the base class, which should take
+        # a flag at construct time saying 'traverse_upwards_to_find_root' or
+        # some such.
         if self.dirname != dirname:
             logging.error(
                 'Got git directory %s for %s!', self.dirname, dirname)
@@ -251,56 +57,6 @@ class BaserockImportException(cliapp.AppException):
     pass
 
 
-class Package(object):
-    '''A package in the processing queue.
-
-    In order to provide helpful errors, this item keeps track of what
-    packages depend on it, and hence of why it was added to the queue.
-
-    '''
-    def __init__(self, kind, name, version):
-        self.kind = kind
-        self.name = name
-        self.version = version
-        self.required_by = []
-        self.morphology = None
-        self.dependencies = None
-        self.is_build_dep = False
-        self.version_in_use = version
-
-    def __cmp__(self, other):
-        return cmp(self.name, other.name)
-
-    def __repr__(self):
-        return '<Package %s-%s>' % (self.name, self.version)
-
-    def __str__(self):
-        if len(self.required_by) > 0:
-            required_msg = ', '.join(self.required_by)
-            required_msg = ', required by: ' + required_msg
-        else:
-            required_msg = ''
-        return '%s-%s%s' % (self.name, self.version, required_msg)
-
-    def add_required_by(self, item):
-        self.required_by.append('%s-%s' % (item.name, item.version))
-
-    def match(self, name, version):
-        return (self.name==name and self.version==version)
-
-    def set_morphology(self, morphology):
-        self.morphology = morphology
-
-    def set_dependencies(self, dependencies):
-        self.dependencies = dependencies
-
-    def set_is_build_dep(self, is_build_dep):
-        self.is_build_dep = is_build_dep
-
-    def set_version_in_use(self, version_in_use):
-        self.version_in_use = version_in_use
-
-
 def find(iterable, match):
     return next((x for x in iterable if match(x)), None)
 
@@ -327,7 +83,7 @@ def run_extension(filename, args, cwd='.'):
     )
 
     # There are better ways of doing this, but it works for now.
-    main_path = os.path.dirname(os.path.realpath(__file__))
+    main_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
     extension_path = os.path.join(main_path, filename)
 
     logging.debug("Running %s %s with cwd %s" % (extension_path, args, cwd))
@@ -359,8 +115,10 @@ class ImportLoop(object):
         self.goal_version = goal_version
         self.extra_args = extra_args
 
-        self.lorry_set = LorrySet(self.app.settings['lorries-dir'])
-        self.morph_set = MorphologySet(self.app.settings['definitions-dir'])
+        self.lorry_set = baserockimport.lorryset.LorrySet(
+            self.app.settings['lorries-dir'])
+        self.morph_set = baserockimport.morphsetondisk.MorphologySetOnDisk(
+            self.app.settings['definitions-dir'])
 
         self.morphloader = morphlib.morphloader.MorphologyLoader()
 
@@ -389,7 +147,8 @@ class ImportLoop(object):
         if not os.path.exists(chunk_dir):
             os.makedirs(chunk_dir)
 
-        goal = Package(self.goal_kind, self.goal_name, self.goal_version)
+        goal = baserockimport.package.Package(
+            self.goal_kind, self.goal_name, self.goal_version)
         to_process = [goal]
         processed = networkx.DiGraph()
 
@@ -488,7 +247,8 @@ class ImportLoop(object):
                 queue_item = find(
                     to_process, lambda i: i.match(dep_name, dep_version))
                 if queue_item is None:
-                    queue_item = Package(kind, dep_name, dep_version)
+                    queue_item = baserockimport.package.Package(
+                        kind, dep_name, dep_version)
                     to_process.append(queue_item)
                 dep_package = queue_item
 
@@ -544,7 +304,7 @@ class ImportLoop(object):
         lorry_text = run_extension(tool, extra_args + [name])
         try:
             lorry = json.loads(lorry_text)
-        except ValueError as e:
+        except ValueError:
             raise cliapp.AppException(
                 'Invalid output from %s: %s' % (tool, lorry_text))
         return lorry
@@ -733,7 +493,7 @@ class ImportLoop(object):
         order = reversed(sorted(graph.nodes()))
         try:
             return networkx.topological_sort(graph, nbunch=order)
-        except networkx.NetworkXUnfeasible as e:
+        except networkx.NetworkXUnfeasible:
             # Cycle detected!
             loop_subgraphs = networkx.strongly_connected_component_subgraphs(
                 graph, copy=False)
@@ -751,11 +511,12 @@ class ImportLoop(object):
             self.app.settings['definitions-dir'], 'strata', '%s.morph' %
             goal_name)
 
-        if os.path.exists(filename) and not self.app.settings['update-existing']:
-            self.app.status(
-                msg='Found stratum morph for %s at %s, not overwriting' %
-                (goal_name, filename))
-            return
+        if os.path.exists(filename):
+            if not self.app.settings['update-existing']:
+                self.app.status(
+                    msg='Found stratum morph for %s at %s, not overwriting' %
+                    (goal_name, filename))
+                return
 
         self.app.status(msg='Generating stratum morph for %s' % goal_name)
 
@@ -800,160 +561,3 @@ class ImportLoop(object):
             json.dumps(stratum), filename=filename)
         self.morphloader.unset_defaults(morphology)
         self.morphloader.save_to_file(filename, morphology)
-
-
-class BaserockImportApplication(cliapp.Application):
-    def add_settings(self):
-        self.settings.string(['lorries-dir'],
-                             "location for Lorry files",
-                             metavar="PATH",
-                             default=os.path.abspath('./lorries'))
-        self.settings.string(['definitions-dir'],
-                             "location for morphology files",
-                             metavar="PATH",
-                             default=os.path.abspath('./definitions'))
-        self.settings.string(['checkouts-dir'],
-                             "location for Git checkouts",
-                             metavar="PATH",
-                             default=os.path.abspath('./checkouts'))
-        self.settings.string(['lorry-working-dir'],
-                             "Lorry working directory",
-                             metavar="PATH",
-                             default=os.path.abspath('./lorry-working-dir'))
-
-        self.settings.boolean(['update-existing'],
-                              "update all the checked-out Git trees and "
-                              "generated definitions",
-                              default=False)
-        self.settings.boolean(['use-local-sources'],
-                              "use file:/// URLs in the stratum 'repo' "
-                              "fields, instead of upstream: URLs",
-                              default=False)
-        self.settings.boolean(['use-master-if-no-tag'],
-                              "if the correct tag for a version can't be "
-                              "found, use 'master' instead of raising an "
-                              "error",
-                              default=False)
-
-    def _stream_has_colours(self, stream):
-        # http://blog.mathieu-leplatre.info/colored-output-in-console-with-python.html
-        if not hasattr(stream, "isatty"):
-            return False
-        if not stream.isatty():
-            return False # auto color only on TTYs
-        try:
-            import curses
-            curses.setupterm()
-            return curses.tigetnum("colors") > 2
-        except:
-            # guess false in case of error
-            return False
-
-    def setup(self):
-        self.add_subcommand('omnibus', self.import_omnibus,
-                            arg_synopsis='REPO PROJECT_NAME SOFTWARE_NAME')
-        self.add_subcommand('rubygems', self.import_rubygems,
-                            arg_synopsis='GEM_NAME')
-
-        self.stdout_has_colours = self._stream_has_colours(sys.stdout)
-
-    def setup_logging_formatter_for_file(self):
-        root_logger = logging.getLogger()
-        root_logger.name = 'main'
-
-        # You need recent cliapp for this to work, with commit "Split logging
-        # setup into further overrideable methods".
-        return logging.Formatter("%(name)s: %(levelname)s: %(message)s")
-
-    def process_args(self, args):
-        if len(args) == 0:
-            # Cliapp default is to just say "ERROR: must give subcommand" if
-            # no args are passed, I prefer this.
-            args = ['help']
-
-        super(BaserockImportApplication, self).process_args(args)
-
-    def status(self, msg, *args, **kwargs):
-        text = msg % args
-        if kwargs.get('error') == True:
-            logging.error(text)
-            if self.stdout_has_colours:
-                sys.stdout.write(ansicolor.red(text))
-            else:
-                sys.stdout.write(text)
-        else:
-            logging.info(text)
-            sys.stdout.write(text)
-        sys.stdout.write('\n')
-
-    def import_omnibus(self, args):
-        '''Import a software component from an Omnibus project.
-
-        Omnibus is a tool for generating application bundles for various
-        platforms. See <https://github.com/opscode/omnibus> for more
-        information.
-
-        '''
-        if len(args) != 3:
-            raise cliapp.AppException(
-                'Please give the location of the Omnibus definitions repo, '
-                'and the name of the project and the top-level software '
-                'component.')
-
-        def running_inside_bundler():
-            return 'BUNDLE_GEMFILE' in os.environ
-
-        def command_to_run_python_in_directory(directory, args):
-            # Bundler requires that we run it from the Omnibus project
-            # directory. That messes up any relative paths the user may have
-            # passed on the commandline, so we do a bit of a hack to change
-            # back to the original directory inside the `bundle exec` process.
-            subshell_command = "(cd %s; exec python %s)" % \
-                (pipes.quote(directory), ' '.join(map(pipes.quote, args)))
-            shell_command = "sh -c %s" % pipes.quote(subshell_command)
-            return shell_command
-
-        def reexecute_self_with_bundler(path):
-            script = sys.argv[0]
-
-            logging.info('Reexecuting %s within Bundler, so that extensions '
-                         'use the correct dependencies for Omnibus and the '
-                         'Omnibus project definitions.', script)
-            command = command_to_run_python_in_directory(os.getcwd(), sys.argv)
-
-            logging.debug('Running: `bundle exec %s` in dir %s', command, path)
-            os.chdir(path)
-            os.execvp('bundle', [script, 'exec', command])
-
-        # Omnibus definitions are spread across multiple repos, and there is
-        # no stability guarantee for the definition format. The official advice
-        # is to use Bundler to execute Omnibus, so let's do that.
-        if not running_inside_bundler():
-            reexecute_self_with_bundler(args[0])
-
-        definitions_dir = args[0]
-        project_name = args[1]
-
-        loop = ImportLoop(
-            app=self,
-            goal_kind='omnibus', goal_name=args[2], goal_version='master')
-        loop.enable_importer('omnibus',
-                             extra_args=[definitions_dir, project_name])
-        loop.enable_importer('rubygems')
-        loop.run()
-
-    def import_rubygems(self, args):
-        '''Import one or more RubyGems.'''
-        if len(args) != 1:
-            raise cliapp.AppException(
-                'Please pass the name of a RubyGem on the commandline.')
-
-        loop = ImportLoop(
-            app=self,
-            goal_kind='rubygems', goal_name=args[0], goal_version='master')
-        loop.enable_importer('rubygems')
-        loop.run()
-
-
-app = BaserockImportApplication(progname='import')
-app.run()
diff --git a/baserockimport/morphsetondisk.py b/baserockimport/morphsetondisk.py
new file mode 100644
index 0000000..565ad24
--- /dev/null
+++ b/baserockimport/morphsetondisk.py
@@ -0,0 +1,79 @@
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import morphlib
+
+import copy
+import logging
+import os
+
+
+class MorphologySetOnDisk(morphlib.morphset.MorphologySet):
+    '''Extensions to the morphlib.MorphologySet class.
+
+    The base class deals only with reading morphologies into memory. This class
+    extends it to support reading and writing them from disk.
+
+    FIXME: this should perhaps be merged into the base class in morphlib.
+
+    '''
+
+    def __init__(self, path):
+        super(MorphologySetOnDisk, self).__init__()
+
+        self.path = path
+        self.loader = morphlib.morphloader.MorphologyLoader()
+
+        if os.path.exists(path):
+            self.load_all_morphologies()
+        else:
+            os.makedirs(path)
+
+    def load_all_morphologies(self):
+        logging.info('Loading all .morph files under %s', self.path)
+
+        class FakeGitDir(morphlib.gitdir.GitDirectory):
+            '''FIXME: Ugh
+
+            This is here because the default constructor will search up the
+            directory heirarchy until it finds a '.git' directory, but that
+            may be totally the wrong place for our purpose: we don't have a
+            Git directory at all.
+
+            '''
+            def __init__(self, path):
+                self.dirname = path
+                self._config = {}
+
+        gitdir = FakeGitDir(self.path)
+        finder = morphlib.morphologyfinder.MorphologyFinder(gitdir)
+        for filename in (f for f in finder.list_morphologies()
+                         if not gitdir.is_symlink(f)):
+            text = finder.read_morphology(filename)
+            morph = self.loader.load_from_string(text, filename=filename)
+            morph.repo_url = None  # self.root_repository_url
+            morph.ref = None  # self.system_branch_name
+            self.add_morphology(morph)
+
+    def get_morphology(self, repo_url, ref, filename):
+        return self._get_morphology(repo_url, ref, filename)
+
+    def save_morphology(self, filename, morphology):
+        self.add_morphology(morphology)
+        morphology_to_save = copy.copy(morphology)
+        self.loader.unset_defaults(morphology_to_save)
+        filename = os.path.join(self.path, filename)
+        self.loader.save_to_file(filename, morphology_to_save)
diff --git a/baserockimport/package.py b/baserockimport/package.py
new file mode 100644
index 0000000..6095b30
--- /dev/null
+++ b/baserockimport/package.py
@@ -0,0 +1,68 @@
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+class Package(object):
+    '''A package in the processing queue.
+
+    In order to provide helpful errors, this item keeps track of what
+    packages depend on it, and hence of why it was added to the queue.
+
+    '''
+    def __init__(self, kind, name, version):
+        self.kind = kind
+        self.name = name
+        self.version = version
+        self.required_by = []
+        self.morphology = None
+        self.dependencies = None
+        self.is_build_dep = False
+        self.version_in_use = version
+
+    def __cmp__(self, other):
+        return cmp(self.name, other.name)
+
+    def __repr__(self):
+        return '<Package %s-%s>' % (self.name, self.version)
+
+    def __str__(self):
+        if len(self.required_by) > 0:
+            required_msg = ', '.join(self.required_by)
+            required_msg = ', required by: ' + required_msg
+        else:
+            required_msg = ''
+        return '%s-%s%s' % (self.name, self.version, required_msg)
+
+    def add_required_by(self, item):
+        self.required_by.append('%s-%s' % (item.name, item.version))
+
+    def match(self, name, version):
+        return (self.name==name and self.version==version)
+
+    # FIXME: these accessors are useless, but I want there to be some way
+    # of making it clear that some of the state of the Package object is
+    # mutable and some of the state is not ...
+
+    def set_morphology(self, morphology):
+        self.morphology = morphology
+
+    def set_dependencies(self, dependencies):
+        self.dependencies = dependencies
+
+    def set_is_build_dep(self, is_build_dep):
+        self.is_build_dep = is_build_dep
+
+    def set_version_in_use(self, version_in_use):
+        self.version_in_use = version_in_use