Store dependency info in a separate file, not in the chunk morphology

This means that the tool can work without requiring changes to Morph to
allow new fields in morphologies. It also makes it easier to throw away
the foreign dependency information once an import is complete.

The downside is that there are now 3 types of extension, not two, and
a lot of code needed factoring out into base classes (and more needs
factoring out still). There are now .to_lorry, .to_chunk and .find_deps
extensions.

8 files changed, 521 insertions, 287 deletions

diff --git a/importer_base.rb b/importer_base.rb
index 4e7a7b5..b16b436 100644
--- a/importer_base.rb
+++ b/importer_base.rb
@@ -63,6 +63,11 @@ module Importer
       file.write(YAML.dump(morph))
     end
 
+    def write_dependencies(file, dependencies)
+      format_options = { :indent => '    ' }
+      file.puts(JSON.pretty_generate(dependencies, format_options))
+    end
+
     def create_logger
       # Use the logger that was passed in from the 'main' import process, if
       # detected.
diff --git a/importer_bundler_extensions.rb b/importer_bundler_extensions.rb
new file mode 100644
index 0000000..90a5ae4
--- /dev/null
+++ b/importer_bundler_extensions.rb
@@ -0,0 +1,89 @@
+#!/usr/bin/env ruby
+#
+# Extensions to Bundler library which allow using it in importers.
+#
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+require 'bundler'
+
+class << Bundler
+  def default_gemfile
+    # This is a hack to make things not crash when there's no Gemfile
+    Pathname.new('.')
+  end
+end
+
+module Importer
+  module BundlerExtensions
+    def create_bundler_definition_for_gemspec(gem_name)
+      # Using the real Gemfile doesn't get great results, because people can put
+      # lots of stuff in there that is handy for developers to have but
+      # irrelevant if you just want to produce a .gem. Also, there is only one
+      # Gemfile per repo, but a repo may include multiple .gemspecs that we want
+      # to process individually. Also, some projects don't use Bundler and may
+      # not have a Gemfile at all.
+      #
+      # Instead of reading the real Gemfile, invent one that simply includes the
+      # chosen .gemspec. If present, the Gemfile.lock will be honoured.
+      fake_gemfile = Bundler::Dsl.new
+      fake_gemfile.source('https://rubygems.org')
+      begin
+        fake_gemfile.gemspec({:name => gem_name})
+      rescue Bundler::InvalidOption
+        error "Did not find #{gem_name}.gemspec in current directory."
+        exit 1
+      end
+
+      fake_gemfile.to_definition('Gemfile.lock', true)
+    end
+
+    def get_spec_for_gem(specs, gem_name)
+      found = specs[gem_name].select {|s| Gem::Platform.match(s.platform)}
+      if found.empty?
+        raise Exception,
+          "No Gemspecs found matching '#{gem_name}'"
+      elsif found.length != 1
+        raise Exception,
+          "Unsure which Gem to use for #{gem_name}, got #{found}"
+      end
+      found[0]
+    end
+
+    def spec_is_from_current_source_tree(spec, source_dir)
+      Dir.chdir(source_dir) do
+        spec.source.instance_of? Bundler::Source::Path and
+          File.identical?(spec.source.path, '.')
+      end
+    end
+
+    def validate_spec(spec, source_dir_name, expected_version)
+      if not spec_is_from_current_source_tree(spec, source_dir_name)
+        error "Specified gem '#{spec.name}' doesn't live in the source in " +
+              "'#{source_dir_name}'"
+        log.debug "SPEC: #{spec.inspect} #{spec.source}"
+        exit 1
+      end
+
+      if expected_version != nil && spec.version != expected_version
+        # This check is brought to you by Coderay, which changes its version
+        # number based on an environment variable. Other Gems may do this too.
+        error "Source in #{source_dir_name} produces #{spec.full_name}, but " +
+              "the expected version was #{expected_version}."
+        exit 1
+      end
+    end
+  end
+end
diff --git a/importer_omnibus_extensions.rb b/importer_omnibus_extensions.rb
new file mode 100644
index 0000000..ce0d780
--- /dev/null
+++ b/importer_omnibus_extensions.rb
@@ -0,0 +1,93 @@
+# Extensions for the Omnibus tool that allow using it to generate morphologies.
+#
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+require 'omnibus'
+
+require 'optparse'
+require 'rubygems/commands/build_command'
+require 'rubygems/commands/install_command'
+require 'shellwords'
+
+class Omnibus::Builder
+  # It's possible to use `gem install` in build commands, which is a great
+  # way of subverting the dependency tracking Omnibus provides. It's done
+  # in `omnibus-chef/config/software/chefdk.rb`, for example.
+  #
+  # To handle this, here we extend the class that executes the build commands
+  # to detect when `gem install` is run. It uses the Gem library to turn the
+  # commandline back into a Bundler::Dependency object that we can use.
+  #
+  # We also trap `gem build` so we know when a software component is a RubyGem
+  # that should be handled by 'rubygems.to_chunk'.
+
+  class GemBuildCommandParser < Gem::Commands::BuildCommand
+    def gemspec_path(args)
+      handle_options args
+      if options[:args].length != 1
+        raise Exception, "Invalid `gem build` commandline: 1 argument " +
+                         "expected, got #{options[:args]}."
+      end
+      options[:args][0]
+    end
+  end
+
+  class GemInstallCommandParser < Gem::Commands::InstallCommand
+    def dependency_list_from_commandline(args)
+      handle_options args
+
+      # `gem install foo*` is sometimes used when installing a locally built
+      # Gem, to avoid needing to know the exact version number that was built.
+      # We only care about remote Gems being installed, so anything with a '*'
+      # in its name can be ignored.
+      gem_names = options[:args].delete_if { |name| name.include?('*') }
+
+      gem_names.collect do |gem_name|
+        Bundler::Dependency.new(gem_name, options[:version])
+      end
+    end
+  end
+
+  def gem(command, options = {})
+    # This function re-implements the 'gem' function in the build-commands DSL.
+    if command.start_with? 'build'
+      parser = GemBuildCommandParser.new
+      args = Shellwords.split(command).drop(1)
+      if built_gemspec != nil
+        raise Exception, "More than one `gem build` command was run as part " +
+                         "of the build process. The 'rubygems.to_chunk' " +
+                         "program currently supports only one .gemspec " +
+                         "build per chunk, so this can't be processed " +
+                         "automatically."
+      end
+      @built_gemspec = parser.gemspec_path(args)
+    elsif command.start_with? 'install'
+      parser = GemInstallCommandParser.new
+      args = Shellwords.split(command).drop(1)
+      args_without_build_flags = args.take_while { |item| item != '--' }
+      gems = parser.dependency_list_from_commandline(args_without_build_flags)
+      manually_installed_rubygems.concat gems
+    end
+  end
+
+  def built_gemspec
+    @built_gemspec
+  end
+
+  def manually_installed_rubygems
+    @manually_installed_rubygems ||= []
+  end
+end
diff --git a/main.py b/main.py
index b5ebece..7f9a1d6 100644
--- a/main.py
+++ b/main.py
@@ -264,6 +264,7 @@ class Package(object):
         self.version = version
         self.required_by = []
         self.morphology = None
+        self.dependencies = None
         self.is_build_dep = False
         self.version_in_use = version
 
@@ -290,6 +291,9 @@ class Package(object):
     def set_morphology(self, morphology):
         self.morphology = morphology
 
+    def set_dependencies(self, dependencies):
+        self.dependencies = dependencies
+
     def set_is_build_dep(self, is_build_dep):
         self.is_build_dep = is_build_dep
 
@@ -405,8 +409,8 @@ class ImportLoop(object):
             processed.add_node(current_item)
 
             if not error:
-                self._process_dependencies_from_morphology(
-                    current_item, current_item.morphology, to_process,
+                self._process_dependencies(
+                    current_item, current_item.dependencies, to_process,
                     processed)
 
         if len(errors) > 0:
@@ -448,34 +452,25 @@ class ImportLoop(object):
 
         package.set_morphology(chunk_morph)
 
-    def _process_dependencies_from_morphology(self, current_item, morphology,
-                                              to_process, processed):
-        '''Enqueue all dependencies of a package that are yet to be processed.
+        dependencies = self._find_or_create_dependency_list(
+            kind, name, checked_out_version, source_repo)
 
-        Dependencies are communicated using extra fields in morphologies,
-        currently.
+        package.set_dependencies(dependencies)
 
-        '''
-        for key, value in morphology.iteritems():
-            if key.startswith('x-build-dependencies-'):
-                kind = key[len('x-build-dependencies-'):]
-                is_build_deps = True
-            elif key.startswith('x-runtime-dependencies-'):
-                kind = key[len('x-runtime-dependencies-'):]
-                is_build_deps = False
-            else:
-                continue
+    def _process_dependencies(self, current_item, dependencies, to_process,
+                              processed):
+        '''Enqueue all dependencies of a package that are yet to be processed.
 
-            # We need to validate this field because it doesn't go through the
-            # normal MorphologyFactory validation, being an extension.
-            if not hasattr(value, 'iteritems'):
-                value_type = type(value).__name__
-                raise cliapp.AppException(
-                    "Morphology for %s has invalid '%s': should be a dict, but "
-                    "got a %s." % (morphology['name'], key, value_type))
+        '''
+        for key, value in dependencies.iteritems():
+            kind = key
 
             self._process_dependency_list(
-                current_item, kind, value, to_process, processed, is_build_deps)
+                current_item, kind, value['build-dependencies'], to_process,
+                processed, True)
+            self._process_dependency_list(
+                current_item, kind, value['runtime-dependencies'], to_process,
+                processed, False)
 
     def _process_dependency_list(self, current_item, kind, deps, to_process,
                                  processed, these_are_build_deps):
@@ -690,6 +685,50 @@ class ImportLoop(object):
 
         return self.morphloader.load_from_string(text, filename)
 
+    def _find_or_create_dependency_list(self, kind, name, version,
+                                        source_repo):
+        depends_filename = 'strata/%s/%s-%s.foreign-dependencies' % (
+            self.goal_name, name, version)
+        depends_path = os.path.join(
+            self.app.settings['definitions-dir'], depends_filename)
+
+        def calculate_dependencies():
+            dependencies = self._calculate_dependencies_for_package(
+                source_repo, kind, name, version, depends_path)
+            with open(depends_path, 'w') as f:
+                json.dump(dependencies, f)
+            return dependencies
+
+        if self.app.settings['update-existing']:
+            dependencies = calculate_dependencies()
+        elif os.path.exists(depends_path):
+            with open(depends_path) as f:
+                dependencies = json.load(f)
+        else:
+            logging.debug("Didn't find %s", depends_path)
+            dependencies = calculate_dependencies()
+
+        return dependencies
+
+    def _calculate_dependencies_for_package(self, source_repo, kind, name,
+                                            version, filename):
+        tool = '%s.find_deps' % kind
+
+        if kind not in self.importers:
+            raise Exception('Importer for %s was not enabled.' % kind)
+        extra_args = self.importers[kind]['extra_args']
+
+        self.app.status(
+            'Calling %s to calculate dependencies for %s %s', tool, name,
+            version)
+
+        args = extra_args + [source_repo.dirname, name]
+        if version != 'master':
+            args.append(version)
+        text = run_extension(tool, args)
+
+        return json.loads(text)
+
     def _sort_chunks_by_build_order(self, graph):
         order = reversed(sorted(graph.nodes()))
         try:
diff --git a/omnibus.find_deps b/omnibus.find_deps
new file mode 100755
index 0000000..8fea31d
--- /dev/null
+++ b/omnibus.find_deps
@@ -0,0 +1,138 @@
+#!/usr/bin/env ruby
+#
+# Find dependencies for an Omnibus software component.
+#
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+require 'bundler'
+
+require_relative 'importer_base'
+require_relative 'importer_omnibus_extensions'
+
+BANNER = "Usage: omnibus.find_deps PROJECT_DIR PROJECT_NAME SOURCE_DIR SOFTWARE_NAME"
+
+DESCRIPTION = <<-END
+Calculate dependencies for a given Omnibus software component.
+END
+
+class OmnibusDependencyFinder < Importer::Base
+  def initialize
+    local_data = YAML.load_file(local_data_path("omnibus.yaml"))
+    @dependency_blacklist = local_data['dependency-blacklist']
+  end
+
+  def parse_options(arguments)
+    opts = create_option_parser(BANNER, DESCRIPTION)
+
+    parsed_arguments = opts.parse!(arguments)
+
+    if parsed_arguments.length != 4 and parsed_arguments.length != 5
+      STDERR.puts "Expected 4 or 5 arguments, got #{parsed_arguments}."
+      opts.parse(['-?'])
+      exit 255
+    end
+
+    project_dir, project_name, source_dir, software_name, expected_version = \
+      parsed_arguments
+    # Not yet implemented
+    #if expected_version != nil
+    #  expected_version = Gem::Version.new(expected_version)
+    #end
+    [project_dir, project_name, source_dir, software_name, expected_version]
+  end
+
+  def resolve_rubygems_deps(requirements)
+    return {} if requirements.empty?
+
+    log.info('Resolving RubyGem requirements with Bundler')
+
+    fake_gemfile = Bundler::Dsl.new
+    fake_gemfile.source('https://rubygems.org')
+
+    requirements.each do |dep|
+      fake_gemfile.gem(dep.name, dep.requirement)
+    end
+
+    definition = fake_gemfile.to_definition('Gemfile.lock', true)
+    resolved_specs = definition.resolve_remotely!
+
+    Hash[resolved_specs.collect { |spec| [spec.name, spec.version.to_s]}]
+  end
+
+  def calculate_dependencies_for_software(project, software, source_dir)
+    omnibus_deps = {}
+    rubygems_deps = {}
+
+    software.dependencies.each do |name|
+      software = Omnibus::Software.load(project, name)
+      if @dependency_blacklist.member? name
+        log.info(
+          "Not adding #{name} as a dependency as it is marked to be ignored.")
+      elsif software.fetcher.instance_of?(Omnibus::PathFetcher)
+        log.info(
+          "Not adding #{name} as a dependency: it's installed from " +
+          "a path which probably means that it is package configuration, not " +
+          "a 3rd-party component to be imported.")
+      elsif software.fetcher.instance_of?(Omnibus::NullFetcher)
+        if software.builder.built_gemspec
+          log.info(
+            "Adding #{name} as a RubyGem dependency because it builds " +
+            "#{software.builder.built_gemspec}")
+          rubygems_deps[name] = software.version
+        else
+          log.info(
+            "Not adding #{name} as a dependency: no sources listed.")
+        end
+      else
+        omnibus_deps[name] = software.version
+      end
+    end
+
+    gem_requirements = software.builder.manually_installed_rubygems
+    rubygems_deps = resolve_rubygems_deps(gem_requirements)
+
+    {
+      "omnibus" => {
+        # FIXME: are these build or runtime dependencies? We'll assume both.
+        "build-dependencies" => omnibus_deps,
+        "runtime-dependencies" => omnibus_deps,
+      },
+      "rubygems" => {
+        "build-dependencies" => {},
+        "runtime-dependencies" => rubygems_deps,
+      }
+    }
+  end
+
+  def run
+    project_dir, project_name, source_dir, software_name = parse_options(ARGV)
+
+    log.info("Calculating dependencies for #{software_name} from project " +
+             "#{project_name}, defined in #{project_dir}")
+
+    Dir.chdir(project_dir)
+
+    project = Omnibus::Project.load(project_name)
+
+    software = Omnibus::Software.load(@project, software_name)
+
+    dependencies = calculate_dependencies_for_software(
+      project, software, source_dir)
+    write_dependencies(STDOUT, dependencies)
+  end
+end
+
+OmnibusDependencyFinder.new.run
diff --git a/omnibus.to_chunk b/omnibus.to_chunk
index 1189199..5e527a9 100755
--- a/omnibus.to_chunk
+++ b/omnibus.to_chunk
@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 #
-# Create a chunk morphology to integrate Omnibus software in Baserock
+# Create a chunk morphology to build Omnibus software in Baserock
 #
 # Copyright (C) 2014  Codethink Limited
 #
@@ -17,15 +17,8 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-require 'bundler'
-require 'omnibus'
-
-require 'optparse'
-require 'rubygems/commands/build_command'
-require 'rubygems/commands/install_command'
-require 'shellwords'
-
 require_relative 'importer_base'
+require_relative 'importer_omnibus_extensions'
 
 BANNER = "Usage: omnibus.to_chunk PROJECT_DIR PROJECT_NAME SOURCE_DIR SOFTWARE_NAME"
 
@@ -33,82 +26,7 @@ DESCRIPTION = <<-END
 Generate a .morph file for a given Omnibus software component.
 END
 
-class Omnibus::Builder
-  # It's possible to use `gem install` in build commands, which is a great
-  # way of subverting the dependency tracking Omnibus provides. It's done
-  # in `omnibus-chef/config/software/chefdk.rb`, for example.
-  #
-  # To handle this, here we extend the class that executes the build commands
-  # to detect when `gem install` is run. It uses the Gem library to turn the
-  # commandline back into a Bundler::Dependency object that we can use.
-  #
-  # We also trap `gem build` so we know when a software component is a RubyGem
-  # that should be handled by 'rubygems.to_chunk'.
-
-  class GemBuildCommandParser < Gem::Commands::BuildCommand
-    def gemspec_path(args)
-      handle_options args
-      if options[:args].length != 1
-        raise Exception, "Invalid `gem build` commandline: 1 argument " +
-                         "expected, got #{options[:args]}."
-      end
-      options[:args][0]
-    end
-  end
-
-  class GemInstallCommandParser < Gem::Commands::InstallCommand
-    def dependency_list_from_commandline(args)
-      handle_options args
-
-      # `gem install foo*` is sometimes used when installing a locally built
-      # Gem, to avoid needing to know the exact version number that was built.
-      # We only care about remote Gems being installed, so anything with a '*'
-      # in its name can be ignored.
-      gem_names = options[:args].delete_if { |name| name.include?('*') }
-
-      gem_names.collect do |gem_name|
-        Bundler::Dependency.new(gem_name, options[:version])
-      end
-    end
-  end
-
-  def gem(command, options = {})
-    # This function re-implements the 'gem' function in the build-commands DSL.
-    if command.start_with? 'build'
-      parser = GemBuildCommandParser.new
-      args = Shellwords.split(command).drop(1)
-      if built_gemspec != nil
-        raise Exception, "More than one `gem build` command was run as part " +
-                         "of the build process. The 'rubygems.to_chunk' " +
-                         "program currently supports only one .gemspec " +
-                         "build per chunk, so this can't be processed " +
-                         "automatically."
-      end
-      @built_gemspec = parser.gemspec_path(args)
-    elsif command.start_with? 'install'
-      parser = GemInstallCommandParser.new
-      args = Shellwords.split(command).drop(1)
-      args_without_build_flags = args.take_while { |item| item != '--' }
-      gems = parser.dependency_list_from_commandline(args_without_build_flags)
-      manually_installed_rubygems.concat gems
-    end
-  end
-
-  def built_gemspec
-    @built_gemspec
-  end
-
-  def manually_installed_rubygems
-    @manually_installed_rubygems ||= []
-  end
-end
-
 class OmnibusChunkMorphologyGenerator < Importer::Base
-  def initialize
-    local_data = YAML.load_file(local_data_path("omnibus.yaml"))
-    @dependency_blacklist = local_data['dependency-blacklist']
-  end
-
   def parse_options(arguments)
     opts = create_option_parser(BANNER, DESCRIPTION)
 
@@ -133,6 +51,7 @@ class OmnibusChunkMorphologyGenerator < Importer::Base
   end
 
   def run_tool_capture_output(tool_name, *args)
+    scripts_dir = local_data_path('.')
     tool_path = local_data_path(tool_name)
 
     # FIXME: something breaks when we try to share this FD, it's not
@@ -171,24 +90,6 @@ class OmnibusChunkMorphologyGenerator < Importer::Base
     exit 1
   end
 
-  def resolve_rubygems_deps(requirements)
-    return {} if requirements.empty?
-
-    log.info('Resolving RubyGem requirements with Bundler')
-
-    fake_gemfile = Bundler::Dsl.new
-    fake_gemfile.source('https://rubygems.org')
-
-    requirements.each do |dep|
-      fake_gemfile.gem(dep.name, dep.requirement)
-    end
-
-    definition = fake_gemfile.to_definition('Gemfile.lock', true)
-    resolved_specs = definition.resolve_remotely!
-
-    Hash[resolved_specs.collect { |spec| [spec.name, spec.version.to_s]}]
-  end
-
   def generate_chunk_morph_for_software(project, software, source_dir)
     if software.builder.built_gemspec != nil
       morphology = generate_chunk_morph_for_rubygems_software(software,
@@ -201,49 +102,9 @@ class OmnibusChunkMorphologyGenerator < Importer::Base
       }
     end
 
-    omnibus_deps = {}
-    rubygems_deps = {}
-
-    software.dependencies.each do |name|
-      software = Omnibus::Software.load(project, name)
-      if @dependency_blacklist.member? name
-        log.info(
-          "Not adding #{name} as a dependency as it is marked to be ignored.")
-      elsif software.fetcher.instance_of?(Omnibus::PathFetcher)
-        log.info(
-          "Not adding #{name} as a dependency: it's installed from " +
-          "a path which probably means that it is package configuration, not " +
-          "a 3rd-party component to be imported.")
-      elsif software.fetcher.instance_of?(Omnibus::NullFetcher)
-        if software.builder.built_gemspec
-          log.info(
-            "Adding #{name} as a RubyGem dependency because it builds " +
-            "#{software.builder.built_gemspec}")
-          rubygems_deps[name] = software.version
-        else
-          log.info(
-            "Not adding #{name} as a dependency: no sources listed.")
-        end
-      else
-        omnibus_deps[name] = software.version
-      end
-    end
-
-    gem_requirements = software.builder.manually_installed_rubygems
-    rubygems_deps = resolve_rubygems_deps(gem_requirements)
-
-    morphology.update({
-      # Possibly this tool should look at software.build and
-      # generate suitable configure, build and install-commands.
-      # For now: don't bother!
-
-      # FIXME: are these build or runtime dependencies? We'll assume both.
-      "x-build-dependencies-omnibus" => omnibus_deps,
-      "x-runtime-dependencies-omnibus" => omnibus_deps,
-
-      "x-build-dependencies-rubygems" => {},
-      "x-runtime-dependencies-rubygems" => rubygems_deps,
-    })
+    # Possibly this tool should look at software.build and
+    # generate suitable configure, build and install-commands.
+    # For now: don't bother!
 
     if software.description
       morphology['description'] = software.description + '\n\n' +
@@ -266,7 +127,6 @@ class OmnibusChunkMorphologyGenerator < Importer::Base
     software = Omnibus::Software.load(@project, software_name)
 
     morph = generate_chunk_morph_for_software(project, software, source_dir)
-
     write_morph(STDOUT, morph)
   end
 end
diff --git a/rubygems.find_deps b/rubygems.find_deps
new file mode 100755
index 0000000..a9f4f08
--- /dev/null
+++ b/rubygems.find_deps
@@ -0,0 +1,114 @@
+#!/usr/bin/env ruby
+#
+# Find dependencies for a RubyGem.
+#
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+require_relative 'importer_base'
+require_relative 'importer_bundler_extensions'
+
+BANNER = "Usage: rubygems.find_deps SOURCE_DIR GEM_NAME [VERSION]"
+
+DESCRIPTION = <<-END
+This tool looks for a .gemspec file for GEM_NAME in SOURCE_DIR, and outputs the
+set of RubyGems dependencies required to build it. It will honour a
+Gemfile.lock file if one is present.
+
+It is intended for use with the `baserock-import` tool.
+END
+
+class RubyGemDependencyFinder < Importer::Base
+  include Importer::BundlerExtensions
+
+  def initialize
+    local_data = YAML.load_file(local_data_path("rubygems.yaml"))
+    @build_dependency_whitelist = local_data['build-dependency-whitelist']
+  end
+
+  def parse_options(arguments)
+    opts = create_option_parser(BANNER, DESCRIPTION)
+
+    parsed_arguments = opts.parse!(arguments)
+
+    if parsed_arguments.length != 2 && parsed_arguments.length != 3
+      STDERR.puts "Expected 2 or 3 arguments, got #{parsed_arguments}."
+      opts.parse(['-?'])
+      exit 255
+    end
+
+    source_dir, gem_name, expected_version = parsed_arguments
+    source_dir = File.absolute_path(source_dir)
+    if expected_version != nil
+      expected_version = Gem::Version.new(expected_version.dup)
+    end
+    [source_dir, gem_name, expected_version]
+  end
+
+  def build_deps_for_gem(spec)
+    deps = spec.dependencies.select do |d|
+      d.type == :development && @build_dependency_whitelist.member?(d.name)
+    end
+  end
+
+  def runtime_deps_for_gem(spec)
+    spec.dependencies.select {|d| d.type == :runtime}
+  end
+
+  def run
+    source_dir_name, gem_name, expected_version = parse_options(ARGV)
+
+    log.info("Finding dependencies for #{gem_name} based on " +
+             "source code in #{source_dir_name}")
+
+    resolved_specs = Dir.chdir(source_dir_name) do
+      definition = create_bundler_definition_for_gemspec(gem_name)
+      definition.resolve_remotely!
+    end
+
+    spec = get_spec_for_gem(resolved_specs, gem_name)
+    validate_spec(spec, source_dir_name, expected_version)
+
+    # One might think that you could use the Bundler::Dependency.groups
+    # field to filter but it doesn't seem to be useful. Instead we go back to
+    # the Gem::Specification of the target Gem and use the dependencies fild
+    # there. We look up each dependency in the resolved_specset to find out
+    # what version Bundler has chosen of it.
+
+    def format_deps(specset, dep_list)
+      info = dep_list.collect do |dep|
+        spec = specset[dep][0]
+        [spec.name, spec.version.to_s]
+      end
+      Hash[info]
+    end
+
+    build_deps = format_deps(
+      resolved_specs, build_deps_for_gem(spec))
+    runtime_deps = format_deps(
+      resolved_specs, runtime_deps_for_gem(spec))
+
+    deps = {
+      'rubygems' => {
+        'build-dependencies' => build_deps,
+        'runtime-dependencies' => runtime_deps,
+      }
+    }
+
+    write_dependencies(STDOUT, deps)
+  end
+end
+
+RubyGemDependencyFinder.new.run
diff --git a/rubygems.to_chunk b/rubygems.to_chunk
index 796fe89..e1f7132 100755
--- a/rubygems.to_chunk
+++ b/rubygems.to_chunk
@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 #
-# Create a chunk morphology to integrate a RubyGem in Baserock
+# Create a chunk morphology to build a RubyGem in Baserock
 #
 # Copyright (C) 2014  Codethink Limited
 #
@@ -20,35 +20,20 @@
 require 'bundler'
 
 require_relative 'importer_base'
-
-class << Bundler
-  def default_gemfile
-    # This is a hack to make things not crash when there's no Gemfile
-    Pathname.new('.')
-  end
-end
-
-def spec_is_from_current_source_tree(spec, source_dir)
-  spec.source.instance_of? Bundler::Source::Path and
-    File.identical?(spec.source.path, source_dir)
-end
+require_relative 'importer_bundler_extensions'
 
 BANNER = "Usage: rubygems.to_chunk SOURCE_DIR GEM_NAME [VERSION]"
 
 DESCRIPTION = <<-END
-This tool reads the Gemfile and optionally the Gemfile.lock from a Ruby project
-source tree in SOURCE_DIR. It outputs a chunk morphology for GEM_NAME on
-stdout. If VERSION is supplied, it is used to check that the build instructions
-will produce the expected version of the Gem.
+This tool looks in SOURCE_DIR to generate a chunk morphology with build
+instructions for GEM_NAME. If VERSION is supplied, it is used to check that the
+build instructions will produce the expected version of the Gem.
 
 It is intended for use with the `baserock-import` tool.
 END
 
 class RubyGemChunkMorphologyGenerator < Importer::Base
-  def initialize
-    local_data = YAML.load_file(local_data_path("rubygems.yaml"))
-    @build_dependency_whitelist = local_data['build-dependency-whitelist']
-  end
+  include Importer::BundlerExtensions
 
   def parse_options(arguments)
     opts = create_option_parser(BANNER, DESCRIPTION)
@@ -69,39 +54,6 @@ class RubyGemChunkMorphologyGenerator < Importer::Base
     [source_dir, gem_name, expected_version]
   end
 
-  def load_local_gemspecs()
-    # Look for .gemspec files in the source repo.
-    #
-    # If there is no .gemspec, but you set 'name' and 'version' then
-    # inside Bundler::Source::Path.load_spec_files this call will create a
-    # fake gemspec matching that name and version. That's probably not useful.
-
-    dir = '.'
-
-    source = Bundler::Source::Path.new({
-      'path' => dir,
-    })
-
-    log.info "Loaded #{source.specs.count} specs from source dir."
-    source.specs.each do |spec|
-      log.debug "  * #{spec.inspect} #{spec.dependencies.inspect}"
-    end
-
-    source
-  end
-
-  def get_spec_for_gem(specs, gem_name)
-    found = specs[gem_name].select {|s| Gem::Platform.match(s.platform)}
-    if found.empty?
-      raise Exception,
-        "No Gemspecs found matching '#{gem_name}'"
-    elsif found.length != 1
-      raise Exception,
-        "Unsure which Gem to use for #{gem_name}, got #{found}"
-    end
-    found[0]
-  end
-
   def chunk_name_for_gemspec(spec)
     # Chunk names are the Gem's "full name" (name + version number), so
     # that we don't break in the rare but possible case that two different
@@ -195,79 +147,23 @@ class RubyGemChunkMorphologyGenerator < Importer::Base
     }
   end
 
-  def build_deps_for_gem(spec)
-    deps = spec.dependencies.select do |d|
-      d.type == :development && @build_dependency_whitelist.member?(d.name)
-    end
-  end
-
-  def runtime_deps_for_gem(spec)
-    spec.dependencies.select {|d| d.type == :runtime}
-  end
-
   def run
     source_dir_name, gem_name, expected_version = parse_options(ARGV)
 
     log.info("Creating chunk morph for #{gem_name} based on " +
              "source code in #{source_dir_name}")
 
-    Dir.chdir(source_dir_name)
-
-    # Instead of reading the real Gemfile, invent one that simply includes the
-    # chosen .gemspec. If present, the Gemfile.lock will be honoured.
-    fake_gemfile = Bundler::Dsl.new
-    fake_gemfile.source('https://rubygems.org')
-    begin
-      fake_gemfile.gemspec({:name => gem_name})
-    rescue Bundler::InvalidOption
-      error "Did not find #{gem_name}.gemspec in #{source_dir_name}"
-      exit 1
+    resolved_specs = Dir.chdir(source_dir_name) do
+      # FIXME: we don't need to do this here, it'd be enough just to load
+      # the given gemspec 
+      definition = create_bundler_definition_for_gemspec(gem_name)
+      definition.resolve_remotely!
     end
 
-    definition = fake_gemfile.to_definition('Gemfile.lock', true)
-    resolved_specs = definition.resolve_remotely!
-
     spec = get_spec_for_gem(resolved_specs, gem_name)
-
-    if not spec_is_from_current_source_tree(spec, source_dir_name)
-      error "Specified gem '#{spec.name}' doesn't live in the source in " +
-            "'#{source_dir_name}'"
-      log.debug "SPEC: #{spec.inspect} #{spec.source}"
-      exit 1
-    end
-
-    if expected_version != nil && spec.version != expected_version
-      # This check is brought to you by Coderay, which changes its version
-      # number based on an environment variable. Other Gems may do this too.
-      error "Source in #{source_dir_name} produces #{spec.full_name}, but " +
-            "the expected version was #{expected_version}."
-      exit 1
-    end
+    validate_spec(spec, source_dir_name, expected_version)
 
     morph = generate_chunk_morph_for_gem(spec)
-
-    # One might think that you could use the Bundler::Dependency.groups
-    # field to filter but it doesn't seem to be useful. Instead we go back to
-    # the Gem::Specification of the target Gem and use the dependencies fild
-    # there. We look up each dependency in the resolved_specset to find out
-    # what version Bundler has chosen of it.
-
-    def format_deps_for_morphology(specset, dep_list)
-      info = dep_list.collect do |dep|
-        spec = specset[dep][0]
-        [spec.name, spec.version.to_s]
-      end
-      Hash[info]
-    end
-
-    build_deps = format_deps_for_morphology(
-      resolved_specs, build_deps_for_gem(spec))
-    runtime_deps = format_deps_for_morphology(
-      resolved_specs, runtime_deps_for_gem(spec))
-
-    morph['x-build-dependencies-rubygems'] = build_deps
-    morph['x-runtime-dependencies-rubygems'] = runtime_deps
-
     write_morph(STDOUT, morph)
   end
 end