diff options
| -rw-r--r-- | importer_base.rb | 5 | ||||
| -rw-r--r-- | importer_bundler_extensions.rb | 89 | ||||
| -rw-r--r-- | importer_omnibus_extensions.rb | 93 | ||||
| -rw-r--r-- | main.py | 89 | ||||
| -rwxr-xr-x | omnibus.find_deps | 138 | ||||
| -rwxr-xr-x | omnibus.to_chunk | 152 | ||||
| -rwxr-xr-x | rubygems.find_deps | 114 | ||||
| -rwxr-xr-x | rubygems.to_chunk | 128 |
8 files changed, 521 insertions, 287 deletions
diff --git a/importer_base.rb b/importer_base.rb index 4e7a7b5..b16b436 100644 --- a/importer_base.rb +++ b/importer_base.rb @@ -63,6 +63,11 @@ module Importer file.write(YAML.dump(morph)) end + def write_dependencies(file, dependencies) + format_options = { :indent => ' ' } + file.puts(JSON.pretty_generate(dependencies, format_options)) + end + def create_logger # Use the logger that was passed in from the 'main' import process, if # detected. diff --git a/importer_bundler_extensions.rb b/importer_bundler_extensions.rb new file mode 100644 index 0000000..90a5ae4 --- /dev/null +++ b/importer_bundler_extensions.rb @@ -0,0 +1,89 @@ +#!/usr/bin/env ruby +# +# Extensions to Bundler library which allow using it in importers. +# +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +require 'bundler' + +class << Bundler + def default_gemfile + # This is a hack to make things not crash when there's no Gemfile + Pathname.new('.') + end +end + +module Importer + module BundlerExtensions + def create_bundler_definition_for_gemspec(gem_name) + # Using the real Gemfile doesn't get great results, because people can put + # lots of stuff in there that is handy for developers to have but + # irrelevant if you just want to produce a .gem. Also, there is only one + # Gemfile per repo, but a repo may include multiple .gemspecs that we want + # to process individually. Also, some projects don't use Bundler and may + # not have a Gemfile at all. + # + # Instead of reading the real Gemfile, invent one that simply includes the + # chosen .gemspec. If present, the Gemfile.lock will be honoured. + fake_gemfile = Bundler::Dsl.new + fake_gemfile.source('https://rubygems.org') + begin + fake_gemfile.gemspec({:name => gem_name}) + rescue Bundler::InvalidOption + error "Did not find #{gem_name}.gemspec in current directory." + exit 1 + end + + fake_gemfile.to_definition('Gemfile.lock', true) + end + + def get_spec_for_gem(specs, gem_name) + found = specs[gem_name].select {|s| Gem::Platform.match(s.platform)} + if found.empty? + raise Exception, + "No Gemspecs found matching '#{gem_name}'" + elsif found.length != 1 + raise Exception, + "Unsure which Gem to use for #{gem_name}, got #{found}" + end + found[0] + end + + def spec_is_from_current_source_tree(spec, source_dir) + Dir.chdir(source_dir) do + spec.source.instance_of? Bundler::Source::Path and + File.identical?(spec.source.path, '.') + end + end + + def validate_spec(spec, source_dir_name, expected_version) + if not spec_is_from_current_source_tree(spec, source_dir_name) + error "Specified gem '#{spec.name}' doesn't live in the source in " + + "'#{source_dir_name}'" + log.debug "SPEC: #{spec.inspect} #{spec.source}" + exit 1 + end + + if expected_version != nil && spec.version != expected_version + # This check is brought to you by Coderay, which changes its version + # number based on an environment variable. Other Gems may do this too. + error "Source in #{source_dir_name} produces #{spec.full_name}, but " + + "the expected version was #{expected_version}." + exit 1 + end + end + end +end diff --git a/importer_omnibus_extensions.rb b/importer_omnibus_extensions.rb new file mode 100644 index 0000000..ce0d780 --- /dev/null +++ b/importer_omnibus_extensions.rb @@ -0,0 +1,93 @@ +# Extensions for the Omnibus tool that allow using it to generate morphologies. +# +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +require 'omnibus' + +require 'optparse' +require 'rubygems/commands/build_command' +require 'rubygems/commands/install_command' +require 'shellwords' + +class Omnibus::Builder + # It's possible to use `gem install` in build commands, which is a great + # way of subverting the dependency tracking Omnibus provides. It's done + # in `omnibus-chef/config/software/chefdk.rb`, for example. + # + # To handle this, here we extend the class that executes the build commands + # to detect when `gem install` is run. It uses the Gem library to turn the + # commandline back into a Bundler::Dependency object that we can use. + # + # We also trap `gem build` so we know when a software component is a RubyGem + # that should be handled by 'rubygems.to_chunk'. + + class GemBuildCommandParser < Gem::Commands::BuildCommand + def gemspec_path(args) + handle_options args + if options[:args].length != 1 + raise Exception, "Invalid `gem build` commandline: 1 argument " + + "expected, got #{options[:args]}." + end + options[:args][0] + end + end + + class GemInstallCommandParser < Gem::Commands::InstallCommand + def dependency_list_from_commandline(args) + handle_options args + + # `gem install foo*` is sometimes used when installing a locally built + # Gem, to avoid needing to know the exact version number that was built. + # We only care about remote Gems being installed, so anything with a '*' + # in its name can be ignored. + gem_names = options[:args].delete_if { |name| name.include?('*') } + + gem_names.collect do |gem_name| + Bundler::Dependency.new(gem_name, options[:version]) + end + end + end + + def gem(command, options = {}) + # This function re-implements the 'gem' function in the build-commands DSL. + if command.start_with? 'build' + parser = GemBuildCommandParser.new + args = Shellwords.split(command).drop(1) + if built_gemspec != nil + raise Exception, "More than one `gem build` command was run as part " + + "of the build process. The 'rubygems.to_chunk' " + + "program currently supports only one .gemspec " + + "build per chunk, so this can't be processed " + + "automatically." + end + @built_gemspec = parser.gemspec_path(args) + elsif command.start_with? 'install' + parser = GemInstallCommandParser.new + args = Shellwords.split(command).drop(1) + args_without_build_flags = args.take_while { |item| item != '--' } + gems = parser.dependency_list_from_commandline(args_without_build_flags) + manually_installed_rubygems.concat gems + end + end + + def built_gemspec + @built_gemspec + end + + def manually_installed_rubygems + @manually_installed_rubygems ||= [] + end +end @@ -264,6 +264,7 @@ class Package(object): self.version = version self.required_by = [] self.morphology = None + self.dependencies = None self.is_build_dep = False self.version_in_use = version @@ -290,6 +291,9 @@ class Package(object): def set_morphology(self, morphology): self.morphology = morphology + def set_dependencies(self, dependencies): + self.dependencies = dependencies + def set_is_build_dep(self, is_build_dep): self.is_build_dep = is_build_dep @@ -405,8 +409,8 @@ class ImportLoop(object): processed.add_node(current_item) if not error: - self._process_dependencies_from_morphology( - current_item, current_item.morphology, to_process, + self._process_dependencies( + current_item, current_item.dependencies, to_process, processed) if len(errors) > 0: @@ -448,34 +452,25 @@ class ImportLoop(object): package.set_morphology(chunk_morph) - def _process_dependencies_from_morphology(self, current_item, morphology, - to_process, processed): - '''Enqueue all dependencies of a package that are yet to be processed. + dependencies = self._find_or_create_dependency_list( + kind, name, checked_out_version, source_repo) - Dependencies are communicated using extra fields in morphologies, - currently. + package.set_dependencies(dependencies) - ''' - for key, value in morphology.iteritems(): - if key.startswith('x-build-dependencies-'): - kind = key[len('x-build-dependencies-'):] - is_build_deps = True - elif key.startswith('x-runtime-dependencies-'): - kind = key[len('x-runtime-dependencies-'):] - is_build_deps = False - else: - continue + def _process_dependencies(self, current_item, dependencies, to_process, + processed): + '''Enqueue all dependencies of a package that are yet to be processed. - # We need to validate this field because it doesn't go through the - # normal MorphologyFactory validation, being an extension. - if not hasattr(value, 'iteritems'): - value_type = type(value).__name__ - raise cliapp.AppException( - "Morphology for %s has invalid '%s': should be a dict, but " - "got a %s." % (morphology['name'], key, value_type)) + ''' + for key, value in dependencies.iteritems(): + kind = key self._process_dependency_list( - current_item, kind, value, to_process, processed, is_build_deps) + current_item, kind, value['build-dependencies'], to_process, + processed, True) + self._process_dependency_list( + current_item, kind, value['runtime-dependencies'], to_process, + processed, False) def _process_dependency_list(self, current_item, kind, deps, to_process, processed, these_are_build_deps): @@ -690,6 +685,50 @@ class ImportLoop(object): return self.morphloader.load_from_string(text, filename) + def _find_or_create_dependency_list(self, kind, name, version, + source_repo): + depends_filename = 'strata/%s/%s-%s.foreign-dependencies' % ( + self.goal_name, name, version) + depends_path = os.path.join( + self.app.settings['definitions-dir'], depends_filename) + + def calculate_dependencies(): + dependencies = self._calculate_dependencies_for_package( + source_repo, kind, name, version, depends_path) + with open(depends_path, 'w') as f: + json.dump(dependencies, f) + return dependencies + + if self.app.settings['update-existing']: + dependencies = calculate_dependencies() + elif os.path.exists(depends_path): + with open(depends_path) as f: + dependencies = json.load(f) + else: + logging.debug("Didn't find %s", depends_path) + dependencies = calculate_dependencies() + + return dependencies + + def _calculate_dependencies_for_package(self, source_repo, kind, name, + version, filename): + tool = '%s.find_deps' % kind + + if kind not in self.importers: + raise Exception('Importer for %s was not enabled.' % kind) + extra_args = self.importers[kind]['extra_args'] + + self.app.status( + 'Calling %s to calculate dependencies for %s %s', tool, name, + version) + + args = extra_args + [source_repo.dirname, name] + if version != 'master': + args.append(version) + text = run_extension(tool, args) + + return json.loads(text) + def _sort_chunks_by_build_order(self, graph): order = reversed(sorted(graph.nodes())) try: diff --git a/omnibus.find_deps b/omnibus.find_deps new file mode 100755 index 0000000..8fea31d --- /dev/null +++ b/omnibus.find_deps @@ -0,0 +1,138 @@ +#!/usr/bin/env ruby +# +# Find dependencies for an Omnibus software component. +# +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +require 'bundler' + +require_relative 'importer_base' +require_relative 'importer_omnibus_extensions' + +BANNER = "Usage: omnibus.find_deps PROJECT_DIR PROJECT_NAME SOURCE_DIR SOFTWARE_NAME" + +DESCRIPTION = <<-END +Calculate dependencies for a given Omnibus software component. +END + +class OmnibusDependencyFinder < Importer::Base + def initialize + local_data = YAML.load_file(local_data_path("omnibus.yaml")) + @dependency_blacklist = local_data['dependency-blacklist'] + end + + def parse_options(arguments) + opts = create_option_parser(BANNER, DESCRIPTION) + + parsed_arguments = opts.parse!(arguments) + + if parsed_arguments.length != 4 and parsed_arguments.length != 5 + STDERR.puts "Expected 4 or 5 arguments, got #{parsed_arguments}." + opts.parse(['-?']) + exit 255 + end + + project_dir, project_name, source_dir, software_name, expected_version = \ + parsed_arguments + # Not yet implemented + #if expected_version != nil + # expected_version = Gem::Version.new(expected_version) + #end + [project_dir, project_name, source_dir, software_name, expected_version] + end + + def resolve_rubygems_deps(requirements) + return {} if requirements.empty? + + log.info('Resolving RubyGem requirements with Bundler') + + fake_gemfile = Bundler::Dsl.new + fake_gemfile.source('https://rubygems.org') + + requirements.each do |dep| + fake_gemfile.gem(dep.name, dep.requirement) + end + + definition = fake_gemfile.to_definition('Gemfile.lock', true) + resolved_specs = definition.resolve_remotely! + + Hash[resolved_specs.collect { |spec| [spec.name, spec.version.to_s]}] + end + + def calculate_dependencies_for_software(project, software, source_dir) + omnibus_deps = {} + rubygems_deps = {} + + software.dependencies.each do |name| + software = Omnibus::Software.load(project, name) + if @dependency_blacklist.member? name + log.info( + "Not adding #{name} as a dependency as it is marked to be ignored.") + elsif software.fetcher.instance_of?(Omnibus::PathFetcher) + log.info( + "Not adding #{name} as a dependency: it's installed from " + + "a path which probably means that it is package configuration, not " + + "a 3rd-party component to be imported.") + elsif software.fetcher.instance_of?(Omnibus::NullFetcher) + if software.builder.built_gemspec + log.info( + "Adding #{name} as a RubyGem dependency because it builds " + + "#{software.builder.built_gemspec}") + rubygems_deps[name] = software.version + else + log.info( + "Not adding #{name} as a dependency: no sources listed.") + end + else + omnibus_deps[name] = software.version + end + end + + gem_requirements = software.builder.manually_installed_rubygems + rubygems_deps = resolve_rubygems_deps(gem_requirements) + + { + "omnibus" => { + # FIXME: are these build or runtime dependencies? We'll assume both. + "build-dependencies" => omnibus_deps, + "runtime-dependencies" => omnibus_deps, + }, + "rubygems" => { + "build-dependencies" => {}, + "runtime-dependencies" => rubygems_deps, + } + } + end + + def run + project_dir, project_name, source_dir, software_name = parse_options(ARGV) + + log.info("Calculating dependencies for #{software_name} from project " + + "#{project_name}, defined in #{project_dir}") + + Dir.chdir(project_dir) + + project = Omnibus::Project.load(project_name) + + software = Omnibus::Software.load(@project, software_name) + + dependencies = calculate_dependencies_for_software( + project, software, source_dir) + write_dependencies(STDOUT, dependencies) + end +end + +OmnibusDependencyFinder.new.run diff --git a/omnibus.to_chunk b/omnibus.to_chunk index 1189199..5e527a9 100755 --- a/omnibus.to_chunk +++ b/omnibus.to_chunk @@ -1,6 +1,6 @@ #!/usr/bin/env ruby # -# Create a chunk morphology to integrate Omnibus software in Baserock +# Create a chunk morphology to build Omnibus software in Baserock # # Copyright (C) 2014 Codethink Limited # @@ -17,15 +17,8 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -require 'bundler' -require 'omnibus' - -require 'optparse' -require 'rubygems/commands/build_command' -require 'rubygems/commands/install_command' -require 'shellwords' - require_relative 'importer_base' +require_relative 'importer_omnibus_extensions' BANNER = "Usage: omnibus.to_chunk PROJECT_DIR PROJECT_NAME SOURCE_DIR SOFTWARE_NAME" @@ -33,82 +26,7 @@ DESCRIPTION = <<-END Generate a .morph file for a given Omnibus software component. END -class Omnibus::Builder - # It's possible to use `gem install` in build commands, which is a great - # way of subverting the dependency tracking Omnibus provides. It's done - # in `omnibus-chef/config/software/chefdk.rb`, for example. - # - # To handle this, here we extend the class that executes the build commands - # to detect when `gem install` is run. It uses the Gem library to turn the - # commandline back into a Bundler::Dependency object that we can use. - # - # We also trap `gem build` so we know when a software component is a RubyGem - # that should be handled by 'rubygems.to_chunk'. - - class GemBuildCommandParser < Gem::Commands::BuildCommand - def gemspec_path(args) - handle_options args - if options[:args].length != 1 - raise Exception, "Invalid `gem build` commandline: 1 argument " + - "expected, got #{options[:args]}." - end - options[:args][0] - end - end - - class GemInstallCommandParser < Gem::Commands::InstallCommand - def dependency_list_from_commandline(args) - handle_options args - - # `gem install foo*` is sometimes used when installing a locally built - # Gem, to avoid needing to know the exact version number that was built. - # We only care about remote Gems being installed, so anything with a '*' - # in its name can be ignored. - gem_names = options[:args].delete_if { |name| name.include?('*') } - - gem_names.collect do |gem_name| - Bundler::Dependency.new(gem_name, options[:version]) - end - end - end - - def gem(command, options = {}) - # This function re-implements the 'gem' function in the build-commands DSL. - if command.start_with? 'build' - parser = GemBuildCommandParser.new - args = Shellwords.split(command).drop(1) - if built_gemspec != nil - raise Exception, "More than one `gem build` command was run as part " + - "of the build process. The 'rubygems.to_chunk' " + - "program currently supports only one .gemspec " + - "build per chunk, so this can't be processed " + - "automatically." - end - @built_gemspec = parser.gemspec_path(args) - elsif command.start_with? 'install' - parser = GemInstallCommandParser.new - args = Shellwords.split(command).drop(1) - args_without_build_flags = args.take_while { |item| item != '--' } - gems = parser.dependency_list_from_commandline(args_without_build_flags) - manually_installed_rubygems.concat gems - end - end - - def built_gemspec - @built_gemspec - end - - def manually_installed_rubygems - @manually_installed_rubygems ||= [] - end -end - class OmnibusChunkMorphologyGenerator < Importer::Base - def initialize - local_data = YAML.load_file(local_data_path("omnibus.yaml")) - @dependency_blacklist = local_data['dependency-blacklist'] - end - def parse_options(arguments) opts = create_option_parser(BANNER, DESCRIPTION) @@ -133,6 +51,7 @@ class OmnibusChunkMorphologyGenerator < Importer::Base end def run_tool_capture_output(tool_name, *args) + scripts_dir = local_data_path('.') tool_path = local_data_path(tool_name) # FIXME: something breaks when we try to share this FD, it's not @@ -171,24 +90,6 @@ class OmnibusChunkMorphologyGenerator < Importer::Base exit 1 end - def resolve_rubygems_deps(requirements) - return {} if requirements.empty? - - log.info('Resolving RubyGem requirements with Bundler') - - fake_gemfile = Bundler::Dsl.new - fake_gemfile.source('https://rubygems.org') - - requirements.each do |dep| - fake_gemfile.gem(dep.name, dep.requirement) - end - - definition = fake_gemfile.to_definition('Gemfile.lock', true) - resolved_specs = definition.resolve_remotely! - - Hash[resolved_specs.collect { |spec| [spec.name, spec.version.to_s]}] - end - def generate_chunk_morph_for_software(project, software, source_dir) if software.builder.built_gemspec != nil morphology = generate_chunk_morph_for_rubygems_software(software, @@ -201,49 +102,9 @@ class OmnibusChunkMorphologyGenerator < Importer::Base } end - omnibus_deps = {} - rubygems_deps = {} - - software.dependencies.each do |name| - software = Omnibus::Software.load(project, name) - if @dependency_blacklist.member? name - log.info( - "Not adding #{name} as a dependency as it is marked to be ignored.") - elsif software.fetcher.instance_of?(Omnibus::PathFetcher) - log.info( - "Not adding #{name} as a dependency: it's installed from " + - "a path which probably means that it is package configuration, not " + - "a 3rd-party component to be imported.") - elsif software.fetcher.instance_of?(Omnibus::NullFetcher) - if software.builder.built_gemspec - log.info( - "Adding #{name} as a RubyGem dependency because it builds " + - "#{software.builder.built_gemspec}") - rubygems_deps[name] = software.version - else - log.info( - "Not adding #{name} as a dependency: no sources listed.") - end - else - omnibus_deps[name] = software.version - end - end - - gem_requirements = software.builder.manually_installed_rubygems - rubygems_deps = resolve_rubygems_deps(gem_requirements) - - morphology.update({ - # Possibly this tool should look at software.build and - # generate suitable configure, build and install-commands. - # For now: don't bother! - - # FIXME: are these build or runtime dependencies? We'll assume both. - "x-build-dependencies-omnibus" => omnibus_deps, - "x-runtime-dependencies-omnibus" => omnibus_deps, - - "x-build-dependencies-rubygems" => {}, - "x-runtime-dependencies-rubygems" => rubygems_deps, - }) + # Possibly this tool should look at software.build and + # generate suitable configure, build and install-commands. + # For now: don't bother! if software.description morphology['description'] = software.description + '\n\n' + @@ -266,7 +127,6 @@ class OmnibusChunkMorphologyGenerator < Importer::Base software = Omnibus::Software.load(@project, software_name) morph = generate_chunk_morph_for_software(project, software, source_dir) - write_morph(STDOUT, morph) end end diff --git a/rubygems.find_deps b/rubygems.find_deps new file mode 100755 index 0000000..a9f4f08 --- /dev/null +++ b/rubygems.find_deps @@ -0,0 +1,114 @@ +#!/usr/bin/env ruby +# +# Find dependencies for a RubyGem. +# +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +require_relative 'importer_base' +require_relative 'importer_bundler_extensions' + +BANNER = "Usage: rubygems.find_deps SOURCE_DIR GEM_NAME [VERSION]" + +DESCRIPTION = <<-END +This tool looks for a .gemspec file for GEM_NAME in SOURCE_DIR, and outputs the +set of RubyGems dependencies required to build it. It will honour a +Gemfile.lock file if one is present. + +It is intended for use with the `baserock-import` tool. +END + +class RubyGemDependencyFinder < Importer::Base + include Importer::BundlerExtensions + + def initialize + local_data = YAML.load_file(local_data_path("rubygems.yaml")) + @build_dependency_whitelist = local_data['build-dependency-whitelist'] + end + + def parse_options(arguments) + opts = create_option_parser(BANNER, DESCRIPTION) + + parsed_arguments = opts.parse!(arguments) + + if parsed_arguments.length != 2 && parsed_arguments.length != 3 + STDERR.puts "Expected 2 or 3 arguments, got #{parsed_arguments}." + opts.parse(['-?']) + exit 255 + end + + source_dir, gem_name, expected_version = parsed_arguments + source_dir = File.absolute_path(source_dir) + if expected_version != nil + expected_version = Gem::Version.new(expected_version.dup) + end + [source_dir, gem_name, expected_version] + end + + def build_deps_for_gem(spec) + deps = spec.dependencies.select do |d| + d.type == :development && @build_dependency_whitelist.member?(d.name) + end + end + + def runtime_deps_for_gem(spec) + spec.dependencies.select {|d| d.type == :runtime} + end + + def run + source_dir_name, gem_name, expected_version = parse_options(ARGV) + + log.info("Finding dependencies for #{gem_name} based on " + + "source code in #{source_dir_name}") + + resolved_specs = Dir.chdir(source_dir_name) do + definition = create_bundler_definition_for_gemspec(gem_name) + definition.resolve_remotely! + end + + spec = get_spec_for_gem(resolved_specs, gem_name) + validate_spec(spec, source_dir_name, expected_version) + + # One might think that you could use the Bundler::Dependency.groups + # field to filter but it doesn't seem to be useful. Instead we go back to + # the Gem::Specification of the target Gem and use the dependencies fild + # there. We look up each dependency in the resolved_specset to find out + # what version Bundler has chosen of it. + + def format_deps(specset, dep_list) + info = dep_list.collect do |dep| + spec = specset[dep][0] + [spec.name, spec.version.to_s] + end + Hash[info] + end + + build_deps = format_deps( + resolved_specs, build_deps_for_gem(spec)) + runtime_deps = format_deps( + resolved_specs, runtime_deps_for_gem(spec)) + + deps = { + 'rubygems' => { + 'build-dependencies' => build_deps, + 'runtime-dependencies' => runtime_deps, + } + } + + write_dependencies(STDOUT, deps) + end +end + +RubyGemDependencyFinder.new.run diff --git a/rubygems.to_chunk b/rubygems.to_chunk index 796fe89..e1f7132 100755 --- a/rubygems.to_chunk +++ b/rubygems.to_chunk @@ -1,6 +1,6 @@ #!/usr/bin/env ruby # -# Create a chunk morphology to integrate a RubyGem in Baserock +# Create a chunk morphology to build a RubyGem in Baserock # # Copyright (C) 2014 Codethink Limited # @@ -20,35 +20,20 @@ require 'bundler' require_relative 'importer_base' - -class << Bundler - def default_gemfile - # This is a hack to make things not crash when there's no Gemfile - Pathname.new('.') - end -end - -def spec_is_from_current_source_tree(spec, source_dir) - spec.source.instance_of? Bundler::Source::Path and - File.identical?(spec.source.path, source_dir) -end +require_relative 'importer_bundler_extensions' BANNER = "Usage: rubygems.to_chunk SOURCE_DIR GEM_NAME [VERSION]" DESCRIPTION = <<-END -This tool reads the Gemfile and optionally the Gemfile.lock from a Ruby project -source tree in SOURCE_DIR. It outputs a chunk morphology for GEM_NAME on -stdout. If VERSION is supplied, it is used to check that the build instructions -will produce the expected version of the Gem. +This tool looks in SOURCE_DIR to generate a chunk morphology with build +instructions for GEM_NAME. If VERSION is supplied, it is used to check that the +build instructions will produce the expected version of the Gem. It is intended for use with the `baserock-import` tool. END class RubyGemChunkMorphologyGenerator < Importer::Base - def initialize - local_data = YAML.load_file(local_data_path("rubygems.yaml")) - @build_dependency_whitelist = local_data['build-dependency-whitelist'] - end + include Importer::BundlerExtensions def parse_options(arguments) opts = create_option_parser(BANNER, DESCRIPTION) @@ -69,39 +54,6 @@ class RubyGemChunkMorphologyGenerator < Importer::Base [source_dir, gem_name, expected_version] end - def load_local_gemspecs() - # Look for .gemspec files in the source repo. - # - # If there is no .gemspec, but you set 'name' and 'version' then - # inside Bundler::Source::Path.load_spec_files this call will create a - # fake gemspec matching that name and version. That's probably not useful. - - dir = '.' - - source = Bundler::Source::Path.new({ - 'path' => dir, - }) - - log.info "Loaded #{source.specs.count} specs from source dir." - source.specs.each do |spec| - log.debug " * #{spec.inspect} #{spec.dependencies.inspect}" - end - - source - end - - def get_spec_for_gem(specs, gem_name) - found = specs[gem_name].select {|s| Gem::Platform.match(s.platform)} - if found.empty? - raise Exception, - "No Gemspecs found matching '#{gem_name}'" - elsif found.length != 1 - raise Exception, - "Unsure which Gem to use for #{gem_name}, got #{found}" - end - found[0] - end - def chunk_name_for_gemspec(spec) # Chunk names are the Gem's "full name" (name + version number), so # that we don't break in the rare but possible case that two different @@ -195,79 +147,23 @@ class RubyGemChunkMorphologyGenerator < Importer::Base } end - def build_deps_for_gem(spec) - deps = spec.dependencies.select do |d| - d.type == :development && @build_dependency_whitelist.member?(d.name) - end - end - - def runtime_deps_for_gem(spec) - spec.dependencies.select {|d| d.type == :runtime} - end - def run source_dir_name, gem_name, expected_version = parse_options(ARGV) log.info("Creating chunk morph for #{gem_name} based on " + "source code in #{source_dir_name}") - Dir.chdir(source_dir_name) - - # Instead of reading the real Gemfile, invent one that simply includes the - # chosen .gemspec. If present, the Gemfile.lock will be honoured. - fake_gemfile = Bundler::Dsl.new - fake_gemfile.source('https://rubygems.org') - begin - fake_gemfile.gemspec({:name => gem_name}) - rescue Bundler::InvalidOption - error "Did not find #{gem_name}.gemspec in #{source_dir_name}" - exit 1 + resolved_specs = Dir.chdir(source_dir_name) do + # FIXME: we don't need to do this here, it'd be enough just to load + # the given gemspec + definition = create_bundler_definition_for_gemspec(gem_name) + definition.resolve_remotely! end - definition = fake_gemfile.to_definition('Gemfile.lock', true) - resolved_specs = definition.resolve_remotely! - spec = get_spec_for_gem(resolved_specs, gem_name) - - if not spec_is_from_current_source_tree(spec, source_dir_name) - error "Specified gem '#{spec.name}' doesn't live in the source in " + - "'#{source_dir_name}'" - log.debug "SPEC: #{spec.inspect} #{spec.source}" - exit 1 - end - - if expected_version != nil && spec.version != expected_version - # This check is brought to you by Coderay, which changes its version - # number based on an environment variable. Other Gems may do this too. - error "Source in #{source_dir_name} produces #{spec.full_name}, but " + - "the expected version was #{expected_version}." - exit 1 - end + validate_spec(spec, source_dir_name, expected_version) morph = generate_chunk_morph_for_gem(spec) - - # One might think that you could use the Bundler::Dependency.groups - # field to filter but it doesn't seem to be useful. Instead we go back to - # the Gem::Specification of the target Gem and use the dependencies fild - # there. We look up each dependency in the resolved_specset to find out - # what version Bundler has chosen of it. - - def format_deps_for_morphology(specset, dep_list) - info = dep_list.collect do |dep| - spec = specset[dep][0] - [spec.name, spec.version.to_s] - end - Hash[info] - end - - build_deps = format_deps_for_morphology( - resolved_specs, build_deps_for_gem(spec)) - runtime_deps = format_deps_for_morphology( - resolved_specs, runtime_deps_for_gem(spec)) - - morph['x-build-dependencies-rubygems'] = build_deps - morph['x-runtime-dependencies-rubygems'] = runtime_deps - write_morph(STDOUT, morph) end end |