path: root/import/rubygem.to_chunk

#!/usr/bin/env ruby
#
# Create a chunk morphology to integrate a RubyGem in Baserock
#
# Copyright (C) 2014  Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

require 'bundler'
require 'optparse'
require 'yaml'

BASEROCK_RUBY_VERSION = '2.0.0'

IGNORED_GROUPS = [:compat_testing, :test]

# Users of traditional distros seem to find it useful to override the versions
# of these Gems that come bundled with the MRI Ruby intepreter with newer
# versions from rubygems.org. In Baserock it should be just as easy to update
# MRI. We should avoid building components from two places.
BUNDLED_GEMS = [
    'rake',
]

# Ignoring the :test group isn't enough for these Gems, they are often in the
# :development group too and thus we need to explicitly ignore them.
TEST_GEMS = [
    'rspec',
    'rspec_junit_formatter',
    'rspec-core',
    'rspec-expectations',
    'rspec-mocks',
    'simplecov',
]

IGNORED_GEMS = BUNDLED_GEMS + TEST_GEMS

def spec_is_from_current_source_tree(spec)
    spec.source.instance_of? Bundler::Source::Path and
        spec.source.path.fnmatch?('.')
end

# Good testcases for this code:
#   qu:
#     http://opensoul.org/2012/05/30/releasing-multiple-gems-from-one-repository/
#     'qu-mongodb' shouldn't pull in any rails deps
#   rails:
#     'activesupport' doesn't depend on any other rails components, make
#     sure the script gets this right. This is a different codepath to 'qu'.

class Dsl < Bundler::Dsl
    # The Bundler::Dsl class parses the Gemfile. We override it so that we can
    # extend the class of the Bundler::Definition instance that is created, and
    # so we can filter the results down to a specific Gem from the repo rather
    # than the top-level one.

    def self.evaluate(gemfile, lockfile, unlock, target_gem_name)
      builder = new
      builder.eval_gemfile(gemfile)
      builder.to_definition(lockfile, unlock, target_gem_name)
    end

    def to_definition(lockfile, unlock, target_gem_name)
        @sources << rubygems_source unless @sources.include?(rubygems_source)

        # Find the local Bundler::Source object, remove everything from that
        # source except the Gem we actually care about. This is necessary
        # because Bundler is designed for people who want to develop or deploy
        # all Gems from a given repo, but in this case we only care about *one*
        # Gem from the repo, which may not be the top level one.

        # Note that this doesn't solve all our problems!!!! For Rails, for
        # example, the top-level Gemfile lists a bunch of stuff that isn't
        # needed for all the Gems. For example some databases, which are not at
        # all necessary for activesupport! And jquery-rails, which brings in
        # railties, which brings in actionpack, which is just not needed!
        #
        # To be honest, I have no idea what to do about this right now. Maybe
        # a blacklist for certain nested Gems?
        local_source = nil
        new_deps = []
        have_target = false
        @dependencies.each do |dep|
            if spec_is_from_current_source_tree(dep)
                local_source = local_source || dep.source
                if dep.name == target_gem_name
                    new_deps << dep
                    have_target = true
                end
            else
                new_deps << dep
            end
        end
        if not local_source
            raise Exception, "Did not find any local Gems defined"
        end
        if not have_target
            target_dep = Bundler::Dependency.new(
                target_gem_name, '>= 0',
                {"type" => :runtime, "source" => local_source}
            )
            new_deps << target_dep
            STDERR.puts "TARGET DEP: #{target_dep}  #{target_dep.source.inspect}"
        end
        @dependencies = new_deps
        STDERR.puts "\n\nNEW DEPS: #{@dependencies}"

        Definition.new(lockfile, @dependencies, @sources, unlock, @ruby_version)
    end
end

class Definition < Bundler::Definition
    # The Bundler::Definition class holds the dependency info we need.

    def self.build(gemfile, lockfile, unlock, target_gem_name)
        # Overridden so that our subclassed Dsl is used.
        unlock ||= {}
        gemfile = Pathname.new(gemfile).expand_path

        unless gemfile.file?
            raise GemfileNotFound, "#{gemfile} not found"
        end

        Dsl.evaluate(gemfile, lockfile, unlock, target_gem_name)
    end

    def requested_dependencies
        # Overridden to remove more stuff from the list: excluding certain
        # groups using Bundler.settings.without is a good first step, but some
        # test tools seem to be in the generic :development group and thus
        # need to be explicitly removed from the list.
        result = super.reject { |d| IGNORED_GEMS.member? d.name }
        removed = dependencies - result
        STDERR.puts "Removed dependencies: #{removed.collect {|d| d.name}}"

        result
    end

    def resolve_build_dependencies()
        # The term "build dependencies" is my own. RubyGems seem to mostly care
        # about "needed at runtime" (:runtime) vs. "useful during development"
        # (:development). We actually want "needed at runtime or during `rake
        # install`" but we have to work this out for ourselves.

        # Note you can set ENV['DEBUG_RESOLVER'] for more debug info.

        # Here we do the equivalent of resolve_remotely! and resolve_cached!
        # combined. In the hope that they work OK together. Ideally we'd
        # cache the specs after fetching them the first time so that on the
        # next run we only needed to fetch the ones we didn't already have. Not
        # sure the Bundler code makes this at all easy though. Probably
        # extending Source::Rubygems would be the way forwards.
        @remote = true
        @sources.each { |s| s.remote! }
        @sources.each { |s| s.cached! }
        specs
    end
end

def parse_options(arguments)
    # No options so far ..
    opts = OptionParser.new

    opts.banner = "Usage: rubygem.import SOURCE_DIR GEM_NAME"
    opts.separator ""
    opts.separator "This tool reads the Gemfile and optionally the " +
                   "Gemfile.lock from a Ruby project "
    opts.separator "source tree in SOURCE_DIR. It outputs a chunk " +
                   "morphology for GEM_NAME on stdout."
    opts.separator ""
    opts.separator "It is intended for use with the `baserock-import` tool."

    parsed_arguments = opts.parse!(arguments)

    if parsed_arguments.length != 2 then
        STDERR.puts opts.help
        exit 1
    end

    parsed_arguments
end

def load_definition(target_gem_name)
    # Load and parse the Gemfile and, if found, the Gemfile.lock file.
    definition = Definition.build(
        'Gemfile', 'Gemfile.lock', update=false, target_gem_name)
rescue Bundler::GemfileNotFound
    STDERR.puts "Did not find a Gemfile in #{dir_name}."
    exit 1
end

def get_spec_for_gem(specs, gem_name)
    found = specs[gem_name]
    if found.empty?
        raise Exception,
            "No Gemspecs found matching '#{gem_name}'"
    elsif found.length != 1
        raise Exception,
            "Unsure which Gem to use for #{dep}, got #{found}"
    end
    found[0]
end

def chunk_name_for_gemspec(spec)
    # Chunk names are the Gem's "full name" (name + version number), so that we
    # don't break in the rare but possible case that two different versions of
    # the same Gem are required for something to work. It'd be nicer to only
    # use the full_name if we detect such a conflict.
    spec.full_name
end

def generate_chunk_morph_for_gem(spec)
    description = 'Automatically generated by rubygem.import'

    bin_dir = "\"$DESTDIR/$PREFIX/bin\""
    gem_dir = "\"$DESTDIR/$PREFIX/lib/ruby/gems/#{BASEROCK_RUBY_VERSION}\""

    # There's more splitting to be done, but putting the docs in the
    # correct artifact is the single biggest win for enabling smaller
    # system images.
    split_rules = [
        {
            'artifact' => "#{spec.full_name}-doc",
            'include' => [
                "usr/lib/ruby/gems/#{BASEROCK_RUBY_VERSION}/doc/.*"
            ]
        }
    ]

    # FIXME: these should build from source instead!
    install_commands = [
        "mkdir -p #{gem_dir}",
        "gem install --install-dir #{gem_dir} --bindir #{bin_dir} " +
             "--ignore-dependencies --local #{spec.full_name}.gem"
    ]

    {
        'name' => chunk_name_for_gemspec(spec),
        'kind' => 'chunk',
        'description' => description,
        'build-system' => 'manual',
        ##'gem-url' => "http://rubygems.org/downloads/#{spec.full_name}.gem",
        'products' => split_rules,
        'install-commands' => install_commands
    }
end

def write_morph(file, morph)
    file.write(YAML.dump(morph))
end

def run
    source_dir_name, gem_name = parse_options(ARGV)

    Dir.chdir(source_dir_name)

    definition = load_definition(gem_name)

    specset = definition.resolve_build_dependencies()

    spec = get_spec_for_gem(specset, gem_name)

    if not spec_is_from_current_source_tree(spec)
        STDERR.puts "Specified gem '#{spec.name}' doesn't live in the " +
            "source in '#{source_dir_name}'"
        STDERR.puts "SPEC: #{spec.inspect} #{spec.source}"
        rails_spec = get_spec_for_gem(specset, 'rails')
        STDERR.puts "Rails: #{rails_spec.inspect}"
        exit 1
    end

    morph = generate_chunk_morph_for_gem(spec)

    deps = Hash[specset.collect { |d| [d.name, d.version.to_s] }]
    morph['x-dependencies-rubygem'] = deps

    write_morph(STDOUT, morph)
end

run