diff options
Diffstat (limited to 'baserockimport')
-rw-r--r-- | baserockimport/app.py | 17 | ||||
-rw-r--r-- | baserockimport/data/rubygems.yaml | 19 | ||||
-rw-r--r-- | baserockimport/exts/importer_bundler_extensions.rb | 39 | ||||
-rwxr-xr-x | baserockimport/exts/rubygems.find_deps | 7 | ||||
-rwxr-xr-x | baserockimport/exts/rubygems.to_chunk | 8 | ||||
-rw-r--r-- | baserockimport/mainloop.py | 79 |
6 files changed, 120 insertions, 49 deletions
diff --git a/baserockimport/app.py b/baserockimport/app.py index 6ae3424..6f4d7c3 100644 --- a/baserockimport/app.py +++ b/baserockimport/app.py @@ -44,6 +44,11 @@ class BaserockImportApplication(cliapp.Application): metavar="PATH", default=os.path.abspath('./lorry-working-dir')) + self.settings.boolean(['force-stratum-generation', 'force-stratum'], + "always create a stratum, overwriting any " + "existing stratum morphology, and ignoring any " + "components where errors occurred during import", + default=False) self.settings.boolean(['update-existing'], "update all the checked-out Git trees and " "generated definitions", @@ -76,7 +81,7 @@ class BaserockImportApplication(cliapp.Application): self.add_subcommand('omnibus', self.import_omnibus, arg_synopsis='REPO PROJECT_NAME SOFTWARE_NAME') self.add_subcommand('rubygems', self.import_rubygems, - arg_synopsis='GEM_NAME') + arg_synopsis='GEM_NAME [GEM_VERSION]') self.stdout_has_colours = self._stream_has_colours(sys.stdout) @@ -165,12 +170,16 @@ class BaserockImportApplication(cliapp.Application): def import_rubygems(self, args): '''Import one or more RubyGems.''' - if len(args) != 1: + if len(args) not in [1, 2]: raise cliapp.AppException( - 'Please pass the name of a RubyGem on the commandline.') + 'Please pass the name and version of a RubyGem on the ' + 'commandline.') + + goal_name = args[0] + goal_version = args[1] if len(args) == 2 else 'master' loop = baserockimport.mainloop.ImportLoop( app=self, - goal_kind='rubygems', goal_name=args[0], goal_version='master') + goal_kind='rubygems', goal_name=goal_name, goal_version=goal_version) loop.enable_importer('rubygems') loop.run() diff --git a/baserockimport/data/rubygems.yaml b/baserockimport/data/rubygems.yaml index e1e6fcc..8c13689 100644 --- a/baserockimport/data/rubygems.yaml +++ b/baserockimport/data/rubygems.yaml @@ -8,9 +8,24 @@ lorry-prefix: ruby-gems/ # tools because of the number of circular dependencies. Instead, only those # tools which are known to be required at Gem build time are listed as # build-dependencies, and any other :development dependencies are ignored. -build-dependency-whitelist: +# +# This list is currently empty, because everything that was in it has been +# added to the 'ruby' stratum and so is present implicitly. +build-dependency-whitelist: [] + +# List of Gems which are built into Ruby or included in the 'ruby' stratum +# in http://git.baserock.org/cgi-bin/cgit.cgi/baserock/baserock/definitions.git +# +# This doesn't take the versions that are provided into account (and it'd be +# quite a maintenance burden if it did). Thus it's not an ideal solution, as +# something may for example depend on Rake 4.9 when Rake 2.0 is actually the +# only thing available and this won't be noticed until the user tries to build +# the generated definitions. +ignore-list: + - bundler - hoe - # rake is bundled with Ruby, so it is not included in the whitelist. + - rake + - rake-compiler # The following Gems don't provide a source_code_uri in their Gem metadata. # Ideally ... they would do. diff --git a/baserockimport/exts/importer_bundler_extensions.rb b/baserockimport/exts/importer_bundler_extensions.rb index 034b3c2..88c82e2 100644 --- a/baserockimport/exts/importer_bundler_extensions.rb +++ b/baserockimport/exts/importer_bundler_extensions.rb @@ -28,7 +28,25 @@ end module Importer module BundlerExtensions - def create_bundler_definition_for_gemspec(gem_name) + def locate_gemspec(gem_name, path) + target = "#{gem_name}.gemspec" + matches = Dir["#{path}/#{Bundler::Source::Path::DEFAULT_GLOB}"].select do |filename| + File.basename(filename) == target + end + if matches.empty? + error "Did not find any files matching #{target} within #{path}." + exit 1 + elsif matches.length > 1 + error "Multiple files matching #{target} found within #{path}. It's " \ + "not clear which one to use!" + exit 1 + end + matches[0] + end + + def create_bundler_definition_for_gemspec(gem_name, path) + gemspec_file = locate_gemspec(gem_name, path) + # Using the real Gemfile doesn't get great results, because people can put # lots of stuff in there that is handy for developers to have but # irrelevant if you just want to produce a .gem. Also, there is only one @@ -40,13 +58,8 @@ module Importer # chosen .gemspec. If present, the Gemfile.lock will be honoured. fake_gemfile = Bundler::Dsl.new fake_gemfile.source('https://rubygems.org') - begin - fake_gemfile.gemspec({:name => gem_name}) - rescue Bundler::InvalidOption - error "Did not find #{gem_name}.gemspec in current directory." - exit 1 - end - + fake_gemfile.gemspec({:name => gem_name, + :path => File.dirname(gemspec_file)}) fake_gemfile.to_definition('Gemfile.lock', true) end @@ -60,11 +73,13 @@ module Importer found[0] end + def directory_is_within(path, expected_subpath) + File.realpath(expected_subpath).start_with?(File.realpath(path)) + end + def spec_is_from_current_source_tree(spec, source_dir) - Dir.chdir(source_dir) do - spec.source.instance_of? Bundler::Source::Path and - File.identical?(spec.source.path, '.') - end + spec.source.instance_of? Bundler::Source::Path and + directory_is_within(source_dir, spec.source.path) end def validate_spec(spec, source_dir_name, expected_version) diff --git a/baserockimport/exts/rubygems.find_deps b/baserockimport/exts/rubygems.find_deps index 228c88b..ae08b65 100755 --- a/baserockimport/exts/rubygems.find_deps +++ b/baserockimport/exts/rubygems.find_deps @@ -36,6 +36,7 @@ class RubyGemDependencyFinder < Importer::Base def initialize local_data = YAML.load_file(local_data_path("rubygems.yaml")) @build_dependency_whitelist = local_data['build-dependency-whitelist'] + @ignore_list = local_data['ignore-list'] end def parse_options(arguments) @@ -64,7 +65,9 @@ class RubyGemDependencyFinder < Importer::Base end def runtime_deps_for_gem(spec) - spec.dependencies.select {|d| d.type == :runtime} + spec.dependencies.select do |d| + d.type == :runtime && ! @ignore_list.member?(d.name) + end end def run @@ -74,7 +77,7 @@ class RubyGemDependencyFinder < Importer::Base "#{source_dir_name}") resolved_specs = Dir.chdir(source_dir_name) do - definition = create_bundler_definition_for_gemspec(gem_name) + definition = create_bundler_definition_for_gemspec(gem_name, source_dir_name) definition.resolve_remotely! end diff --git a/baserockimport/exts/rubygems.to_chunk b/baserockimport/exts/rubygems.to_chunk index c1a3e7c..1573d8b 100755 --- a/baserockimport/exts/rubygems.to_chunk +++ b/baserockimport/exts/rubygems.to_chunk @@ -154,9 +154,11 @@ class RubyGemChunkMorphologyGenerator < Importer::Base "source code in #{source_dir_name}") resolved_specs = Dir.chdir(source_dir_name) do - # FIXME: we don't need to do this here, it'd be enough just to load - # the given gemspec - definition = create_bundler_definition_for_gemspec(gem_name) + # FIXME: resolving the specs for all the dependencies of the target gem + # isn't necessary here. In fact, just reading/executing the .gemspec + # would be enough, and would speed this program up and remove a lot of + # pointless network access to rubygems.org. + definition = create_bundler_definition_for_gemspec(gem_name, source_dir_name) definition.resolve_remotely! end diff --git a/baserockimport/mainloop.py b/baserockimport/mainloop.py index ccb695d..4ca677d 100644 --- a/baserockimport/mainloop.py +++ b/baserockimport/mainloop.py @@ -182,14 +182,7 @@ class ImportLoop(object): current_item, current_item.dependencies, to_process, processed) - if len(errors) > 0: - self.app.status( - '\nErrors encountered, not generating a stratum morphology.') - self.app.status( - 'See the README files for guidance.') - else: - self._generate_stratum_morph_if_none_exists( - processed, self.goal_name) + self._maybe_generate_stratum(processed, errors, self.goal_name) duration = time.time() - start_time end_displaytime = time.strftime('%x %X %Z', time.localtime()) @@ -210,10 +203,19 @@ class ImportLoop(object): lorry = self._find_or_create_lorry_file(kind, name) source_repo, url = self._fetch_or_update_source(lorry) - checked_out_version, ref = self._checkout_source_version( - source_repo, name, version) + checked_out_version, ref = self._checkout_source_version_for_package( + source_repo, package) package.set_version_in_use(checked_out_version) + repo_path = os.path.relpath(source_repo.dirname) + if morphlib.git.is_valid_sha1(ref): + self.app.status( + "%s %s: using %s commit %s", name, version, repo_path, ref) + else: + self.app.status( + "%s %s: using %s ref %s (commit %s)", name, version, repo_path, + ref, source_repo.resolve_ref_to_commit(ref)) + # 2. Create a chunk morphology with build instructions. chunk_morph = self._find_or_create_chunk_morph( @@ -317,7 +319,8 @@ class ImportLoop(object): if kind not in self.importers: raise Exception('Importer for %s was not enabled.' % kind) extra_args = self.importers[kind]['extra_args'] - self.app.status('Calling %s to generate lorry for %s', tool, name) + self.app.status( + '%s: calling %s to generate lorry', name, tool) lorry_text = run_extension(tool, extra_args + [name]) try: lorry = json.loads(lorry_text) @@ -373,9 +376,11 @@ class ImportLoop(object): return repo, url - def _checkout_source_version(self, source_repo, name, version): + def _checkout_source_version_for_package(self, source_repo, package): # FIXME: we need to be a bit smarter than this. Right now we assume # that 'version' is a valid Git ref. + name = package.name + version = package.version possible_names = [ version, @@ -397,7 +402,7 @@ class ImportLoop(object): ref = version = 'master' else: raise BaserockImportException( - 'Could not find ref for %s version %s.' % (name, version)) + 'Could not find ref for %s.' % package) return version, ref @@ -449,8 +454,7 @@ class ImportLoop(object): extra_args = self.importers[kind]['extra_args'] self.app.status( - 'Calling %s to generate chunk morph for %s %s', tool, name, - version) + '%s %s: calling %s to generate chunk morph', name, version, tool) args = extra_args + [source_repo.dirname, name] if version != 'master': @@ -493,8 +497,7 @@ class ImportLoop(object): extra_args = self.importers[kind]['extra_args'] self.app.status( - 'Calling %s to calculate dependencies for %s %s', tool, name, - version) + '%s %s: calling %s to calculate dependencies', name, version, tool) args = extra_args + [source_repo.dirname, name] if version != 'master': @@ -520,34 +523,58 @@ class ImportLoop(object): 'One or more cycles detected in build graph: %s' % (', '.join(all_loops_str))) - def _generate_stratum_morph_if_none_exists(self, graph, goal_name): + def _maybe_generate_stratum(self, graph, errors, goal_name): filename = os.path.join( self.app.settings['definitions-dir'], 'strata', '%s.morph' % goal_name) + update_existing = self.app.settings['update-existing'] - if os.path.exists(filename): - if not self.app.settings['update-existing']: - self.app.status( - msg='Found stratum morph for %s at %s, not overwriting' % - (goal_name, filename)) - return + if self.app.settings['force-stratum-generation']: + self._generate_stratum( + graph, goal_name, filename, ignore_errors=True) + elif len(errors) > 0: + self.app.status( + '\nErrors encountered, not generating a stratum morphology.') + self.app.status( + 'See the README files for guidance.') + elif os.path.exists(filename) and not update_existing: + self.app.status( + msg='Found stratum morph for %s at %s, not overwriting' % + (goal_name, filename)) + else: + self._generate_stratum(graph, goal_name, filename) + def _generate_stratum(self, graph, goal_name, filename, + ignore_errors=False): self.app.status(msg='Generating stratum morph for %s' % goal_name) chunk_entries = [] for package in self._sort_chunks_by_build_order(graph): m = package.morphology + if m is None: - raise cliapp.AppException('No morphology for %s' % package) + if ignore_errors: + logging.warn( + 'Ignoring %s because there is no chunk morphology.') + continue + else: + raise cliapp.AppException('No morphology for %s' % package) def format_build_dep(name, version): dep_package = find(graph, lambda p: p.match(name, version)) return '%s-%s' % (name, dep_package.version_in_use) + def get_build_deps(morphology): + deps = dict() + for kind in self.importers: + field = 'x-build-dependencies-%s' % kind + deps.update(morphology.get(field, [])) + return deps + build_depends = [ format_build_dep(name, version) for name, version in - m['x-build-dependencies-rubygems'].iteritems() + get_build_deps(m).iteritems() ] entry = { |