From 0c65063d049d22cc78ed9ae5cb664c0ab524d825 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Thu, 4 Sep 2014 14:18:46 +0100 Subject: import: Move hardcoded URIs into external data file --- import/rubygem.to_lorry | 55 +++++++++++++++---------------------------------- import/rubygems.yaml | 35 +++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 38 deletions(-) create mode 100644 import/rubygems.yaml diff --git a/import/rubygem.to_lorry b/import/rubygem.to_lorry index 60e9d925..bb381da3 100755 --- a/import/rubygem.to_lorry +++ b/import/rubygem.to_lorry @@ -20,6 +20,7 @@ import requests import requests_cache +import yaml import logging import json @@ -28,39 +29,6 @@ import sys import urlparse -known_source_uris = { - 'ast': 'https://github.com/openSUSE/ast', - 'brass': 'https://github.com/rubyworks/brass', - 'coveralls': 'https://github.com/lemurheavy/coveralls-ruby', - 'diff-lcs': 'https://github.com/halostatue/diff-lcs', - 'erubis': 'https://github.com/kwatch/erubis', - 'fog-brightbox': 'https://github.com/brightbox/fog-brightbox', - 'highline': 'https://github.com/JEG2/highline', - 'hoe': 'https://github.com/seattlerb/hoe', - 'indexer': 'https://github.com/rubyworks/indexer', - 'json': 'https://github.com/flori/json', - 'method_source': 'https://github.com/banister/method_source', - 'mixlib-authentication': 'https://github.com/opscode/mixlib-authentication', - 'mixlib-cli': 'https://github.com/opscode/mixlib-cli', - 'mixlib-log': 'https://github.com/opscode/mixlib-log', - 'mixlib-shellout': 'http://github.com/opscode/mixlib-shellout', - 'ohai': 'http://github.com/opscode/ohai', - 'rack-cache': 'https://github.com/rtomayko/rack-cache', - 'actionmailer': 'https://github.com/rails/rails', - 'actionpack': 'https://github.com/rails/rails', - 'actionview': 'https://github.com/rails/rails', - 'activemodel': 'https://github.com/rails/rails', - 'activerecord': 'https://github.com/rails/rails', - 'activesupport': 'https://github.com/rails/rails', - 'rails': 'https://github.com/rails/rails', - 'railties': 'https://github.com/rails/rails', - 'pg': 'https://github.com/ged/ruby-pg', - 'sigar': 'https://github.com/hyperic/sigar', - 'sprockets': 'https://github.com/sstephenson/sprockets', - 'tins': 'https://github.com/flori/tins', -} - - class RubyGemsWebServiceClient(object): def __init__(self): # Save hammering the rubygems.org API: 'requests' API calls are @@ -87,12 +55,21 @@ class RubyGemsWebServiceClient(object): class RubyGemLorryGenerator(object): + def __init__(self): + with open('rubygems.yaml', 'r') as f: + local_data = yaml.load(f.read()) + + self.known_source_uris = local_data['known-source-uris'] + + logging.debug( + "Loaded %i known source URIs from local metadata.", len(self.known_source_uris)) + def find_upstream_repo_for_gem(self, gem_name, gem_info): source_code_uri = gem_info['source_code_uri'] - if gem_name in known_source_uris: - logging.debug('Found %s in known_source_uris', gem_name) - known_uri = known_source_uris[gem_name] + if gem_name in self.known_source_uris: + logging.debug('Found %s in known-source-uris', gem_name) + known_uri = self.known_source_uris[gem_name] if source_code_uri is not None and known_uri != source_code_uri: sys.stderr.write( '%s: Hardcoded source URI %s doesn\'t match spec URI %s\n' % @@ -119,8 +96,10 @@ class RubyGemLorryGenerator(object): # https://github.com/search?q=$gemname -> often the first result is # the correct one, but you can never know. - raise Exception('Did not manage to automatically find the upstream ' - 'source URL for Gem %s.' % gem_name) + raise Exception( + "Did not manage to find the upstream source URL for Gem '%s'. " + "Please manually create a .lorry file, or add the Gem to " + "known-source-uris in rubygems.yaml." % gem_name) def project_name_from_repo(self, repo_url): if repo_url.endswith('/tree/master'): diff --git a/import/rubygems.yaml b/import/rubygems.yaml new file mode 100644 index 00000000..8fdf57b3 --- /dev/null +++ b/import/rubygems.yaml @@ -0,0 +1,35 @@ +--- + +# The following Gems don't provide a source_code_uri in their Gem metadata. +# Ideally ... they would do. +known-source-uris: + ast: https://github.com/openSUSE/ast + brass: https://github.com/rubyworks/brass + coveralls: https://github.com/lemurheavy/coveralls-ruby + diff-lcs: https://github.com/halostatue/diff-lcs + erubis: https://github.com/kwatch/erubis + fog-brightbox: https://github.com/brightbox/fog-brightbox + highline: https://github.com/JEG2/highline + hoe: https://github.com/seattlerb/hoe + indexer: https://github.com/rubyworks/indexer + json: https://github.com/flori/json + method_source: https://github.com/banister/method_source + mixlib-authentication: https://github.com/opscode/mixlib-authentication + mixlib-cli: https://github.com/opscode/mixlib-cli + mixlib-log: https://github.com/opscode/mixlib-log + mixlib-shellout: http://github.com/opscode/mixlib-shellout + ohai: http://github.com/opscode/ohai + rack-cache: https://github.com/rtomayko/rack-cache + actionmailer: https://github.com/rails/rails + actionpack: https://github.com/rails/rails + actionview: https://github.com/rails/rails + activejob: https://github.com/rails/rails + activemodel: https://github.com/rails/rails + activerecord: https://github.com/rails/rails + activesupport: https://github.com/rails/rails + rails: https://github.com/rails/rails + railties: https://github.com/rails/rails + pg: https://github.com/ged/ruby-pg + sigar: https://github.com/hyperic/sigar + sprockets: https://github.com/sstephenson/sprockets + tins: https://github.com/flori/tins -- cgit v1.2.1