From 507db79a2b40fc1024b719dbc579857a3807db22 Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Wed, 19 Aug 2015 15:39:05 +0000 Subject: Add cpan command to app Change-Id: I13ad34e8b1bb255f2d644088b32ffcf2ba9e0a27 --- baserockimport/app.py | 162 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 161 insertions(+), 1 deletion(-) diff --git a/baserockimport/app.py b/baserockimport/app.py index 0b190e5..5f3d435 100644 --- a/baserockimport/app.py +++ b/baserockimport/app.py @@ -1,4 +1,6 @@ -# Copyright (C) 2014 Codethink Limited +# -*- coding: utf-8 -*- +# +# Copyright © 2014, 2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -21,6 +23,8 @@ import logging import os import pipes import sys +import requests +import json import baserockimport @@ -86,6 +90,8 @@ class BaserockImportApplication(cliapp.Application): arg_synopsis='GEM_NAME [GEM_VERSION]') self.add_subcommand('python', self.import_python, arg_synopsis='PACKAGE_NAME [VERSION]') + self.add_subcommand('cpan', self.import_cpan, + arg_synopsis='MODULE_NAME [VERSION]') self.stdout_has_colours = self._stream_has_colours(sys.stdout) @@ -225,3 +231,157 @@ class BaserockImportApplication(cliapp.Application): loop.enable_importer('python', strata=['strata/core.morph'], package_comp_callback=comp) loop.run() + + def import_cpan(self, args): + '''Import one or more perl modules + + We have to do a little work before we're ready to run the + first extension, the user will provide a module name as input + but we import entire distributions not individual modules. + + So we first query metacpan to find the distribution that + provides the given module, we also write a miniumum amount + of metadata to a 'ROOT.meta' file which is passed onto + the root package via IMPORT_METAPATH, this metadata is used + to map distributions to the modules they provide. + ''' + + UNEXPECTED_RESPONSE_ERRMSG = ("Couldn't obtain distribution for " + "`%s' version `%s': server returned unexpected query response") + + NO_DIST_WITH_MODULE_FOR_VERSION_ERRMSG = ( + "Couldn't find a distribution containing module `%s' " + "with version `%s'") + + NO_DIST_WITH_MODULE_ERRMSG = ( + "Couldn't find distribution for module `%s'") + + def get_module_metadata(module_name, module_version): + return (get_metadata_for_module_with_version(module_name, + module_version) + if module_version is not None + else get_metadata_for_module(module_name)) + + def get_metadata_for_module(module_name): + ''' Gets metadata for the latest release of a module from + metacpan ''' + + try: + r = requests.get('http://api.metacpan.org/module/%s' + % module_name) + r.raise_for_status() + except Exception as e: + errmsg = ("%s: %s" + % (NO_DIST_WITH_MODULE_ERRMSG % module_name, e)) + raise cliapp.AppException(errmsg) + + json = r.json() + if 'distribution' not in json: + raise cliapp.AppException(NO_DIST_WITH_MODULE_ERRMSG + % module_name) + + return {'distribution': json['distribution'], 'version': None} + + + def get_metadata_for_module_with_version(module_name, module_version): + ''' Gets metadata for a specific version of a module from + metacpan ''' + + q = {'query': { 'filtered': {'query': {'match_all': {}}, + 'filter': {'and': [ + {'term': {'file.module.name': module_name}}, + {'term': {'file.module.version': module_version}} + ]} + }}, + 'fields': ['distribution', 'version'] + } + + distribution = None + version = None + try: + query_url = 'http://api.metacpan.org/v0/file/_search' + + r = requests.post(query_url, json=q) + r.raise_for_status() + except Exception as e: + errmsg = ("Couldn't query metacpan with %s: %s" + % (query_url, e)) + raise cliapp.AppException(errmsg) + + try: + hits = r.json()['hits']['total'] + if hits == 0: + raise cliapp.AppException( + NO_DIST_WITH_MODULE_FOR_VERSION_ERRMSG + % (module_name, module_version)) + + fields = r.json()['hits']['hits'][0]['fields'] + distribution = fields['distribution'] + version = fields['version'] + except KeyError: + raise cliapp.AppException(UNEXPECTED_RESPONSE_ERRMSG + % (module_name, module_version)) + return {'distribution': distribution, 'version': version} + + def write_root_metadata(metadata, module, module_version): + ''' Constructs the initial metadata file to be passed + to the first import we run via IMPORT_METAPATH. + + ROOT.meta will map the module we require to the distribution + that provides it. + ''' + + distribution = metadata['distribution'] + + depends_filename = 'strata/%s/ROOT.meta' % distribution + depends_path = os.path.join(self.settings['definitions-dir'], + depends_filename) + + p, _ = os.path.split(depends_path) + if not os.path.exists(p): + try: + os.makedirs(p) + except OSError as e: + if e.errno != errno.EEXIST: + raise e + + metadata = {'cpan': + {'dist-meta': + {distribution: + {'modules': + {module: + {'minimum_version': module_version} + }, + 'pathname': None}}}} + with open(depends_path, 'w') as f: + json.dump(metadata, f) + + return depends_path + + if len(args) not in (1, 2): + raise cliapp.AppException('usage: %s cpan MODULE_NAME [VERSION]' + % sys.argv[0]) + + module_name = args[0] + module_version = args[1] if len(args) == 2 else None + + metadata = get_module_metadata(module_name, module_version) + metadata_path = write_root_metadata(metadata, module_name, + module_version) + os.environ['IMPORT_METAPATH'] = metadata_path + + dist_name = metadata['distribution'] + dist_version = metadata['version'] or 'master' + self.status("Distribution `%s' provides module `%s', " + "importing `%s-%s'...\n", + dist_name, module_name, + dist_name, dist_version, bold=True) + + loop = baserockimport.mainloop.ImportLoop(app=self, + goal_kind='cpan', + goal_name=dist_name, + goal_version=dist_version, + generate_chunk_morphs=False, + ignore_version_field=True) + loop.enable_importer('cpan', strata=['strata/core.morph']) + loop.run() -- cgit v1.2.1