Add cpan command to app

Change-Id: I13ad34e8b1bb255f2d644088b32ffcf2ba9e0a27

1 files changed, 161 insertions, 1 deletions

diff --git a/baserockimport/app.py b/baserockimport/app.py
index 0b190e5..5f3d435 100644
--- a/baserockimport/app.py
+++ b/baserockimport/app.py
@@ -1,4 +1,6 @@
-# Copyright (C) 2014  Codethink Limited
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014, 2015  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -21,6 +23,8 @@ import logging
 import os
 import pipes
 import sys
+import requests
+import json
 
 import baserockimport
 
@@ -86,6 +90,8 @@ class BaserockImportApplication(cliapp.Application):
                             arg_synopsis='GEM_NAME [GEM_VERSION]')
         self.add_subcommand('python', self.import_python,
                             arg_synopsis='PACKAGE_NAME [VERSION]')
+        self.add_subcommand('cpan', self.import_cpan,
+                            arg_synopsis='MODULE_NAME [VERSION]')
 
         self.stdout_has_colours = self._stream_has_colours(sys.stdout)
 
@@ -225,3 +231,157 @@ class BaserockImportApplication(cliapp.Application):
         loop.enable_importer('python', strata=['strata/core.morph'],
                              package_comp_callback=comp)
         loop.run()
+
+    def import_cpan(self, args):
+        '''Import one or more perl modules
+
+           We have to do a little work before we're ready to run the
+           first extension, the user will provide a module name as input
+           but we import entire distributions not individual modules.
+
+           So we first query metacpan to find the distribution that
+           provides the given module, we also write a miniumum amount
+           of metadata to a 'ROOT.meta' file which is passed onto
+           the root package via IMPORT_METAPATH, this metadata is used
+           to map distributions to the modules they provide.
+        '''
+
+        UNEXPECTED_RESPONSE_ERRMSG = ("Couldn't obtain distribution for "
+            "`%s' version `%s': server returned unexpected query response")
+
+        NO_DIST_WITH_MODULE_FOR_VERSION_ERRMSG = (
+            "Couldn't find a distribution containing module `%s' "
+            "with version `%s'")
+
+        NO_DIST_WITH_MODULE_ERRMSG = (
+            "Couldn't find distribution for module `%s'")
+
+        def get_module_metadata(module_name, module_version):
+            return (get_metadata_for_module_with_version(module_name,
+                                                         module_version)
+                    if module_version is not None
+                    else get_metadata_for_module(module_name))
+
+        def get_metadata_for_module(module_name):
+            ''' Gets metadata for the latest release of a module from
+                metacpan '''
+
+            try:
+                r = requests.get('http://api.metacpan.org/module/%s'
+                                 % module_name)
+                r.raise_for_status()
+            except Exception as e:
+                errmsg = ("%s: %s"
+                          % (NO_DIST_WITH_MODULE_ERRMSG % module_name, e))
+                raise cliapp.AppException(errmsg)
+
+            json = r.json()
+            if 'distribution' not in json:
+                raise cliapp.AppException(NO_DIST_WITH_MODULE_ERRMSG
+                                          % module_name)
+
+            return {'distribution': json['distribution'], 'version': None}
+
+
+        def get_metadata_for_module_with_version(module_name, module_version):
+            ''' Gets metadata for a specific version of a module from
+                metacpan '''
+
+            q = {'query': { 'filtered': {'query': {'match_all': {}},
+                    'filter': {'and': [
+                        {'term': {'file.module.name': module_name}},
+                        {'term': {'file.module.version': module_version}}
+                    ]}
+                 }},
+                 'fields': ['distribution', 'version']
+                }
+
+            distribution = None
+            version = None
+            try:
+                query_url = 'http://api.metacpan.org/v0/file/_search'
+
+                r = requests.post(query_url, json=q)
+                r.raise_for_status()
+            except Exception as e:
+                errmsg = ("Couldn't query metacpan with %s: %s"
+                          % (query_url, e))
+                raise cliapp.AppException(errmsg)
+
+            try:
+                hits = r.json()['hits']['total']
+                if hits == 0:
+                    raise cliapp.AppException(
+                        NO_DIST_WITH_MODULE_FOR_VERSION_ERRMSG
+                        % (module_name, module_version))
+
+                fields = r.json()['hits']['hits'][0]['fields']
+                distribution = fields['distribution']
+                version = fields['version']
+            except KeyError:
+                raise cliapp.AppException(UNEXPECTED_RESPONSE_ERRMSG
+                                          % (module_name, module_version))
+            return {'distribution': distribution, 'version': version}
+
+        def write_root_metadata(metadata, module, module_version):
+            ''' Constructs the initial metadata file to be passed
+                to the first import we run via IMPORT_METAPATH.
+
+                ROOT.meta will map the module we require to the distribution
+                that provides it.
+            '''
+
+            distribution = metadata['distribution']
+
+            depends_filename = 'strata/%s/ROOT.meta' % distribution
+            depends_path = os.path.join(self.settings['definitions-dir'],
+                                        depends_filename)
+
+            p, _ = os.path.split(depends_path)
+            if not os.path.exists(p):
+                try:
+                    os.makedirs(p)
+                except OSError as e:
+                    if e.errno != errno.EEXIST:
+                        raise e
+
+            metadata = {'cpan':
+                            {'dist-meta':
+                                {distribution:
+                                    {'modules':
+                                        {module:
+                                            {'minimum_version': module_version}
+                                        },
+                                        'pathname': None}}}}
+            with open(depends_path, 'w') as f:
+                json.dump(metadata, f)
+
+            return depends_path
+
+        if len(args) not in (1, 2):
+            raise cliapp.AppException('usage: %s cpan MODULE_NAME [VERSION]'
+                                      % sys.argv[0])
+
+        module_name = args[0]
+        module_version = args[1] if len(args) == 2 else None
+
+        metadata = get_module_metadata(module_name, module_version)
+        metadata_path = write_root_metadata(metadata, module_name,
+                                            module_version)
+        os.environ['IMPORT_METAPATH'] = metadata_path
+
+        dist_name = metadata['distribution']
+        dist_version = metadata['version'] or 'master'
+        self.status("Distribution `%s' provides module `%s', "
+                    "importing `%s-%s'...\n",
+                    dist_name, module_name,
+                    dist_name, dist_version, bold=True)
+
+        loop = baserockimport.mainloop.ImportLoop(app=self,
+                                                  goal_kind='cpan',
+                                                  goal_name=dist_name,
+                                                  goal_version=dist_version,
+                                                  generate_chunk_morphs=False,
+                                                  ignore_version_field=True)
+        loop.enable_importer('cpan', strata=['strata/core.morph'])
+        loop.run()