path: root/baserockimport/app.py

# -*- coding: utf-8 -*-
#
# Copyright © 2014, 2015  Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.


import ansicolor
import cliapp

import logging
import os
import pipes
import sys
import requests
import json

import morphlib
import baserockimport


class BaserockImportApplication(cliapp.Application):
    def add_settings(self):
        self.settings.string(['lorries-dir'],
                             "location for Lorry files",
                             metavar="PATH",
                             default=os.path.abspath('./lorries'))
        self.settings.string(['definitions-dir'],
                             "location for morphology files",
                             metavar="PATH",
                             default=os.path.abspath('./definitions'))
        self.settings.string(['checkouts-dir'],
                             "location for Git checkouts",
                             metavar="PATH",
                             default=os.path.abspath('./checkouts'))
        self.settings.string(['lorry-working-dir'],
                             "Lorry working directory",
                             metavar="PATH",
                             default=os.path.abspath('./lorry-working-dir'))

        self.settings.boolean(['force-stratum-generation', 'force-stratum'],
                              "always create a stratum, overwriting any "
                              "existing stratum morphology, and ignoring any "
                              "components where errors occurred during import",
                              default=False)
        self.settings.boolean(['update-existing'],
                              "update all the checked-out Git trees and "
                              "generated definitions",
                              default=False)
        self.settings.boolean(['use-local-sources'],
                              "use file:/// URLs in the stratum 'repo' "
                              "fields, instead of upstream: URLs",
                              default=False)
        self.settings.boolean(['use-master-if-no-tag'],
                              "if the correct tag for a version can't be "
                              "found, use 'master' instead of raising an "
                              "error",
                              default=False)

    def _stream_has_colours(self, stream):
        # http://blog.mathieu-leplatre.info/colored-output-in-console-with-python.html
        if not hasattr(stream, "isatty"):
            return False
        if not stream.isatty():
            return False # auto color only on TTYs
        try:
            import curses
            curses.setupterm()
            return curses.tigetnum("colors") > 2
        except:
            # guess false in case of error
            return False

    def setup(self):
        self.add_subcommand('npm', self.import_npm,
                            arg_synopsis='PACKAGE [VERSION]')
        self.add_subcommand('omnibus', self.import_omnibus,
                            arg_synopsis='REPO PROJECT_NAME SOFTWARE_NAME')
        self.add_subcommand('rubygems', self.import_rubygems,
                            arg_synopsis='GEM_NAME [GEM_VERSION]')
        self.add_subcommand('python', self.import_python,
                            arg_synopsis='PACKAGE_NAME [VERSION]')
        self.add_subcommand('cpan', self.import_cpan,
                            arg_synopsis='MODULE_NAME [VERSION]')

        self.stdout_has_colours = self._stream_has_colours(sys.stdout)

    def setup_logging_formatter_for_file(self):
        root_logger = logging.getLogger()
        root_logger.name = 'main'

        # You need recent cliapp for this to work, with commit "Split logging
        # setup into further overrideable methods".
        return logging.Formatter("%(name)s: %(levelname)s: %(message)s")

    def process_args(self, args):
        if len(args) == 0:
            # Cliapp default is to just say "ERROR: must give subcommand" if
            # no args are passed, I prefer this.
            args = ['help']

        super(BaserockImportApplication, self).process_args(args)

    def status(self, msg, *args, **kwargs):
        text = msg % args
        if kwargs.get('error') == True:
            logging.error(text)
            if self.stdout_has_colours:
                sys.stdout.write(ansicolor.red(text))
            else:
                sys.stdout.write(text)
        else:
            logging.info(text)
            sys.stdout.write(text)
        sys.stdout.write('\n')

    def import_omnibus(self, args):
        '''Import a software component from an Omnibus project.

        Omnibus is a tool for generating application bundles for various
        platforms. See <https://github.com/opscode/omnibus> for more
        information.

        '''
        if len(args) != 3:
            raise cliapp.AppException(
                'Please give the location of the Omnibus definitions repo, '
                'and the name of the project and the top-level software '
                'component.')

        def running_inside_bundler():
            return 'BUNDLE_GEMFILE' in os.environ

        def command_to_run_python_in_directory(directory, args):
            # Bundler requires that we run it from the Omnibus project
            # directory. That messes up any relative paths the user may have
            # passed on the commandline, so we do a bit of a hack to change
            # back to the original directory inside the `bundle exec` process.
            return "cd %s; exec python %s" % (
                pipes.quote(directory), ' '.join(map(pipes.quote, args)))

        def reexecute_self_with_bundler(path):
            script = sys.argv[0]

            logging.info('Reexecuting %s within Bundler, so that extensions '
                         'use the correct dependencies for Omnibus and the '
                         'Omnibus project definitions.', script)
            command = command_to_run_python_in_directory(os.getcwd(), sys.argv)

            logging.debug('Running: `bundle exec %s` in dir %s', command, path)
            os.chdir(path)
            os.execvp('bundle', [script, 'exec', command])

        # Omnibus definitions are spread across multiple repos, and there is
        # no stability guarantee for the definition format. The official advice
        # is to use Bundler to execute Omnibus, so let's do that.
        if not running_inside_bundler():
            reexecute_self_with_bundler(args[0])

        definitions_dir = args[0]
        project_name = args[1]

        loop = baserockimport.mainloop.ImportLoop(
            app=self,
            goal_kind='omnibus', goal_name=args[2], goal_version='master')
        loop.enable_importer('omnibus',
                             extra_args=[definitions_dir, project_name])
        loop.enable_importer('rubygems')
        loop.run()

    def import_npm(self, args):
        '''Import one or more Node.js npm packages.'''
        if len(args) not in [1, 2]:
            raise cliapp.AppException(
                'Please pass the name and version of an npm package on the '
                'commandline.')

        goal_name = args[0]
        goal_version = args[1] if len(args) == 2 else 'master'

        loop = baserockimport.mainloop.ImportLoop(
            app=self,
            goal_kind='npm', goal_name=goal_name, goal_version=goal_version)
        loop.enable_importer('npm', strata=['strata/nodejs.morph'])
        loop.run()

    def import_rubygems(self, args):
        '''Import one or more RubyGems.'''
        if len(args) not in [1, 2]:
            raise cliapp.AppException(
                'Please pass the name and version of a RubyGem on the '
                'commandline.')

        goal_name = args[0]
        goal_version = args[1] if len(args) == 2 else 'master'

        loop = baserockimport.mainloop.ImportLoop(
            app=self,
            goal_kind='rubygems', goal_name=args[0], goal_version=goal_version)
        loop.enable_importer('rubygems', strata=['strata/ruby.morph'])
        loop.run()

    def import_python(self, args):
        '''Import one or more python packages.'''

        def comp(x, y):
            return x.replace('-', '_').lower() == y.replace('-', '_').lower()

        if len(args) < 1 or len(args) > 2:
            raise cliapp.AppException(
                'Please pass the name of the python package on the commandline.')

        package_name = args[0]

        package_version = args[1] if len(args) == 2 else 'master'

        loop = baserockimport.mainloop.ImportLoop(app=self,
                                                  goal_kind='python',
                                                  goal_name=package_name,
                                                  goal_version=package_version,
                                                  ignore_version_field=True)
        loop.enable_importer('python', strata=['strata/core.morph'],
                             package_comp_callback=comp)
        loop.run()

    def import_cpan(self, args):
        '''Import one or more perl modules

           We have to do a little work before we're ready to run the
           first extension, the user will provide a module name as input
           but we import entire distributions not individual modules.

           So we first query metacpan to find the distribution that
           provides the given module, we also write a miniumum amount
           of metadata to a 'ROOT.meta' file which is passed onto
           the root package via IMPORT_METAPATH, this metadata is used
           to map distributions to the modules they provide.
        '''

        UNEXPECTED_RESPONSE_ERRMSG = ("Couldn't obtain distribution for "
            "`%s' version `%s': server returned unexpected query response")

        NO_DIST_WITH_MODULE_FOR_VERSION_ERRMSG = (
            "Couldn't find a distribution containing module `%s' "
            "with version `%s'")

        NO_DIST_WITH_MODULE_ERRMSG = (
            "Couldn't find distribution for module `%s'")

        def get_module_metadata(module_name, module_version):
            return (get_metadata_for_module_with_version(module_name,
                                                         module_version)
                    if module_version is not None
                    else get_metadata_for_module(module_name))

        def get_metadata_for_module(module_name):
            ''' Gets metadata for the latest release of a module from
                metacpan '''

            try:
                r = requests.get('http://api.metacpan.org/module/%s'
                                 % module_name)
                r.raise_for_status()
            except Exception as e:
                errmsg = ("%s: %s"
                          % (NO_DIST_WITH_MODULE_ERRMSG % module_name, e))
                raise cliapp.AppException(errmsg)

            json = r.json()
            if 'distribution' not in json:
                raise cliapp.AppException(NO_DIST_WITH_MODULE_ERRMSG
                                          % module_name)

            return {'distribution': json['distribution'], 'version': None}


        def get_metadata_for_module_with_version(module_name, module_version):
            ''' Gets metadata for a specific version of a module from
                metacpan '''

            q = {'query': { 'filtered': {'query': {'match_all': {}},
                    'filter': {'and': [
                        {'term': {'file.module.name': module_name}},
                        {'term': {'file.module.version': module_version}}
                    ]}
                 }},
                 'fields': ['distribution', 'version']
                }

            distribution = None
            version = None
            try:
                query_url = 'http://api.metacpan.org/v0/file/_search'

                r = requests.post(query_url, json=q)
                r.raise_for_status()
            except Exception as e:
                errmsg = ("Couldn't query metacpan with %s: %s"
                          % (query_url, e))
                raise cliapp.AppException(errmsg)

            try:
                hits = r.json()['hits']['total']
                if hits == 0:
                    raise cliapp.AppException(
                        NO_DIST_WITH_MODULE_FOR_VERSION_ERRMSG
                        % (module_name, module_version))

                fields = r.json()['hits']['hits'][0]['fields']
                distribution = fields['distribution']
                version = fields['version']
            except KeyError:
                raise cliapp.AppException(UNEXPECTED_RESPONSE_ERRMSG
                                          % (module_name, module_version))
            return {'distribution': distribution, 'version': version}

        def write_root_metadata(metadata, module, module_version):
            ''' Constructs the initial metadata file to be passed
                to the first import we run via IMPORT_METAPATH.

                ROOT.meta will map the module we require to the distribution
                that provides it.
            '''

            distribution = metadata['distribution']
            depends_filename = 'strata/%s/ROOT.meta' % distribution
            defsdir = self.settings['definitions-dir']
            depends_path = os.path.join(defsdir, depends_filename)

            if not os.path.exists(defsdir):
                os.makedirs(defsdir)
                morphlib.gitdir.init(defsdir)

            p, _ = os.path.split(depends_path)
            if not os.path.exists(p):
                try:
                    os.makedirs(p)
                except OSError as e:
                    if e.errno != errno.EEXIST:
                        raise e

            metadata = {'cpan':
                            {'dist-meta':
                                {distribution:
                                    {'modules':
                                        {module:
                                            {'minimum_version': module_version}
                                        },
                                        'pathname': None}}}}
            with open(depends_path, 'w') as f:
                json.dump(metadata, f)

            return depends_path

        if len(args) not in (1, 2):
            raise cliapp.AppException('usage: %s cpan MODULE_NAME [VERSION]'
                                      % sys.argv[0])

        module_name = args[0]
        module_version = args[1] if len(args) == 2 else None

        metadata = get_module_metadata(module_name, module_version)
        metadata_path = write_root_metadata(metadata, module_name,
                                            module_version)
        os.environ['IMPORT_METAPATH'] = metadata_path

        dist_name = metadata['distribution']
        dist_version = metadata['version'] or 'master'
        self.status("Distribution `%s' provides module `%s', "
                    "importing `%s-%s'...\n",
                    dist_name, module_name,
                    dist_name, dist_version, bold=True)

        loop = baserockimport.mainloop.ImportLoop(app=self,
                                                  goal_kind='cpan',
                                                  goal_name=dist_name,
                                                  goal_version=dist_version,
                                                  generate_chunk_morphs=False,
                                                  ignore_version_field=True)
        loop.enable_importer('cpan', strata=['strata/core.morph'])
        loop.run()