path: root/git_review/tests/__init__.py

# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import hashlib
import os
import shutil
import stat
import struct
import sys

import fixtures
import requests
import testtools
from testtools import content

from git_review.tests import utils

if sys.version < '3':
    import urllib
    import urlparse
    urlparse = urlparse.urlparse
else:
    import urllib.parse
    import urllib.request
    urlparse = urllib.parse.urlparse


WAR_URL = 'http://tarballs.openstack.org/' \
          'ci/gerrit/gerrit-v2.11.4.13.cb9800e.war'
# Update GOLDEN_SITE_VER for every change altering golden site, including
# WAR_URL changes. Set new value to something unique (just +1 it for example)
GOLDEN_SITE_VER = '4'


# NOTE(yorik-sar): This function needs to be a perfect hash function for
# existing test IDs. This is verified by running check_test_id_hashes script
# prior to running tests. Note that this doesn't imply any cryptographic
# requirements for underlying algorithm, so we can use weak hash here.
# Range of results for this function is limited by port numbers selection
# in _pick_gerrit_port_and_dir method (it can't be greater than 10000 now).
def _hash_test_id(test_id):
    if not isinstance(test_id, bytes):
        test_id = test_id.encode('utf-8')
    hash_ = hashlib.md5(test_id).digest()
    num = struct.unpack("=I", hash_[:4])[0]
    return num % 10000


class DirHelpers(object):
    def _dir(self, base, *args):
        """Creates directory name from base name and other parameters."""
        return os.path.join(getattr(self, base + '_dir'), *args)


class IsoEnvDir(DirHelpers, fixtures.Fixture):
    """Created isolated env and associated directories"""

    def __init__(self, base_dir=None):
        super(IsoEnvDir, self).setUp()

        # set up directories for isolation
        if base_dir:
            self.base_dir = base_dir
            self.temp_dir = self._dir('base', 'tmp')
            if not os.path.exists(self.temp_dir):
                os.mkdir(self.temp_dir)
            self.addCleanup(shutil.rmtree, self.temp_dir)
        else:
            self.temp_dir = self.useFixture(fixtures.TempDir()).path

        self.work_dir = self._dir('temp', 'work')
        self.home_dir = self._dir('temp', 'home')
        self.xdg_config_dir = self._dir('home', '.xdgconfig')
        for path in [self.home_dir, self.xdg_config_dir, self.work_dir]:
            if not os.path.exists(path):
                os.mkdir(path)

        # ensure user proxy conf doesn't interfere with tests
        self.useFixture(fixtures.EnvironmentVariable('no_proxy', '*'))
        self.useFixture(fixtures.EnvironmentVariable('NO_PROXY', '*'))

        # isolate tests from user and system git configuration
        self.useFixture(fixtures.EnvironmentVariable('HOME', self.home_dir))
        self.useFixture(
            fixtures.EnvironmentVariable('XDG_CONFIG_HOME',
                                         self.xdg_config_dir))
        self.useFixture(
            fixtures.EnvironmentVariable('GIT_CONFIG_NOSYSTEM', "1"))
        self.useFixture(
            fixtures.EnvironmentVariable('EMAIL', "you@example.com"))
        self.useFixture(
            fixtures.EnvironmentVariable('GIT_AUTHOR_NAME',
                                         "gitreview tester"))
        self.useFixture(
            fixtures.EnvironmentVariable('GIT_COMMITTER_NAME',
                                         "gitreview tester"))


class GerritHelpers(DirHelpers):

    def init_dirs(self):
        self.primary_dir = os.path.abspath(os.path.curdir)
        self.gerrit_dir = self._dir('primary', '.gerrit')
        self.gsite_dir = self._dir('gerrit', 'golden_site')
        self.gerrit_war = self._dir('gerrit', WAR_URL.split('/')[-1])

    def ensure_gerrit_war(self):
        # check if gerrit.war file exists in .gerrit directory
        if not os.path.exists(self.gerrit_dir):
            os.mkdir(self.gerrit_dir)

        if not os.path.exists(self.gerrit_war):
            print("Downloading Gerrit binary from %s..." % WAR_URL)
            resp = requests.get(WAR_URL)
            if resp.status_code != 200:
                raise RuntimeError("Problem requesting Gerrit war")
            utils.write_to_file(self.gerrit_war, resp.content)
            print("Saved to %s" % self.gerrit_war)

    def init_gerrit(self):
        """Run Gerrit from the war file and configure it."""
        golden_ver_file = self._dir('gsite', 'golden_ver')
        if os.path.exists(self.gsite_dir):
            if not os.path.exists(golden_ver_file):
                golden_ver = '0'
            else:
                with open(golden_ver_file) as f:
                    golden_ver = f.read().strip()
            if GOLDEN_SITE_VER != golden_ver:
                print("Existing golden site has version %s, removing..." %
                      golden_ver)
                shutil.rmtree(self.gsite_dir)
            else:
                print("Golden site of version %s already exists" %
                      GOLDEN_SITE_VER)
                return

        # We write out the ssh host key for gerrit's ssh server which
        # for undocumented reasons forces gerrit init to download the
        # bouncy castle libs which we need for ssh that works on
        # newer distros like ubuntu xenial.
        os.makedirs(self._dir('gsite', 'etc'))
        # create SSH host key
        host_key_file = self._dir('gsite', 'etc', 'ssh_host_rsa_key')
        utils.run_cmd('ssh-keygen', '-t', 'rsa', '-b', '4096', '-m', 'PEM',
                                    '-f', host_key_file, '-N', '')

        print("Creating a new golden site of version " + GOLDEN_SITE_VER)

        # initialize Gerrit
        utils.run_cmd('java', '-jar', self.gerrit_war,
                      'init', '-d', self.gsite_dir,
                      '--batch', '--no-auto-start', '--install-plugin',
                      'download-commands')
        utils.run_cmd('java', '-jar', self.gerrit_war, 'reindex',
                      '-d', self.gsite_dir)

        with open(golden_ver_file, 'w') as f:
            f.write(GOLDEN_SITE_VER)

        # create SSH public key
        key_file = self._dir('gsite', 'test_ssh_key')
        utils.run_cmd('ssh-keygen', '-t', 'rsa', '-b', '4096', '-m', 'PEM',
                                    '-f', key_file, '-N', '')
        with open(key_file + '.pub', 'rb') as pub_key_file:
            pub_key = pub_key_file.read()

        # create admin user in Gerrit database
        sql_query = """INSERT INTO ACCOUNTS (REGISTERED_ON) VALUES (NOW());
        INSERT INTO ACCOUNT_GROUP_MEMBERS (ACCOUNT_ID, GROUP_ID) \
            VALUES (0, 1);
        INSERT INTO ACCOUNT_EXTERNAL_IDS (ACCOUNT_ID, EXTERNAL_ID, PASSWORD) \
            VALUES (0, 'username:test_user', 'test_pass');
        INSERT INTO ACCOUNT_SSH_KEYS (SSH_PUBLIC_KEY, VALID) \
            VALUES ('%s', 'Y')""" % pub_key.decode()

        utils.run_cmd('java', '-jar',
                      self._dir('gsite', 'bin', 'gerrit.war'),
                      'gsql', '-d', self.gsite_dir, '-c', sql_query)

    def _run_gerrit_cli(self, command, *args):
        """SSH to gerrit Gerrit server and run command there."""
        return utils.run_cmd('ssh', '-p', str(self.gerrit_port),
                             'test_user@' + self.gerrit_host, 'gerrit',
                             command, *args)

    def _run_git_review(self, *args, **kwargs):
        """Run git-review utility from source."""
        git_review = utils.run_cmd('which', 'git-review')
        kwargs.setdefault('chdir', self.test_dir)
        return utils.run_cmd(git_review, *args, **kwargs)


class BaseGitReviewTestCase(testtools.TestCase, GerritHelpers):
    """Base class for the git-review tests."""

    _remote = 'gerrit'

    @property
    def project_uri(self):
        return self.project_ssh_uri

    def setUp(self):
        """Configure testing environment.

        Prepare directory for the testing and clone test Git repository.
        Require Gerrit war file in the .gerrit directory to run Gerrit local.
        """
        super(BaseGitReviewTestCase, self).setUp()
        self.useFixture(fixtures.Timeout(2 * 60, True))

        # ensures git-review command runs in local mode (for functional tests)
        self.useFixture(
            fixtures.EnvironmentVariable('GITREVIEW_LOCAL_MODE', ''))

        self.init_dirs()
        ssh_addr, ssh_port, http_addr, http_port, self.site_dir = \
            self._pick_gerrit_port_and_dir()
        self.gerrit_host, self.gerrit_port = ssh_addr, ssh_port

        self.test_dir = self._dir('site', 'tmp', 'test_project')
        self.ssh_dir = self._dir('site', 'tmp', 'ssh')
        self.project_ssh_uri = (
            'ssh://test_user@%s:%s/test/test_project.git' % (
                ssh_addr, ssh_port))
        self.project_http_uri = (
            'http://test_user:test_pass@%s:%s/test/test_project.git' % (
                http_addr, http_port))

        self._run_gerrit(ssh_addr, ssh_port, http_addr, http_port)
        self._configure_ssh(ssh_addr, ssh_port)

        # create Gerrit empty project
        self._run_gerrit_cli('create-project', 'test/test_project',
                             '--empty-commit')

        # setup isolated area to work under
        self.useFixture(IsoEnvDir(self.site_dir))

        # prepare repository for the testing
        self._run_git('clone', self.project_uri)
        utils.write_to_file(self._dir('test', 'test_file.txt'),
                            'test file created'.encode())
        self._create_gitreview_file()

        # push changes to the Gerrit
        self._run_git('add', '--all')
        self._run_git('commit', '-m', 'Test file and .gitreview added.')
        self._run_git('push', 'origin', 'master')
        # push a branch to gerrit
        self._run_git('checkout', '-b', 'testbranch')
        utils.write_to_file(self._dir('test', 'test_file.txt'),
                            'test file branched'.encode())
        self._create_gitreview_file(defaultbranch='testbranch')
        self._run_git('add', '--all')
        self._run_git('commit', '-m', 'Branched.')
        self._run_git('push', 'origin', 'testbranch')
        # cleanup
        shutil.rmtree(self.test_dir)

        # go to the just cloned test Git repository
        self._run_git('clone', self.project_uri)
        self.configure_gerrit_remote()
        self.addCleanup(shutil.rmtree, self.test_dir)

        # ensure user is configured for all tests
        self._configure_gitreview_username()

    def set_remote(self, uri):
        self._run_git('remote', 'set-url', self._remote, uri)

    def reset_remote(self):
        self._run_git('remote', 'rm', self._remote)

    def attach_on_exception(self, filename):
        @self.addOnException
        def attach_file(exc_info):
            if os.path.exists(filename):
                content.attach_file(self, filename)
            else:
                self.addDetail(os.path.basename(filename),
                               content.text_content('Not found'))

    def _run_git(self, command, *args):
        """Run git command using test git directory."""
        if command == 'clone':
            return utils.run_git(command, args[0], self._dir('test'))
        return utils.run_git('--git-dir=' + self._dir('test', '.git'),
                             '--work-tree=' + self._dir('test'),
                             command, *args)

    def _run_gerrit(self, ssh_addr, ssh_port, http_addr, http_port):
        # create a copy of site dir
        if os.path.exists(self.site_dir):
            shutil.rmtree(self.site_dir)
        shutil.copytree(self.gsite_dir, self.site_dir)
        self.addCleanup(shutil.rmtree, self.site_dir)
        # write config
        with open(self._dir('site', 'etc', 'gerrit.config'), 'w') as _conf:
            new_conf = utils.get_gerrit_conf(
                ssh_addr, ssh_port, http_addr, http_port)
            _conf.write(new_conf)

        # If test fails, attach Gerrit config and logs to the result
        self.attach_on_exception(self._dir('site', 'etc', 'gerrit.config'))
        for name in ['error_log', 'sshd_log', 'httpd_log']:
            self.attach_on_exception(self._dir('site', 'logs', name))

        # start Gerrit
        gerrit_sh = self._dir('site', 'bin', 'gerrit.sh')
        utils.run_cmd(gerrit_sh, 'start')
        self.addCleanup(utils.run_cmd, gerrit_sh, 'stop')

    def _simple_change(self, change_text, commit_message,
                       file_=None):
        """Helper method to create small changes and commit them."""
        if file_ is None:
            file_ = self._dir('test', 'test_file.txt')
        utils.write_to_file(file_, change_text.encode())
        self._run_git('add', file_)
        self._run_git('commit', '-m', commit_message)

    def _simple_amend(self, change_text, file_=None):
        """Helper method to amend existing commit with change."""
        if file_ is None:
            file_ = self._dir('test', 'test_file_new.txt')
        utils.write_to_file(file_, change_text.encode())
        self._run_git('add', file_)
        # cannot use --no-edit because it does not exist in older git
        message = self._run_git('log', '-1', '--format=%s\n\n%b')
        self._run_git('commit', '--amend', '-m', message)

    def _configure_ssh(self, ssh_addr, ssh_port):
        """Setup ssh and scp to run with special options."""

        os.mkdir(self.ssh_dir)

        ssh_key = utils.run_cmd('ssh-keyscan', '-p', str(ssh_port), ssh_addr)
        utils.write_to_file(self._dir('ssh', 'known_hosts'), ssh_key.encode())
        self.addCleanup(os.remove, self._dir('ssh', 'known_hosts'))

        # Attach known_hosts to test results if anything fails
        self.attach_on_exception(self._dir('ssh', 'known_hosts'))

        for cmd in ('ssh', 'scp'):
            cmd_file = self._dir('ssh', cmd)
            s = '#!/bin/sh\n' \
                '/usr/bin/%s -i %s -o UserKnownHostsFile=%s ' \
                '-o IdentitiesOnly=yes ' \
                '-o PasswordAuthentication=no $@' % \
                (cmd,
                 self._dir('gsite', 'test_ssh_key'),
                 self._dir('ssh', 'known_hosts'))
            utils.write_to_file(cmd_file, s.encode())
            os.chmod(cmd_file, os.stat(cmd_file).st_mode | stat.S_IEXEC)

        os.environ['PATH'] = self.ssh_dir + os.pathsep + os.environ['PATH']
        os.environ['GIT_SSH'] = self._dir('ssh', 'ssh')

    def configure_gerrit_remote(self):
        self._run_git('remote', 'add', self._remote, self.project_uri)

    def _configure_gitreview_username(self):
        self._run_git('config', 'gitreview.username', 'test_user')

    def _pick_gerrit_port_and_dir(self):
        hash_ = _hash_test_id(self.id())
        host = '127.0.0.1'
        return (
            host, 12000 + hash_,  # avoid 11211 that is memcached port on CI
            host, 22000 + hash_,  # avoid ephemeral ports at 32678+
            self._dir('gerrit', 'site-' + str(hash_)),
        )

    def _create_gitreview_file(self, **kwargs):
        cfg = ('[gerrit]\n'
               'scheme=%s\n'
               'host=%s\n'
               'port=%s\n'
               'project=test/test_project.git\n'
               '%s')
        parsed = urlparse(self.project_uri)
        host_port = parsed.netloc.rpartition('@')[-1]
        host, __, port = host_port.partition(':')
        extra = '\n'.join('%s=%s' % kv for kv in kwargs.items())
        cfg %= parsed.scheme, host, port, extra
        utils.write_to_file(self._dir('test', '.gitreview'), cfg.encode())


class HttpMixin(object):
    """HTTP remote_url mixin."""

    @property
    def project_uri(self):
        return self.project_http_uri

    def _configure_gitreview_username(self):
        # trick to set http password
        self._run_git('config', 'gitreview.username', 'test_user:test_pass')