path: root/zuul/cmd/migrate.py

# Copyright 2017 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

# TODO(mordred):
# * Read and apply filters from the jobs: section
# * Figure out shared job queues
# * Emit job definitions
#   * figure out from builders whether or not it's a normal job or a
#     a devstack-legacy job
#   * Handle emitting arbitrary tox jobs (see tox-py27dj18)

import argparse
import collections
import copy
import itertools
import getopt
import logging
import os
import subprocess
import tempfile
import re
from typing import Any, Dict, List, Optional

import jenkins_jobs.builder
from jenkins_jobs.formatter import deep_format
import jenkins_jobs.formatter
from jenkins_jobs.parser import matches
import jenkins_jobs.parser
import yaml

JOB_MATCHERS = {}  # type: Dict[str, Dict[str, Dict]]
TEMPLATES_TO_EXPAND = {}  # type: Dict[str, List]
JOBS_FOR_EXPAND = collections.defaultdict(dict)  # type: ignore
JOBS_BY_ORIG_TEMPLATE = {}  # type: ignore
SUFFIXES = []  # type: ignore
SKIP_MACROS = []  # type: ignore
ENVIRONMENT = '{{ zuul | zuul_legacy_vars }}'
DESCRIPTION = """Migrate zuul v2 and Jenkins Job Builder to Zuul v3.

This program takes a zuul v2 layout.yaml and a collection of Jenkins Job
Builder job definitions and transforms them into a Zuul v3 config. An
optional mapping config can be given that defines how to map old jobs
to new jobs.
"""


def deal_with_shebang(data):
    # Ansible shell blocks do not honor shebang lines. That's fine - but
    # we do have a bunch of scripts that have either nothing, -x, -xe,
    # -ex or -eux. Transform those into leading set commands
    if not data.startswith('#!'):
        return (None, data)
    data_lines = data.split('\n')
    data_lines.reverse()
    shebang = data_lines.pop()
    split_line = shebang.split()
    # Strip the # and the !
    executable = split_line[0][2:]
    if executable == '/bin/sh':
        # Ansible default
        executable = None
    if len(split_line) > 1:
        flag_x = False
        flag_e = False
        flag_u = False
        optlist, args = getopt.getopt(split_line[1:], 'uex')
        for opt, _ in optlist:
            if opt == '-x':
                flag_x = True
            elif opt == '-e':
                flag_e = True
            elif opt == '-u':
                flag_u = True

        if flag_x:
            data_lines.append('set -x')
        if flag_e:
            data_lines.append('set -e')
        if flag_u:
            data_lines.append('set -u')
    data_lines.reverse()
    data = '\n'.join(data_lines).lstrip()
    return (executable, data)


def _extract_from_vars(line):
    # export PROJECTS="openstack/blazar $PROJECTS"
    # export DEVSTACK_PROJECT_FROM_GIT=python-swiftclient
    # export DEVSTACK_PROJECT_FROM_GIT="python-octaviaclient"
    # export DEVSTACK_PROJECT_FROM_GIT+=",glean"
    projects = []
    line = line.replace('"', '').replace('+', '').replace(',', ' ')
    if (line.startswith('export PROJECTS') or
            line.startswith('export DEVSTACK_PROJECT_FROM_GIT')):
        nothing, project_string = line.split('=')
        project_string = project_string.replace('$PROJECTS', '').strip()
        projects = project_string.split()
    return projects


def extract_projects(data):
    # clonemap:
    #   - name: openstack/windmill
    #     dest: .
    # EOF
    projects = []
    data_lines = data.split('\n')
    in_clonemap = False
    in_clonemap_cli = False
    for line in data_lines:
        line = line.strip()
        if line == 'clonemap:':
            in_clonemap = True
            continue
        elif line == 'EOF':
            in_clonemap = False
            continue
        elif line.startswith('/usr/zuul-env/bin/zuul-cloner'):
            in_clonemap_cli = True
            continue
        elif in_clonemap_cli and not line.endswith('\\'):
            in_clonemap_cli = False
            continue
        if in_clonemap:
            if line.startswith('- name:'):
                garbage, project = line.split(':')
                project = project.strip().replace("'", '').replace('"', '')
                if project == '$ZUUL_PROJECT':
                    continue
                projects.append(project)
        elif in_clonemap_cli and line.startswith('openstack/'):
            line = line.replace('\\', '').strip()
            projects.append(line)
        elif in_clonemap_cli:
            continue
        else:
            projects.extend(_extract_from_vars(line))
    return projects


def expand_project_names(required, full):
    projects = []
    for name in full:
        repo = name
        if '/' in name:
            org, repo = name.split('/')
        if repo in required or name in required:
            projects.append(name)
    return projects


def list_to_project_dicts(projects):
    project_dicts = dict()
    for project in projects:
        project_dicts[project['project']['name']] = project['project']
    return project_dicts


def project_dicts_to_list(project_dicts):
    project_config = []
    for key in sorted(project_dicts.keys()):
        project_config.append(dict(project=project_dicts[key]))
    return project_config


def merge_project_dict(project_dicts, name, project):
    if name not in project_dicts:
        project_dicts[name] = project
        return

    old = project_dicts[name]
    for key in project:
        if key not in old:
            old[key] = project[key]
        elif isinstance(old[key], list):
            old[key].extend(project[key])
        elif isinstance(old[key], dict) and 'jobs' in old[key]:
            old[key]['jobs'].extend(project[key]['jobs'])
    return


def normalize_project_expansions():
    remove_from_job_matchers = []
    # First find the matchers that are the same for all jobs
    for job_name, project in copy.deepcopy(JOBS_FOR_EXPAND).items():
        JOB_MATCHERS[job_name] = None
        for project_name, expansion in project.items():
            if not JOB_MATCHERS[job_name]:
                JOB_MATCHERS[job_name] = copy.deepcopy(expansion['info'])
            else:
                if JOB_MATCHERS[job_name] != expansion['info']:
                    # We have different expansions for this job, it can't be
                    # done at the job level
                    remove_from_job_matchers.append(job_name)

    for job_name in remove_from_job_matchers:
        JOB_MATCHERS.pop(job_name, None)

    # Second, find out which projects need to expand a given template
    for job_name, project in copy.deepcopy(JOBS_FOR_EXPAND).items():
        # There is a job-level expansion for this one
        if job_name in JOB_MATCHERS:
            continue
        for project_name, expansion in project.items():
            TEMPLATES_TO_EXPAND[project_name] = []
            if expansion['info']:
                # There is an expansion for this project
                TEMPLATES_TO_EXPAND[project_name].append(expansion['template'])


# from :
# http://stackoverflow.com/questions/8640959/how-can-i-control-what-scalar-form-pyyaml-uses-for-my-data  # noqa
def should_use_block(value):
    for c in u"\u000a\u000d\u001c\u001d\u001e\u0085\u2028\u2029":
        if c in value:
            return True
    return False


def my_represent_scalar(self, tag, value, style=None):
    if style is None:
        if should_use_block(value):
            style = '|'
        else:
            style = self.default_style

    node = yaml.representer.ScalarNode(tag, value, style=style)
    if self.alias_key is not None:
        self.represented_objects[self.alias_key] = node
    return node


def project_representer(dumper, data):
    return dumper.represent_mapping('tag:yaml.org,2002:map',
                                    data.items())


def construct_yaml_map(self, node):
    data = collections.OrderedDict()
    yield data
    value = self.construct_mapping(node)

    if isinstance(node, yaml.MappingNode):
        self.flatten_mapping(node)
    else:
        raise yaml.constructor.ConstructorError(
            None, None,
            'expected a mapping node, but found %s' % node.id,
            node.start_mark)

    mapping = collections.OrderedDict()
    for key_node, value_node in node.value:
        key = self.construct_object(key_node, deep=False)
        try:
            hash(key)
        except TypeError as exc:
            raise yaml.constructor.ConstructorError(
                'while constructing a mapping', node.start_mark,
                'found unacceptable key (%s)' % exc, key_node.start_mark)
        value = self.construct_object(value_node, deep=False)
        mapping[key] = value
    data.update(mapping)


class IndentedEmitter(yaml.emitter.Emitter):
    def expect_block_sequence(self):
        self.increase_indent(flow=False, indentless=False)
        self.state = self.expect_first_block_sequence_item


class IndentedDumper(IndentedEmitter, yaml.serializer.Serializer,
                     yaml.representer.Representer, yaml.resolver.Resolver):
    def __init__(self, stream,
                 default_style=None, default_flow_style=None,
                 canonical=None, indent=None, width=None,
                 allow_unicode=None, line_break=None,
                 encoding=None, explicit_start=None, explicit_end=None,
                 version=None, tags=None):
        IndentedEmitter.__init__(
            self, stream, canonical=canonical,
            indent=indent, width=width,
            allow_unicode=allow_unicode,
            line_break=line_break)
        yaml.serializer.Serializer.__init__(
            self, encoding=encoding,
            explicit_start=explicit_start,
            explicit_end=explicit_end,
            version=version, tags=tags)
        yaml.representer.Representer.__init__(
            self, default_style=default_style,
            default_flow_style=default_flow_style)
        yaml.resolver.Resolver.__init__(self)


def ordered_load(stream, *args, **kwargs):
    return yaml.load(stream=stream, *args, **kwargs)


def ordered_dump(data, stream=None, *args, **kwargs):
    dumper = IndentedDumper
    # We need to do this because of how template expasion into a project
    # works. Without it, we end up with YAML references to the expanded jobs.
    dumper.ignore_aliases = lambda self, data: True

    output = yaml.dump(
        data, default_flow_style=False,
        Dumper=dumper, width=80, *args, **kwargs).replace(
            '\n    -', '\n\n    -')
    if stream:
        stream.write(output)
    else:
        return output


def get_single_key(var):
    if isinstance(var, str):
        return var
    elif isinstance(var, list):
        return var[0]
    return list(var.keys())[0]


def has_single_key(var):
    if isinstance(var, list):
        return len(var) == 1
    if isinstance(var, str):
        return True
    dict_keys = list(var.keys())
    if len(dict_keys) != 1:
        return False
    if var[get_single_key(var)]:
        return False
    return True


def combination_matches(combination, match_combinations):
    """
    Checks if the given combination is matches for any of the given combination
    globs, being those a set of combinations where if a key is missing, it's
    considered matching

    (key1=2, key2=3)

    would match the combination match:
    (key2=3)

    but not:
    (key1=2, key2=2)
    """
    for cmatch in match_combinations:
        for key, val in combination.items():
            if cmatch.get(key, val) != val:
                break
        else:
            return True
    return False


def expandYamlForTemplateJob(self, project, template, jobs_glob=None):
    dimensions = []
    template_name = template['name']
    orig_template = copy.deepcopy(template)

    # reject keys that are not useful during yaml expansion
    for k in ['jobs']:
        project.pop(k)
    excludes = project.pop('exclude', [])
    for (k, v) in project.items():
        tmpk = '{{{0}}}'.format(k)
        if tmpk not in template_name:
            continue
        if type(v) == list:
            dimensions.append(zip([k] * len(v), v))
    # XXX somewhat hackish to ensure we actually have a single
    # pass through the loop
    if len(dimensions) == 0:
        dimensions = [(("", ""),)]

    for values in itertools.product(*dimensions):
        params = copy.deepcopy(project)
        params = self.applyDefaults(params, template)

        expanded_values = {}
        for (k, v) in values:
            if isinstance(v, dict):
                inner_key = next(iter(v))
                expanded_values[k] = inner_key
                expanded_values.update(v[inner_key])
            else:
                expanded_values[k] = v

        params.update(expanded_values)
        params = deep_format(params, params)
        if combination_matches(params, excludes):
            log = logging.getLogger("zuul.Migrate.YamlParser")
            log.debug('Excluding combination %s', str(params))
            continue

        allow_empty_variables = self.config \
            and self.config.has_section('job_builder') \
            and self.config.has_option(
                'job_builder', 'allow_empty_variables') \
            and self.config.getboolean(
                'job_builder', 'allow_empty_variables')

        for key in template:
            if key not in params:
                params[key] = template[key]

        params['template-name'] = template_name
        project_name = params['name']
        params['name'] = '$ZUUL_SHORT_PROJECT_NAME'
        expanded = deep_format(template, params, allow_empty_variables)

        job_name = expanded.get('name')
        templated_job_name = job_name
        if job_name:
            job_name = job_name.replace(
                '$ZUUL_SHORT_PROJECT_NAME', project_name)
            expanded['name'] = job_name
        if jobs_glob and not matches(job_name, jobs_glob):
            continue

        self.formatDescription(expanded)
        expanded['orig_template'] = orig_template
        expanded['template_name'] = template_name
        self.jobs.append(expanded)
        JOBS_BY_ORIG_TEMPLATE[templated_job_name] = expanded


jenkins_jobs.parser.YamlParser.expandYamlForTemplateJob = \
    expandYamlForTemplateJob


class JJB(jenkins_jobs.builder.Builder):
    def __init__(self):
        self.global_config = None
        self._plugins_list = []

    def expandComponent(self, component_type, component, template_data):
        component_list_type = component_type + 's'
        new_components = []
        if isinstance(component, dict):
            name, component_data = next(iter(component.items()))
            if template_data:
                component_data = jenkins_jobs.formatter.deep_format(
                    component_data, template_data, True)
        else:
            name = component
            component_data = {}

        new_component = self.parser.data.get(component_type, {}).get(name)
        if new_component:
            for new_sub_component in new_component[component_list_type]:
                new_components.extend(
                    self.expandComponent(component_type,
                                         new_sub_component, component_data))
        else:
            new_components.append({name: component_data})
        return new_components

    def expandMacros(self, job):
        for component_type in ['builder', 'publisher', 'wrapper']:
            component_list_type = component_type + 's'
            new_components = []
            for new_component in job.get(component_list_type, []):
                # Skip macros defined in mappings.yaml
                if new_component in SKIP_MACROS:
                    continue
                new_components.extend(self.expandComponent(component_type,
                                                           new_component, {}))
            job[component_list_type] = new_components


class OldProject:
    def __init__(self, name, gate_jobs):
        self.name = name
        self.gate_jobs = gate_jobs


class OldJob:
    def __init__(self, name):
        self.name = name
        self.queue_name = None

    def __repr__(self):
        return self.name


class Job:

    log = logging.getLogger("zuul.Migrate")

    def __init__(self,
                 orig: str,
                 name: str=None,
                 content: Dict[str, Any]=None,
                 vars: Dict[str, str]=None,
                 nodes: List[str]=None,
                 parent=None) -> None:
        self.orig = orig
        self.voting = True
        self.name = name
        self.content = content.copy() if content else None
        self.vars = vars or {}
        self.required_projects = []  # type: ignore
        self.nodes = nodes or []
        self.parent = parent
        self.branch = None
        self.files = None
        self.jjb_job = None
        self.emit = True

        if self.content and not self.name:
            self.name = get_single_key(content)
        if not self.name:
            self.name = self.orig
        self.name = self.name.replace('-{name}', '').replace('{name}-', '')

        for suffix in SUFFIXES:
            suffix = '-{suffix}'.format(suffix=suffix)

            if self.name.endswith(suffix):
                self.name = self.name.replace(suffix, '')

    def _stripNodeName(self, node):
        node_key = '-{node}'.format(node=node)
        self.name = self.name.replace(node_key, '')

    def setNoEmit(self):
        self.emit = False

    def setVars(self, vars):
        self.vars = vars

    def setParent(self, parent):
        self.parent = parent

    def extractNode(self, default_node, labels):
        matching_label = None
        for label in labels:
            if label in self.orig:
                if not matching_label:
                    matching_label = label
                elif len(label) > len(matching_label):
                    matching_label = label

        if matching_label:
            if matching_label == default_node:
                self._stripNodeName(matching_label)
            else:
                self.nodes.append(matching_label)

    def getDepends(self):
        return [self.parent.name]

    def getNodes(self):
        return self.nodes

    def addJJBJob(self, jobs):
        if '{name}' in self.orig:
            self.jjb_job = JOBS_BY_ORIG_TEMPLATE[self.orig.format(
                name='$ZUUL_SHORT_PROJECT_NAME')]
        else:
            self.jjb_job = jobs[self.orig]

    def getTimeout(self):
        timeout = None
        if self.jjb_job:
            for wrapper in self.jjb_job.get('wrappers', []):
                if isinstance(wrapper, dict):
                    build_timeout = wrapper.get(
                        'build-timeout', wrapper.get('timeout'))
                    if isinstance(build_timeout, dict):
                        timeout = build_timeout.get('timeout')
                        if timeout is not None:
                            timeout = int(timeout) * 60
                            # NOTE: This should be read from tenant config
                            if timeout > 10800:
                                timeout = 10800

        return timeout

    @property
    def short_name(self):
        return self.name.replace('legacy-', '')

    @property
    def job_path(self):
        return 'playbooks/legacy/{name}'.format(name=self.short_name)

    def _getRsyncOptions(self, source):
        # If the source starts with ** then we want to match any
        # number of directories, so don't anchor the include filter.
        # If it does not start with **, then the intent is likely to
        # at least start by matching an immediate file or subdirectory
        # (even if later we have a ** in the middle), so in this case,
        # anchor it to the root of the transfer (the workspace).
        if not source.startswith('**'):
            source = os.path.join('/', source)
        # These options mean: include the thing we want, include any
        # directories (so that we continue to search for the thing we
        # want no matter how deep it is), exclude anything that
        # doesn't match the thing we want or is a directory, then get
        # rid of empty directories left over at the end.
        rsync_opts = ['--include=%s' % source,
                      '--include=*/',
                      '--exclude=*',
                      '--prune-empty-dirs']
        return rsync_opts

    def _makeSCPTask(self, publisher):
        # NOTE(mordred) About docs-draft manipulation:
        # The target of html/ was chosen to put the node contents into the
        # html dir inside of logs such that if the node's contents have an
        # index.html in them setting the success-url to html/ will render
        # things as expected. Existing builder macros look like:
        #
        #   - publisher:
        #     name: upload-sphinx-draft
        #     publishers:
        #       - scp:
        #           site: 'static.openstack.org'
        #           files:
        #             - target: 'docs-draft/$LOG_PATH'
        #               source: 'doc/build/html/**'
        #               keep-hierarchy: true
        #               copy-after-failure: true
        #
        # Which is pulling the tree of the remote html directory starting with
        # doc/build/html and putting that whole thing into
        # docs-draft/$LOG_PATH.
        #
        # Then there is a success-pattern in layout.yaml that looks like:
        #
        #     http://{url}/{log_path}/doc/build/html/
        #
        # Which gets reports. There are many variations on that URL. So rather
        # than needing to figure out varying success-urls to report in v3,
        # we'll remote the ** and not process this through the rsync_opts
        # processing we use for the other publishers, but instead will just
        # pass doc/build/html/ to get the contents of doc/build/html/ and we'll
        # put those in {{ log_root }}/html/ locally meaning the success-url
        # can always be html/. This should work for all values of source
        # from v2.
        tasks = []
        artifacts = False
        draft = False
        site = publisher['scp']['site']
        for scpfile in publisher['scp']['files']:
            if 'ZUUL_PROJECT' in scpfile.get('source', ''):
                self.log.error(
                    "Job {name} uses ZUUL_PROJECT in source".format(
                        name=self.name))
                continue

            if scpfile.get('copy-console'):
                continue
            else:
                src = "{{ ansible_user_dir }}/workspace/"
                rsync_opts = self._getRsyncOptions(scpfile['source'])

            target = scpfile['target']
            # TODO(mordred) Generalize this next section, it's SUPER
            # openstack specific. We can likely do this in mapping.yaml
            if site == 'static.openstack.org':
                for f in ('service-types', 'specs'):
                    if target.startswith(f):
                        self.log.error(
                            "Job {name} uses {f} publishing".format(
                                name=self.name, f=f))
                        continue
                if target.startswith('docs-draft'):
                    target = "{{ zuul.executor.log_root }}/html/"
                    src = scpfile['source'].replace('**', '')
                    rsync_opts = None
                    draft = True
                else:
                    target = target.replace(
                        'logs/$LOG_PATH',
                        "{{ zuul.executor.log_root }}")
            elif site == 'tarballs.openstack.org':
                if not target.startswith('tarballs'):
                    self.log.error(
                        'Job {name} wants to publish artifacts to non'
                        ' tarballs dir'.format(name=self.name))
                    continue
                if target.startswith('tarballs/ci'):
                    target = target.split('/', 3)[-1]
                else:
                    target = target.split('/', 2)[-1]
                target = "{{ zuul.executor.work_root }}/artifacts/" + target
                artifacts = True
            elif site == 'yaml2ical':
                self.log.error('Job {name} uses yaml2ical publisher')
                continue

            syncargs = collections.OrderedDict()
            syncargs['src'] = src
            syncargs['dest'] = target
            syncargs['mode'] = 'pull'
            syncargs['copy_links'] = True
            syncargs['verify_host'] = True
            if rsync_opts:
                syncargs['rsync_opts'] = rsync_opts
            task = collections.OrderedDict()
            task['name'] = 'Copy files from {src} on node'.format(src=src)
            task['synchronize'] = syncargs
            # We don't use retry_args here because there is a bug in
            # the synchronize module that breaks subsequent attempts at
            # retrying. Better to try once and get an accurate error
            # message if it fails.
            # https://github.com/ansible/ansible/issues/18281
            tasks.append(task)

        if artifacts:
            ensure_task = collections.OrderedDict()
            ensure_task['name'] = 'Ensure artifacts directory exists'
            ensure_task['file'] = collections.OrderedDict()
            ensure_task['file']['path'] = \
                "{{ zuul.executor.work_root }}/artifacts"
            ensure_task['file']['state'] = 'directory'
            ensure_task['delegate_to'] = 'localhost'
            tasks.insert(0, ensure_task)
        return dict(tasks=tasks, artifacts=artifacts, draft=draft)

    def _emitShellTask(self, data, syntax_check):
        shell, data = deal_with_shebang(data)
        task = collections.OrderedDict()
        # Putting the data directly into shell: causes here docs to break.
        task['shell'] = collections.OrderedDict()
        task['shell']['cmd'] = data
        if shell:
            task['shell']['executable'] = shell
        task['shell']['chdir'] = '{{ ansible_user_dir }}/workspace'

        if syntax_check:
            # Emit a test playbook with this shell task in it then run
            # ansible-playbook --syntax-check on it. This will fail if there
            # are embedding issues, such as with unbalanced single quotes
            # The end result should be less scripts and more shell
            play = dict(hosts='all', tasks=[task])
            (fd, tmp_path) = tempfile.mkstemp()
            try:
                f = os.fdopen(fd, 'w')
                ordered_dump([play], f)
                f.close()
                proc = subprocess.run(
                    ['ansible-playbook', '--syntax-check', tmp_path],
                    stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                if proc.returncode != 0:
                    # Return of None means we must emit a script
                    self.log.error(
                        "Job {name} had an ansible syntax error, using script"
                        " instead of shell".format(name=self.name))
                    return None
            finally:
                os.unlink(tmp_path)
        return task

    def _emitScriptContent(self, data, playbook_dir, seq):
        script_fn = '%s-%02d.sh' % (self.short_name, seq)
        script_path = os.path.join(playbook_dir, script_fn)

        with open(script_path, 'w') as script:
            if not data.startswith('#!'):
                data = '#!/bin/bash -x\n %s' % (data,)
            script.write(data)

        task = collections.OrderedDict()
        task['name'] = 'Running playbooks/legacy/{playbook}'.format(
            playbook=script_fn)
        task['script'] = script_fn
        return task

    def _makeBuilderTask(self, playbook_dir, builder, sequence, syntax_check):
        # Don't write a script to echo the template line
        # TODO(mordred) Put these into mapping.yaml
        if builder['shell'].startswith('echo JJB template: '):
            return
        if 'echo "Detailed logs:' in builder['shell']:
            return
        if ('cat /etc/dib-builddate.txt' in builder['shell'] and
                'echo "Network configuration data..."' in builder['shell']):
            return

        task = self._emitShellTask(builder['shell'], syntax_check)
        if not task:
            task = self._emitScriptContent(
                builder['shell'], playbook_dir, sequence)
        task['environment'] = ENVIRONMENT

        return task

    def _transformPublishers(self, jjb_job):
        early_publishers = []
        late_publishers = []
        old_publishers = jjb_job.get('publishers', [])
        for publisher in old_publishers:
            early_scpfiles = []
            late_scpfiles = []
            if 'scp' not in publisher:
                early_publishers.append(publisher)
                continue
            copy_console = False
            for scpfile in publisher['scp']['files']:
                if scpfile.get('copy-console'):
                    scpfile['keep-hierarchy'] = True
                    late_scpfiles.append(scpfile)
                    copy_console = True
                else:
                    early_scpfiles.append(scpfile)
            publisher['scp']['files'] = early_scpfiles + late_scpfiles
            if copy_console:
                late_publishers.append(publisher)
            else:
                early_publishers.append(publisher)
        publishers = early_publishers + late_publishers
        if old_publishers != publishers:
            self.log.debug("Transformed job publishers")
        return early_publishers, late_publishers

    def emitPlaybooks(self, jobsdir, syntax_check=False):
        has_artifacts = False
        has_draft = False
        if not self.jjb_job:
            if self.emit:
                self.log.error(
                    'Job {name} has no job content'.format(name=self.name))
            return False, False, False

        playbook_dir = os.path.join(jobsdir, self.job_path)
        if not os.path.exists(playbook_dir):
            os.makedirs(playbook_dir)

        run_playbook = os.path.join(self.job_path, 'run.yaml')
        post_playbook = os.path.join(self.job_path, 'post.yaml')

        tasks = []
        workspace_task = collections.OrderedDict()
        workspace_task['name'] = "Ensure legacy workspace directory"
        workspace_task['file'] = collections.OrderedDict()
        workspace_task['file']['path'] = '{{ ansible_user_dir }}/workspace'
        workspace_task['file']['state'] = 'directory'
        tasks.append(workspace_task)

        sequence = 0
        for builder in self.jjb_job.get('builders', []):
            if 'shell' in builder:
                self.required_projects.extend(
                    extract_projects(builder['shell']))
                task = self._makeBuilderTask(
                    playbook_dir, builder, sequence, syntax_check)
                if task:
                    if 'script' in task:
                        sequence += 1
                    tasks.append(task)
        play = collections.OrderedDict()
        play['hosts'] = 'all'
        play['name'] = 'Autoconverted job {name} from old job {old}'.format(
            name=self.name, old=self.orig)
        play['tasks'] = tasks

        with open(run_playbook, 'w') as run_playbook_out:
            ordered_dump([play], run_playbook_out)

        has_post = False
        tasks = []
        early_publishers, late_publishers = self._transformPublishers(
            self.jjb_job)
        for publishers in [early_publishers, late_publishers]:
            for publisher in publishers:
                if 'scp' in publisher:
                    ret = self._makeSCPTask(publisher)
                    if ret['artifacts']:
                        has_artifacts = True
                    if ret['draft']:
                        has_draft = True
                    tasks.extend(ret['tasks'])
                if 'afs' in builder:
                    self.log.error(
                        "Job {name} uses AFS publisher".format(name=self.name))
        if tasks:
            has_post = True
            play = collections.OrderedDict()
            play['hosts'] = 'all'
            play['tasks'] = tasks
            with open(post_playbook, 'w') as post_playbook_out:
                ordered_dump([play], post_playbook_out)
        return has_artifacts, has_post, has_draft

    def toJobDict(
            self, has_artifacts=False, has_post=False, has_draft=False,
            project_names=[]):
        output = collections.OrderedDict()
        output['name'] = self.name

        expanded_projects = []
        if self.required_projects:
            expanded_projects = expand_project_names(
                self.required_projects, project_names)
            # Look for project names in the job name. Lookie there - the
            # python in operator works on lists and strings.
            expanded_projects.extend(expand_project_names(
                self.name, project_names))

        output['parent'] = 'legacy-base'
        if 'dsvm' in self.name:
            output['parent'] = 'legacy-dsvm-base'
        elif 'puppet-openstack-integration' in self.name:
            output['parent'] = 'legacy-puppet-openstack-integration'
        elif 'openstack/puppet-openstack-integration' in expanded_projects:
            output['parent'] = 'legacy-puppet-openstack-integration'
        elif has_artifacts:
            output['parent'] = 'legacy-publish-openstack-artifacts'
        elif has_draft:
            output['success-url'] = 'html/'
        output['run'] = os.path.join(self.job_path, 'run.yaml')
        if has_post:
            output['post-run'] = os.path.join(self.job_path, 'post.yaml')

        if self.vars:
            output['vars'] = self.vars.copy()

        timeout = self.getTimeout()
        if timeout:
            output['timeout'] = timeout

        if self.nodes:
            if len(self.nodes) == 1:
                output['nodeset'] = self.getNodes()[0]
            else:
                output['nodeset'] = dict(nodes=self.getNodes())

        if expanded_projects:
            output['required-projects'] = sorted(list(set(expanded_projects)))

        if self.name in JOB_MATCHERS:
            for k, v in JOB_MATCHERS[self.name].items():
                if k in output:
                    self.log.error(
                        'Job %s has attributes directly and from matchers',
                        self.name)
                output[k] = v

        return output

    def toPipelineDict(self):
        if self.content:
            output = self.content
        else:
            output = collections.OrderedDict()
            output[self.name] = collections.OrderedDict()

        if self.parent:
            output[self.name].setdefault('dependencies', self.getDepends())

        if not self.voting:
            output[self.name].setdefault('voting', False)

        if self.vars:
            job_vars = output[self.name].get('vars', collections.OrderedDict())
            job_vars.update(self.vars)

        if self.branch:
            output[self.name]['branches'] = self.branch

        if self.files:
            output[self.name]['files'] = self.files

        if not output[self.name]:
            return self.name

        return output


class JobMapping:
    log = logging.getLogger("zuul.Migrate.JobMapping")

    def __init__(self, nodepool_config, layout, mapping_file=None):
        self.layout = layout
        self.job_direct = {}
        self.labels = []
        self.job_mapping = []
        self.template_mapping = {}
        self.jjb_jobs = {}
        self.seen_new_jobs = []
        self.unshare = []
        nodepool_data = ordered_load(open(nodepool_config, 'r'))
        for label in nodepool_data['labels']:
            self.labels.append(label['name'])
        if not mapping_file:
            self.default_node = 'ubuntu-xenial'
        else:
            mapping_data = ordered_load(open(mapping_file, 'r'))
            self.default_node = mapping_data['default-node']
            global SUFFIXES
            SUFFIXES = mapping_data.get('strip-suffixes', [])
            global SKIP_MACROS
            SKIP_MACROS = mapping_data.get('skip-macros', [])
            self.unshare = mapping_data.get('unshare', [])
            for map_info in mapping_data.get('job-mapping', []):
                if map_info['old'].startswith('^'):
                    map_info['pattern'] = re.compile(map_info['old'])
                    self.job_mapping.append(map_info)
                else:
                    self.job_direct[map_info['old']] = map_info['new']

            for map_info in mapping_data.get('template-mapping', []):
                self.template_mapping[map_info['old']] = map_info['new']

    def makeNewName(self, new_name, match_dict):
        return new_name.format(**match_dict)

    def hasProjectTemplate(self, old_name):
        return old_name in self.template_mapping

    def setJJBJobs(self, jjb_jobs):
        self.jjb_jobs = jjb_jobs

    def getNewTemplateName(self, old_name):
        return self.template_mapping.get(old_name, old_name)

    def mapNewJob(self, name, info) -> Optional[Job]:
        matches = info['pattern'].search(name)
        if not matches:
            return None
        match_dict = matches.groupdict()
        if isinstance(info['new'], dict):
            new_job = info['new']
            old_name = get_single_key(new_job)
            new_name = old_name.format(**match_dict)
            job = Job(orig=name, content={new_name: new_job[old_name]})
        else:
            job = Job(orig=name, name=info['new'].format(**match_dict))

        if 'vars' in info:
            job.setVars(self._expandVars(info, match_dict))

        return job

    def _expandVars(self, info, match_dict):
        job_vars = info['vars'].copy()
        for key in job_vars:
            job_vars[key] = job_vars[key].format(**match_dict)
        return job_vars

    def getNewJob(self, job_name, remove_gate):
        if job_name in self.job_direct:
            if isinstance(self.job_direct[job_name], dict):
                job = Job(job_name, content=self.job_direct[job_name])
            else:
                job = Job(job_name, name=self.job_direct[job_name])
            if job_name not in self.seen_new_jobs:
                self.seen_new_jobs.append(self.job_direct[job_name])
            job.setNoEmit()
            return job

        new_job = None
        for map_info in self.job_mapping:
            new_job = self.mapNewJob(job_name, map_info)
            if new_job:
                if job_name not in self.seen_new_jobs:
                    self.seen_new_jobs.append(new_job.name)
                new_job.setNoEmit()
                break
        if not new_job:
            orig_name = job_name
            if remove_gate:
                job_name = job_name.replace('gate-', '', 1)
            job_name = 'legacy-{job_name}'.format(job_name=job_name)
            new_job = Job(orig=orig_name, name=job_name)

        new_job.extractNode(self.default_node, self.labels)

        # Handle matchers
        for layout_job in self.layout.get('jobs', []):
            if re.search(layout_job['name'], new_job.orig):
                # Matchers that can apply to templates must be processed first
                # since project-specific matchers can cause the template to
                # be expanded into a project.
                if not layout_job.get('voting', True):
                    new_job.voting = False
                if layout_job.get('branch'):
                    new_job.branch = layout_job['branch']
                if layout_job.get('files'):
                    new_job.files = layout_job['files']

        new_job.addJJBJob(self.jjb_jobs)
        return new_job


class ChangeQueue:
    def __init__(self):
        self.name = ''
        self.assigned_name = None
        self.generated_name = None
        self.projects = []
        self._jobs = set()

    def getJobs(self):
        return self._jobs

    def getProjects(self):
        return [p.name for p in self.projects]

    def addProject(self, project):
        if project not in self.projects:
            self.projects.append(project)
            self._jobs |= project.gate_jobs

            names = [x.name for x in self.projects]
            names.sort()
            self.generated_name = names[0].split('/')[-1]

            for job in self._jobs:
                if job.queue_name:
                    if (self.assigned_name and
                            job.queue_name != self.assigned_name):
                        raise Exception("More than one name assigned to "
                                        "change queue: %s != %s" %
                                        (self.assigned_name,
                                         job.queue_name))
                    self.assigned_name = job.queue_name
            self.name = self.assigned_name or self.generated_name

    def mergeChangeQueue(self, other):
        for project in other.projects:
            self.addProject(project)


class ZuulMigrate:

    log = logging.getLogger("zuul.Migrate")

    def __init__(self, layout, job_config, nodepool_config,
                 outdir, mapping, syntax_check):
        self.layout = ordered_load(open(layout, 'r'))
        self.job_config = job_config
        self.outdir = outdir
        self.mapping = JobMapping(nodepool_config, self.layout, mapping)
        self.syntax_check = syntax_check

        self.jobs = {}
        self.old_jobs = {}
        self.job_objects = []
        self.new_templates = {}

    def run(self):
        self.loadJobs()
        self.buildChangeQueues()
        self.convertJobs()
        self.writeJobs()

    def loadJobs(self):
        self.log.debug("Loading jobs")
        builder = JJB()
        builder.load_files([self.job_config])
        builder.parser.expandYaml()
        unseen = set(self.jobs.keys())
        for job in builder.parser.jobs:
            builder.expandMacros(job)
            self.jobs[job['name']] = job
            unseen.discard(job['name'])
        for name in unseen:
            del self.jobs[name]
        self.mapping.setJJBJobs(self.jobs)

    def getOldJob(self, name):
        if name not in self.old_jobs:
            self.old_jobs[name] = OldJob(name)
        return self.old_jobs[name]

    def flattenOldJobs(self, tree, name=None):
        if isinstance(tree, str):
            n = tree.format(name=name)
            if n in self.mapping.unshare:
                return []
            return [self.getOldJob(n)]

        new_list = []  # type: ignore
        if isinstance(tree, list):
            for job in tree:
                new_list.extend(self.flattenOldJobs(job, name))
        elif isinstance(tree, dict):
            parent_name = get_single_key(tree)
            jobs = self.flattenOldJobs(tree[parent_name], name)
            for job in jobs:
                if job not in self.mapping.unshare:
                    new_list.append(self.getOldJob(job))
            if parent_name not in self.mapping.unshare:
                new_list.append(self.getOldJob(parent_name))
        return new_list

    def buildChangeQueues(self):
        self.log.debug("Building shared change queues")

        for j in self.layout['jobs']:
            if '^' in j['name'] or '$' in j['name']:
                continue
            job = self.getOldJob(j['name'])
            job.queue_name = j.get('queue-name')

        change_queues = []

        for project in self.layout.get('projects'):
            if 'gate' not in project:
                continue
            gate_jobs = set()
            for template in project.get('template', []):
                for pt in self.layout.get('project-templates'):
                    if pt['name'] != template['name']:
                        continue
                    if 'gate' not in pt['name']:
                        continue
                    gate_jobs |= set(self.flattenOldJobs(pt['gate'],
                                                         project['name']))
            gate_jobs |= set(self.flattenOldJobs(project['gate']))
            old_project = OldProject(project['name'], gate_jobs)
            change_queue = ChangeQueue()
            change_queue.addProject(old_project)
            change_queues.append(change_queue)
            self.log.debug("Created queue: %s" % change_queue)

        # Iterate over all queues trying to combine them, and keep doing
        # so until they can not be combined further.
        last_change_queues = change_queues
        while True:
            new_change_queues = self.combineChangeQueues(last_change_queues)
            if len(last_change_queues) == len(new_change_queues):
                break
            last_change_queues = new_change_queues

        self.log.debug("  Shared change queues:")
        for queue in new_change_queues:
            self.log.debug("    %s containing %s" % (
                queue, queue.generated_name))
        self.change_queues = new_change_queues

    def combineChangeQueues(self, change_queues):
        self.log.debug("Combining shared queues")
        new_change_queues = []
        for a in change_queues:
            merged_a = False
            for b in new_change_queues:
                if not a.getJobs().isdisjoint(b.getJobs()):
                    self.log.debug("Merging queue %s into %s" % (a, b))
                    b.mergeChangeQueue(a)
                    merged_a = True
                    break  # this breaks out of 'for b' and continues 'for a'
            if not merged_a:
                self.log.debug("Keeping queue %s" % (a))
                new_change_queues.append(a)
        return new_change_queues

    def convertJobs(self):
        pass

    def setupDir(self):
        zuul_d = os.path.join(self.outdir, 'zuul.d')
        job_outfile = os.path.join(zuul_d, 'zuul-legacy-jobs.yaml')
        project_outfile = os.path.join(zuul_d, 'projects.yaml')
        projects = []
        project_template_outfile = os.path.join(
            zuul_d, 'zuul-legacy-project-templates.yaml')
        if not os.path.exists(self.outdir):
            os.makedirs(self.outdir)
        if not os.path.exists(zuul_d):
            os.makedirs(zuul_d)
        elif os.path.exists(project_outfile):
            projects = ordered_load(open(project_outfile, 'r'))
        return dict(
            job_outfile=job_outfile,
            project_outfile=project_outfile,
            projects=projects,
            project_template_outfile=project_template_outfile,
        )

    def makeNewJobs(self, old_job, parent: Job=None):
        self.log.debug("makeNewJobs(%s)", old_job)
        if isinstance(old_job, str):
            remove_gate = True
            if old_job.startswith('gate-'):
                # Check to see if gate- and bare versions exist
                if old_job.replace('gate-', '', 1) in self.jobs:
                    remove_gate = False
            job = self.mapping.getNewJob(old_job, remove_gate)
            if parent:
                job.setParent(parent)
            return [job]

        new_list = []  # type: ignore
        if isinstance(old_job, list):
            for job in old_job:
                new_list.extend(self.makeNewJobs(job, parent=parent))

        elif isinstance(old_job, dict):
            parent_name = get_single_key(old_job)
            parent = self.makeNewJobs(parent_name, parent=parent)[0]

            jobs = self.makeNewJobs(old_job[parent_name], parent=parent)
            for job in jobs:
                new_list.append(job)
            new_list.append(parent)
        return new_list

    def writeProjectTemplate(self, template):
        new_template = collections.OrderedDict()
        if 'name' in template:
            new_template['name'] = template['name']
        for key, value in template.items():
            if key == 'name':
                continue

            # keep a cache of the Job objects so we can use it to get old
            # job name to new job name when expanding templates into projects.
            tmp = [job for job in self.makeNewJobs(value)]
            self.job_objects.extend(tmp)
            jobs = [job.toPipelineDict() for job in tmp]
            new_template[key] = dict(jobs=jobs)

        return new_template

    def scanForProjectMatchers(self, project_name):
        ''' Get list of job matchers that reference the given project name '''
        job_matchers = []
        for matcher in self.layout.get('jobs', []):
            for skipper in matcher.get('skip-if', []):
                if skipper.get('project'):
                    if re.search(skipper['project'], project_name):
                        job_matchers.append(matcher)
        return job_matchers

    def findReferencedTemplateNames(self, job_matchers, project_name):
        ''' Search templates in the layout file for matching jobs '''
        template_names = []

        def search_jobs(template):
            def _search(job):
                if isinstance(job, str):
                    for matcher in job_matchers:
                        if re.search(matcher['name'],
                                     job.format(name=project_name)):
                            template_names.append(template['name'])
                            return True
                elif isinstance(job, list):
                    for i in job:
                        if _search(i):
                            return True
                elif isinstance(job, dict):
                    for k, v in job.items():
                        if _search(k) or _search(v):
                            return True
                return False

            for key, value in template.items():
                if key == 'name':
                    continue
                for job in template[key]:
                    if _search(job):
                        return

        for template in self.layout.get('project-templates', []):
            search_jobs(template)
        return template_names

    def expandTemplateIntoProject(self, template_name, project):
        self.log.debug("EXPAND template %s into project %s",
                       template_name, project['name'])
        # find the new template since that's the thing we're expanding
        if template_name not in self.new_templates:
            self.log.error(
                "Template %s not found for expansion into project %s",
                template_name, project['name'])
            return

        template = self.new_templates[template_name]

        for pipeline, value in template.items():
            if pipeline == 'name':
                continue
            if pipeline not in project or 'jobs' not in project[pipeline]:
                project[pipeline] = dict(jobs=[])
            project[pipeline]['jobs'].extend(value['jobs'])

    def getOldJobName(self, new_job_name):
        for job in self.job_objects:
            if job.name == new_job_name:
                return job.orig
        return None

    def applyProjectMatchers(self, matchers, project, final=False):
        '''
        Apply per-project job matchers to the given project.

        :param matchers: Job matchers that referenced the given project.
        :param project: The new project object.
        '''

        def processPipeline(pipeline_jobs, job_name_regex, files):
            new_jobs = []
            for job in pipeline_jobs:
                if isinstance(job, str):
                    old_job_name = self.getOldJobName(job)
                    if old_job_name and re.search(
                            job_name_regex, old_job_name):
                        self.log.debug(
                            "Applied irrelevant-files to job %s in project %s",
                            job, project['name'])
                        job = {job: {'irrelevant-files':
                                     sorted(list(set(files)))}}
                elif isinstance(job, dict):
                    job = job.copy()
                    job_name = get_single_key(job)
                    extras = job[job_name]
                    old_job_name = self.getOldJobName(job_name)
                    if old_job_name and re.search(
                            job_name_regex, old_job_name):
                        self.log.debug(
                            "Applied irrelevant-files to complex job "
                            "%s in project %s", job_name, project['name'])
                        if 'irrelevant-files' not in extras:
                            extras['irrelevant-files'] = []
                        extras['irrelevant-files'].extend(files)
                        extras['irrelevant-files'] = sorted(list(
                            set(extras['irrelevant-files'])))
                    job[job_name] = extras
                new_jobs.append(job)
            return new_jobs

        def applyIrrelevantFiles(job_name_regex, files):
            for k, v in project.items():
                if k in ('templates', 'name'):
                    continue
                project[k]['jobs'] = processPipeline(
                    project[k].get('jobs', []), job_name_regex, files)

        if matchers:
            for matcher in matchers:
                # find the project-specific section
                for skipper in matcher.get('skip-if', []):
                    if skipper.get('project'):
                        if re.search(skipper['project'], project['name']):
                            if 'all-files-match-any' in skipper:
                                applyIrrelevantFiles(
                                    matcher['name'],
                                    skipper['all-files-match-any'])

        if not final:
            return

        for k, v in project.items():
            if k in ('templates', 'name'):
                continue
            jobs = []
            for job in project[k].get('jobs', []):
                if isinstance(job, dict):
                    job_name = get_single_key(job)
                else:
                    job_name = job
                if job_name in JOB_MATCHERS:
                    jobs.append(job)
                    continue
                orig_name = self.getOldJobName(job_name)
                if not orig_name:
                    jobs.append(job)
                    continue
                project_name = project['name']
                if '/' in project['name']:
                    project_name = project['name'].split('/')[1]
                orig_name = orig_name.format(name=project_name)
                info = {}
                for layout_job in self.mapping.layout.get('jobs', []):
                    if 'parameter-function' in layout_job:
                        continue
                    if 'skip-if' in layout_job:
                        continue
                    if re.search(layout_job['name'], orig_name):
                        if not layout_job.get('voting', True):
                            info['voting'] = False
                        if layout_job.get('branch'):
                            info['branches'] = layout_job['branch']
                        if layout_job.get('files'):
                            info['files'] = layout_job['files']
                        if not isinstance(job, dict):
                            job = {job: info}
                        else:
                            job[job_name].update(info)

                jobs.append(job)
            if jobs:
                project[k]['jobs'] = jobs

    def writeProject(self, project):
        '''
        Create a new v3 project definition.

        As part of creating the project, scan for project-specific job matchers
        referencing this project and remove the templates matching the job
        regex for that matcher. Expand the matched template(s) into the project
        so we can apply the project-specific matcher to the job(s).
        '''
        new_project = collections.OrderedDict()
        if 'name' in project:
            new_project['name'] = project['name']

        exp_template_names = TEMPLATES_TO_EXPAND.get(project['name'], [])

        templates_to_expand = []
        if 'template' in project:
            new_project['templates'] = []
            for template in project['template']:
                if template['name'] in exp_template_names:
                    templates_to_expand.append(template['name'])
                    continue
                new_project['templates'].append(
                    self.mapping.getNewTemplateName(template['name']))

        for key, value in project.items():
            if key in ('name', 'template'):
                continue
            else:
                new_project[key] = collections.OrderedDict()
                if key == 'gate':
                    for queue in self.change_queues:
                        if (project['name'] not in queue.getProjects() or
                                len(queue.getProjects()) == 1):
                            continue
                        new_project[key]['queue'] = queue.name
                tmp = [job for job in self.makeNewJobs(value)]
                # Don't insert into self.job_objects - that was done
                # in the speculative pass
                jobs = [job.toPipelineDict() for job in tmp]
                if jobs:
                    new_project[key]['jobs'] = jobs
                if not new_project[key]:
                    del new_project[key]

        for name in templates_to_expand:
            self.expandTemplateIntoProject(name, new_project)

        job_matchers = self.scanForProjectMatchers(project['name'])

        # Need a deep copy after expansion, else our templates end up
        # also getting this change.
        new_project = copy.deepcopy(new_project)
        self.applyProjectMatchers(job_matchers, new_project, final=True)

        return new_project

    def checkSpeculativeProject(self, project):
        '''
        Create a new v3 project definition expanding all templates.
        '''
        new_project = collections.OrderedDict()
        if 'name' in project:
            new_project['name'] = project['name']

        templates_to_expand = []
        for template in project.get('template', []):
            templates_to_expand.append(template['name'])

        # We have to do this section to expand self.job_objects
        for key, value in project.items():
            if key in ('name', 'template'):
                continue
            else:
                new_project[key] = collections.OrderedDict()
                if key == 'gate':
                    for queue in self.change_queues:
                        if project['name'] not in queue.getProjects():
                            continue
                        if len(queue.getProjects()) == 1:
                            continue
                        new_project[key]['queue'] = queue.name
                tmp = [job for job in self.makeNewJobs(value)]
                self.job_objects.extend(tmp)

        for name in templates_to_expand:

            expand_project = copy.deepcopy(new_project)
            self.expandTemplateIntoProject(name, expand_project)

            # Need a deep copy after expansion, else our templates end up
            # also getting this change.
            expand_project = copy.deepcopy(expand_project)
            job_matchers = self.scanForProjectMatchers(project['name'])
            self.applyProjectMatchers(job_matchers, expand_project)

            # We should now have a project-pipeline with only the
            # jobs expanded from this one template
            for project_part in expand_project.values():
                # The pipelines are dicts - we only want pipelines
                if isinstance(project_part, dict):
                    if 'jobs' not in project_part:
                        continue
                    self.processProjectTemplateExpansion(
                        project_part, project, name)

    def processProjectTemplateExpansion(self, project_part, project, template):
        # project_part should be {'jobs': []}
        job_list = project_part['jobs']
        for new_job in job_list:
            if isinstance(new_job, dict):
                new_job_name = get_single_key(new_job)
                info = new_job[new_job_name]
            else:
                new_job_name = new_job
                info = None
            orig_name = self.getOldJobName(new_job_name)
            if not orig_name:
                self.log.error("Job without old name: %s", new_job_name)
                continue
            project_name = project['name']
            if '/' in project['name']:
                project_name = project['name'].split('/')[1]
            orig_name = orig_name.format(name=project_name)

            for layout_job in self.mapping.layout.get('jobs', []):
                if 'parameter-function' in layout_job:
                    continue
                if re.search(layout_job['name'], orig_name):
                    if not info:
                        info = {}
                    if not layout_job.get('voting', True):
                        info['voting'] = False
                    if layout_job.get('branch'):
                        info['branches'] = layout_job['branch']
                    if layout_job.get('files'):
                        info['files'] = layout_job['files']

            if info:
                expansion = dict(info=info, template=template)
                JOBS_FOR_EXPAND[new_job_name][project['name']] = expansion

    def writeJobs(self):
        output_dir = self.setupDir()
        job_outfile = output_dir['job_outfile']
        project_outfile = output_dir['project_outfile']
        project_config = output_dir['projects']
        project_dicts = list_to_project_dicts(project_config)
        project_template_outfile = output_dir['project_template_outfile']
        job_config = []
        template_config = []

        for template in self.layout.get('project-templates', []):
            self.log.debug("Processing template: %s", template)
            new_template = self.writeProjectTemplate(template)
            self.new_templates[new_template['name']] = new_template
            if not self.mapping.hasProjectTemplate(template['name']):
                template_config.append({'project-template': new_template})
        template_config = sorted(
            template_config,
            key=lambda template: template['project-template']['name'])

        for project in self.layout.get('projects', []):
            self.checkSpeculativeProject(project)
        normalize_project_expansions()

        project_names = []
        for project in self.layout.get('projects', []):
            project_names.append(project['name'])
            project_dict = self.writeProject(project)
            merge_project_dict(
                project_dicts, project['name'],
                project_dict)
        project_config = project_dicts_to_list(project_dicts)

        seen_jobs = []
        for job in sorted(self.job_objects, key=lambda job: job.name):
            if (job.name not in seen_jobs and
                    job.name not in self.mapping.seen_new_jobs and
                    job.emit):
                has_artifacts, has_post, has_draft = job.emitPlaybooks(
                    self.outdir, self.syntax_check)
                job_config.append({'job': job.toJobDict(
                    has_artifacts, has_post, has_draft, project_names)})
                seen_jobs.append(job.name)
        job_config = sorted(
            job_config, key=lambda job: job['job']['name'])

        with open(job_outfile, 'w') as yamlout:
            # Insert an extra space between top-level list items
            yamlout.write(ordered_dump(job_config).replace('\n-', '\n\n-'))

        with open(project_template_outfile, 'w') as yamlout:
            # Insert an extra space between top-level list items
            yamlout.write(
                ordered_dump(template_config).replace('\n-', '\n\n-'))

        with open(project_outfile, 'w') as yamlout:
            # Insert an extra space between top-level list items
            yamlout.write(ordered_dump(project_config).replace('\n-', '\n\n-'))


def main():
    yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
                         construct_yaml_map)

    yaml.add_representer(collections.OrderedDict, project_representer,
                         Dumper=IndentedDumper)
    yaml.representer.BaseRepresenter.represent_scalar = my_represent_scalar

    parser = argparse.ArgumentParser(description=DESCRIPTION)
    parser.add_argument(
        'layout',
        help="The Zuul v2 layout.yaml file to read.")
    parser.add_argument(
        'job_config',
        help="Directory containing Jenkins Job Builder job definitions.")
    parser.add_argument(
        'nodepool_config',
        help="Nodepool config file containing complete set of node names")
    parser.add_argument(
        'outdir',
        help="A directory into which the Zuul v3 config will be written.")
    parser.add_argument(
        '--mapping',
        default=None,
        help="A filename with a yaml mapping of old name to new name.")
    parser.add_argument(
        '-v', dest='verbose', action='store_true', help='verbose output')
    parser.add_argument(
        '--syntax-check', dest='syntax_check', action='store_true',
        help='Run ansible-playbook --syntax-check on generated playbooks')
    args = parser.parse_args()
    if args.verbose:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.INFO)

    ZuulMigrate(args.layout, args.job_config, args.nodepool_config,
                args.outdir, args.mapping, args.syntax_check).run()