diff options
33 files changed, 950 insertions, 59 deletions
diff --git a/.zuul.yaml b/.zuul.yaml index a87c1965a..7473ad3db 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -65,7 +65,5 @@ - zuul-stream-functional post: jobs: - - publish-openstack-sphinx-docs-infra: - vars: - sphinx_python: python3 + - publish-openstack-sphinx-docs-infra-python3 - publish-openstack-python-branch-tarball diff --git a/bindep.txt b/bindep.txt index 85254b4cc..3dcc3e7cd 100644 --- a/bindep.txt +++ b/bindep.txt @@ -8,7 +8,7 @@ openssl [test] zookeeperd [platform:dpkg] build-essential [platform:dpkg] gcc [platform:rpm] -graphviz [test] +graphviz [doc] libssl-dev [platform:dpkg] openssl-devel [platform:rpm] libffi-dev [platform:dpkg] diff --git a/doc/source/admin/connections.rst b/doc/source/admin/connections.rst index 29ca3be7c..55ac629c1 100644 --- a/doc/source/admin/connections.rst +++ b/doc/source/admin/connections.rst @@ -55,6 +55,7 @@ Zuul includes the following drivers: drivers/gerrit drivers/github + drivers/git drivers/smtp drivers/sql drivers/timer diff --git a/doc/source/admin/drivers/git.rst b/doc/source/admin/drivers/git.rst new file mode 100644 index 000000000..e0acec116 --- /dev/null +++ b/doc/source/admin/drivers/git.rst @@ -0,0 +1,59 @@ +:title: Git Driver + +Git +=== + +This driver can be used to load Zuul configuration from public Git repositories, +for instance from ``openstack-infra/zuul-jobs`` that is suitable for use by +any Zuul system. It can also be used to trigger jobs from ``ref-updated`` events +in a pipeline. + +Connection Configuration +------------------------ + +The supported options in ``zuul.conf`` connections are: + +.. attr:: <git connection> + + .. attr:: driver + :required: + + .. value:: git + + The connection must set ``driver=git`` for Git connections. + + .. attr:: baseurl + + Path to the base Git URL. Git repos name will be appended to it. + + .. attr:: poll_delay + :default: 7200 + + The delay in seconds of the Git repositories polling loop. + +Trigger Configuration +--------------------- + +.. attr:: pipeline.trigger.<git source> + + The dictionary passed to the Git pipeline ``trigger`` attribute + supports the following attributes: + + .. attr:: event + :required: + + Only ``ref-updated`` is supported. + + .. attr:: ref + + On ref-updated events, a ref such as ``refs/heads/master`` or + ``^refs/tags/.*$``. This field is treated as a regular expression, + and multiple refs may be listed. + + .. attr:: ignore-deletes + :default: true + + When a ref is deleted, a ref-updated event is emitted with a + newrev of all zeros specified. The ``ignore-deletes`` field is a + boolean value that describes whether or not these newrevs + trigger ref-updated events. diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst index 916e66ad9..fff673b55 100644 --- a/doc/source/user/config.rst +++ b/doc/source/user/config.rst @@ -1032,11 +1032,12 @@ pipeline. The following attributes may appear in a project: .. attr:: name - :required: The name of the project. If Zuul is configured with two or more unique projects with the same name, the canonical hostname for the project should be included (e.g., `git.example.com/foo`). + If not given it is implicitly derived from the project where this + is defined. .. attr:: templates diff --git a/tests/base.py b/tests/base.py index 69d9f5522..7e63129ea 100755 --- a/tests/base.py +++ b/tests/base.py @@ -1432,7 +1432,8 @@ class RecordingAnsibleJob(zuul.executor.server.AnsibleJob): self.log.debug("hostlist") hosts = super(RecordingAnsibleJob, self).getHostList(args) for host in hosts: - host['host_vars']['ansible_connection'] = 'local' + if not host['host_vars'].get('ansible_connection'): + host['host_vars']['ansible_connection'] = 'local' hosts.append(dict( name=['localhost'], @@ -1738,6 +1739,9 @@ class FakeNodepool(object): executor='fake-nodepool') if 'fakeuser' in node_type: data['username'] = 'fakeuser' + if 'windows' in node_type: + data['connection_type'] = 'winrm' + data = json.dumps(data).encode('utf8') path = self.client.create(path, data, makepath=True, @@ -2833,6 +2837,16 @@ class ZuulTestCase(BaseTestCase): os.path.join(FIXTURE_DIR, f.name)) self.setupAllProjectKeys() + def addTagToRepo(self, project, name, sha): + path = os.path.join(self.upstream_root, project) + repo = git.Repo(path) + repo.git.tag(name, sha) + + def delTagFromRepo(self, project, name): + path = os.path.join(self.upstream_root, project) + repo = git.Repo(path) + repo.git.tag('-d', name) + def addCommitToRepo(self, project, message, files, branch='master', tag=None): path = os.path.join(self.upstream_root, project) diff --git a/tests/fixtures/config/git-driver/git/common-config/playbooks/project-test2.yaml b/tests/fixtures/config/git-driver/git/common-config/playbooks/project-test2.yaml new file mode 100644 index 000000000..f679dceae --- /dev/null +++ b/tests/fixtures/config/git-driver/git/common-config/playbooks/project-test2.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/git-driver/git/common-config/zuul.yaml b/tests/fixtures/config/git-driver/git/common-config/zuul.yaml index 784b5f2b6..53fc21073 100644 --- a/tests/fixtures/config/git-driver/git/common-config/zuul.yaml +++ b/tests/fixtures/config/git-driver/git/common-config/zuul.yaml @@ -19,6 +19,10 @@ name: project-test1 run: playbooks/project-test1.yaml +- job: + name: project-test2 + run: playbooks/project-test2.yaml + - project: name: org/project check: diff --git a/tests/fixtures/config/implicit-project/git/common-config/playbooks/test-common.yaml b/tests/fixtures/config/implicit-project/git/common-config/playbooks/test-common.yaml new file mode 100644 index 000000000..f679dceae --- /dev/null +++ b/tests/fixtures/config/implicit-project/git/common-config/playbooks/test-common.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/implicit-project/git/common-config/zuul.yaml b/tests/fixtures/config/implicit-project/git/common-config/zuul.yaml new file mode 100644 index 000000000..038c412dd --- /dev/null +++ b/tests/fixtures/config/implicit-project/git/common-config/zuul.yaml @@ -0,0 +1,57 @@ +- pipeline: + name: check + manager: independent + post-review: true + trigger: + gerrit: + - event: patchset-created + success: + gerrit: + Verified: 1 + failure: + gerrit: + Verified: -1 + +- pipeline: + name: gate + manager: dependent + success-message: Build succeeded (gate). + trigger: + gerrit: + - event: comment-added + approval: + - Approved: 1 + success: + gerrit: + Verified: 2 + submit: true + failure: + gerrit: + Verified: -2 + start: + gerrit: + Verified: 0 + precedence: high + + +- job: + name: base + parent: null + +- job: + name: test-common + run: playbooks/test-common.yaml + +- project: + check: + jobs: + - test-common + +- project: + name: org/project + check: + jobs: + - test-common + gate: + jobs: + - test-common diff --git a/tests/fixtures/config/implicit-project/git/org_project/.zuul.yaml b/tests/fixtures/config/implicit-project/git/org_project/.zuul.yaml new file mode 100644 index 000000000..bce195cc6 --- /dev/null +++ b/tests/fixtures/config/implicit-project/git/org_project/.zuul.yaml @@ -0,0 +1,11 @@ +- job: + name: test-project + run: playbooks/test-project.yaml + +- project: + check: + jobs: + - test-project + gate: + jobs: + - test-project diff --git a/tests/fixtures/config/implicit-project/git/org_project/playbooks/test-project.yaml b/tests/fixtures/config/implicit-project/git/org_project/playbooks/test-project.yaml new file mode 100644 index 000000000..f679dceae --- /dev/null +++ b/tests/fixtures/config/implicit-project/git/org_project/playbooks/test-project.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/implicit-project/main.yaml b/tests/fixtures/config/implicit-project/main.yaml new file mode 100644 index 000000000..208e274b1 --- /dev/null +++ b/tests/fixtures/config/implicit-project/main.yaml @@ -0,0 +1,8 @@ +- tenant: + name: tenant-one + source: + gerrit: + config-projects: + - common-config + untrusted-projects: + - org/project diff --git a/tests/fixtures/config/inventory/git/common-config/zuul.yaml b/tests/fixtures/config/inventory/git/common-config/zuul.yaml index ad530a783..36789a321 100644 --- a/tests/fixtures/config/inventory/git/common-config/zuul.yaml +++ b/tests/fixtures/config/inventory/git/common-config/zuul.yaml @@ -38,6 +38,8 @@ label: default-label - name: fakeuser label: fakeuser-label + - name: windows + label: windows-label - job: name: base diff --git a/tests/fixtures/layouts/basic-git.yaml b/tests/fixtures/layouts/basic-git.yaml new file mode 100644 index 000000000..068d0a0ea --- /dev/null +++ b/tests/fixtures/layouts/basic-git.yaml @@ -0,0 +1,37 @@ +- pipeline: + name: post + manager: independent + trigger: + git: + - event: ref-updated + ref: ^refs/heads/.*$ + +- pipeline: + name: tag + manager: independent + trigger: + git: + - event: ref-updated + ref: ^refs/tags/.*$ + +- job: + name: base + parent: null + run: playbooks/base.yaml + +- job: + name: post-job + run: playbooks/post-job.yaml + +- job: + name: tag-job + run: playbooks/post-job.yaml + +- project: + name: org/project + post: + jobs: + - post-job + tag: + jobs: + - tag-job diff --git a/tests/fixtures/zuul-git-driver.conf b/tests/fixtures/zuul-git-driver.conf index b24b0a1b4..23a2a622c 100644 --- a/tests/fixtures/zuul-git-driver.conf +++ b/tests/fixtures/zuul-git-driver.conf @@ -21,6 +21,7 @@ sshkey=none [connection git] driver=git baseurl="" +poll_delay=0.1 [connection outgoing_smtp] driver=smtp diff --git a/tests/unit/test_git_driver.py b/tests/unit/test_git_driver.py index 1cfadf470..b9e6c6e92 100644 --- a/tests/unit/test_git_driver.py +++ b/tests/unit/test_git_driver.py @@ -12,7 +12,12 @@ # License for the specific language governing permissions and limitations # under the License. -from tests.base import ZuulTestCase + +import os +import time +import yaml + +from tests.base import ZuulTestCase, simple_layout class TestGitDriver(ZuulTestCase): @@ -23,7 +28,7 @@ class TestGitDriver(ZuulTestCase): super(TestGitDriver, self).setup_config() self.config.set('connection git', 'baseurl', self.upstream_root) - def test_git_driver(self): + def test_basic(self): tenant = self.sched.abide.tenants.get('tenant-one') # Check that we have the git source for common-config and the # gerrit source for the project. @@ -40,3 +45,127 @@ class TestGitDriver(ZuulTestCase): self.waitUntilSettled() self.assertEqual(len(self.history), 1) self.assertEqual(A.reported, 1) + + def test_config_refreshed(self): + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + self.assertEqual(len(self.history), 1) + self.assertEqual(A.reported, 1) + self.assertEqual(self.history[0].name, 'project-test1') + + # Update zuul.yaml to force a tenant reconfiguration + path = os.path.join(self.upstream_root, 'common-config', 'zuul.yaml') + config = yaml.load(open(path, 'r').read()) + change = { + 'name': 'org/project', + 'check': { + 'jobs': [ + 'project-test2' + ] + } + } + config[4]['project'] = change + files = {'zuul.yaml': yaml.dump(config)} + self.addCommitToRepo( + 'common-config', 'Change zuul.yaml configuration', files) + + # Let some time for the tenant reconfiguration to happen + time.sleep(2) + self.waitUntilSettled() + + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + self.assertEqual(len(self.history), 2) + self.assertEqual(A.reported, 1) + # We make sure the new job has run + self.assertEqual(self.history[1].name, 'project-test2') + + # Let's stop the git Watcher to let us merge some changes commits + # We want to verify that config changes are detected for commits + # on the range oldrev..newrev + self.sched.connections.getSource('git').connection.w_pause = True + # Add a config change + change = { + 'name': 'org/project', + 'check': { + 'jobs': [ + 'project-test1' + ] + } + } + config[4]['project'] = change + files = {'zuul.yaml': yaml.dump(config)} + self.addCommitToRepo( + 'common-config', 'Change zuul.yaml configuration', files) + # Add two other changes + self.addCommitToRepo( + 'common-config', 'Adding f1', + {'f1': "Content"}) + self.addCommitToRepo( + 'common-config', 'Adding f2', + {'f2': "Content"}) + # Restart the git watcher + self.sched.connections.getSource('git').connection.w_pause = False + + # Let some time for the tenant reconfiguration to happen + time.sleep(2) + self.waitUntilSettled() + + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + self.assertEqual(len(self.history), 3) + self.assertEqual(A.reported, 1) + # We make sure the new job has run + self.assertEqual(self.history[2].name, 'project-test1') + + def ensure_watcher_has_context(self): + # Make sure watcher have read initial refs shas + cnx = self.sched.connections.getSource('git').connection + delay = 0.1 + max_delay = 1 + while not cnx.projects_refs: + time.sleep(delay) + max_delay -= delay + if max_delay <= 0: + raise Exception("Timeout waiting for initial read") + + @simple_layout('layouts/basic-git.yaml', driver='git') + def test_ref_updated_event(self): + self.ensure_watcher_has_context() + # Add a commit to trigger a ref-updated event + self.addCommitToRepo( + 'org/project', 'A change for ref-updated', {'f1': 'Content'}) + # Let some time for the git watcher to detect the ref-update event + time.sleep(0.2) + self.waitUntilSettled() + self.assertEqual(len(self.history), 1) + self.assertEqual('SUCCESS', + self.getJobFromHistory('post-job').result) + + @simple_layout('layouts/basic-git.yaml', driver='git') + def test_ref_created(self): + self.ensure_watcher_has_context() + # Tag HEAD to trigger a ref-updated event + self.addTagToRepo( + 'org/project', 'atag', 'HEAD') + # Let some time for the git watcher to detect the ref-update event + time.sleep(0.2) + self.waitUntilSettled() + self.assertEqual(len(self.history), 1) + self.assertEqual('SUCCESS', + self.getJobFromHistory('tag-job').result) + + @simple_layout('layouts/basic-git.yaml', driver='git') + def test_ref_deleted(self): + self.ensure_watcher_has_context() + # Delete default tag init to trigger a ref-updated event + self.delTagFromRepo( + 'org/project', 'init') + # Let some time for the git watcher to detect the ref-update event + time.sleep(0.2) + self.waitUntilSettled() + # Make sure no job as run as ignore-delete is True by default + self.assertEqual(len(self.history), 0) diff --git a/tests/unit/test_inventory.py b/tests/unit/test_inventory.py index 1c41f5fa5..be504475a 100644 --- a/tests/unit/test_inventory.py +++ b/tests/unit/test_inventory.py @@ -119,5 +119,15 @@ class TestInventory(ZuulTestCase): self.assertEqual( inventory['all']['hosts'][node_name]['ansible_user'], username) + # check if the nodes use the correct or no ansible_connection + if node_name == 'windows': + self.assertEqual( + inventory['all']['hosts'][node_name]['ansible_connection'], + 'winrm') + else: + self.assertEqual( + 'local', + inventory['all']['hosts'][node_name]['ansible_connection']) + self.executor_server.release() self.waitUntilSettled() diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py index aacc81e00..6bbf098fb 100755 --- a/tests/unit/test_scheduler.py +++ b/tests/unit/test_scheduler.py @@ -6070,6 +6070,77 @@ class TestSemaphoreMultiTenant(ZuulTestCase): self.assertEqual(B.reported, 1) +class TestImplicitProject(ZuulTestCase): + tenant_config_file = 'config/implicit-project/main.yaml' + + def test_implicit_project(self): + # config project should work with implicit project name + A = self.fake_gerrit.addFakeChange('common-config', 'master', 'A') + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + + # untrusted project should work with implicit project name + B = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) + + self.waitUntilSettled() + + self.assertEqual(A.data['status'], 'NEW') + self.assertEqual(A.reported, 1) + self.assertEqual(B.data['status'], 'NEW') + self.assertEqual(B.reported, 1) + self.assertHistory([ + dict(name='test-common', result='SUCCESS', changes='1,1'), + dict(name='test-common', result='SUCCESS', changes='2,1'), + dict(name='test-project', result='SUCCESS', changes='2,1'), + ], ordered=False) + + # now test adding a further project in repo + in_repo_conf = textwrap.dedent( + """ + - job: + name: test-project + run: playbooks/test-project.yaml + - job: + name: test2-project + run: playbooks/test-project.yaml + + - project: + check: + jobs: + - test-project + gate: + jobs: + - test-project + + - project: + check: + jobs: + - test2-project + gate: + jobs: + - test2-project + + """) + file_dict = {'.zuul.yaml': in_repo_conf} + C = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', + files=file_dict) + C.addApproval('Code-Review', 2) + self.fake_gerrit.addEvent(C.addApproval('Approved', 1)) + self.waitUntilSettled() + + # change C must be merged + self.assertEqual(C.data['status'], 'MERGED') + self.assertEqual(C.reported, 2) + self.assertHistory([ + dict(name='test-common', result='SUCCESS', changes='1,1'), + dict(name='test-common', result='SUCCESS', changes='2,1'), + dict(name='test-project', result='SUCCESS', changes='2,1'), + dict(name='test-common', result='SUCCESS', changes='3,1'), + dict(name='test-project', result='SUCCESS', changes='3,1'), + dict(name='test2-project', result='SUCCESS', changes='3,1'), + ], ordered=False) + + class TestSemaphoreInRepo(ZuulTestCase): config_file = 'zuul-connections-gerrit-and-github.conf' tenant_config_file = 'config/in-repo/main.yaml' diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py index 44aa96665..2779e6e66 100755 --- a/tests/unit/test_v3.py +++ b/tests/unit/test_v3.py @@ -543,11 +543,23 @@ class TestInRepoConfig(ZuulTestCase): name: project-test2 run: playbooks/project-test2.yaml + - job: + name: project-test3 + run: playbooks/project-test2.yaml + + # add a job by the short project name - project: name: org/project tenant-one-gate: jobs: - project-test2 + + # add a job by the canonical project name + - project: + name: review.example.com/org/project + tenant-one-gate: + jobs: + - project-test3 """) in_repo_playbook = textwrap.dedent( @@ -569,7 +581,9 @@ class TestInRepoConfig(ZuulTestCase): self.assertIn('tenant-one-gate', A.messages[1], "A should transit tenant-one gate") self.assertHistory([ - dict(name='project-test2', result='SUCCESS', changes='1,1')]) + dict(name='project-test2', result='SUCCESS', changes='1,1'), + dict(name='project-test3', result='SUCCESS', changes='1,1'), + ], ordered=False) self.fake_gerrit.addEvent(A.getChangeMergedEvent()) self.waitUntilSettled() @@ -584,7 +598,10 @@ class TestInRepoConfig(ZuulTestCase): 'SUCCESS') self.assertHistory([ dict(name='project-test2', result='SUCCESS', changes='1,1'), - dict(name='project-test2', result='SUCCESS', changes='2,1')]) + dict(name='project-test3', result='SUCCESS', changes='1,1'), + dict(name='project-test2', result='SUCCESS', changes='2,1'), + dict(name='project-test3', result='SUCCESS', changes='2,1'), + ], ordered=False) def test_dynamic_template(self): # Tests that a project can't update a template in another diff --git a/tools/github-debugging.py b/tools/github-debugging.py new file mode 100644 index 000000000..171627ab9 --- /dev/null +++ b/tools/github-debugging.py @@ -0,0 +1,55 @@ +import github3 +import logging +import time + +# This is a template with boilerplate code for debugging github issues + +# TODO: for real use override the following variables +url = 'https://example.com' +api_token = 'xxxx' +org = 'org' +project = 'project' +pull_nr = 3 + + +# Send the logs to stderr as well +stream_handler = logging.StreamHandler() + + +logger_urllib3 = logging.getLogger('requests.packages.logger_urllib3') +# logger_urllib3.addHandler(stream_handler) +logger_urllib3.setLevel(logging.DEBUG) + +logger = logging.getLogger('github3') +# logger.addHandler(stream_handler) +logger.setLevel(logging.DEBUG) + + +github = github3.GitHubEnterprise(url) + + +# This is the currently broken cache adapter, enable or replace it to debug +# caching + +# import cachecontrol +# from cachecontrol.cache import DictCache +# cache_adapter = cachecontrol.CacheControlAdapter( +# DictCache(), +# cache_etags=True) +# +# github.session.mount('http://', cache_adapter) +# github.session.mount('https://', cache_adapter) + + +github.login(token=api_token) + +i = 0 +while True: + pr = github.pull_request(org, project, pull_nr) + prdict = pr.as_dict() + issue = pr.issue() + labels = list(issue.labels()) + print(labels) + i += 1 + print(i) + time.sleep(1) diff --git a/zuul/configloader.py b/zuul/configloader.py index 71c4ccc83..3a7e9b970 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -852,7 +852,7 @@ class ProjectParser(object): def getSchema(self): project = { - vs.Required('name'): str, + 'name': str, 'description': str, 'templates': [str], 'merge-mode': vs.Any('merge', 'merge-resolve', @@ -1228,8 +1228,8 @@ class TenantParser(object): tenant.config_projects, tenant.untrusted_projects, cached, tenant) - unparsed_config.extend(tenant.config_projects_config, tenant=tenant) - unparsed_config.extend(tenant.untrusted_projects_config, tenant=tenant) + unparsed_config.extend(tenant.config_projects_config, tenant) + unparsed_config.extend(tenant.untrusted_projects_config, tenant) tenant.layout = TenantParser._parseLayout(base, tenant, unparsed_config, scheduler, @@ -1484,10 +1484,10 @@ class TenantParser(object): (job.project,)) if job.config_project: config_projects_config.extend( - job.project.unparsed_config) + job.project.unparsed_config, tenant) else: untrusted_projects_config.extend( - job.project.unparsed_config) + job.project.unparsed_config, tenant) continue TenantParser.log.debug("Waiting for cat job %s" % (job,)) job.wait() @@ -1518,17 +1518,18 @@ class TenantParser(object): branch = source_context.branch if source_context.trusted: incdata = TenantParser._parseConfigProjectLayout( - job.files[fn], source_context) - config_projects_config.extend(incdata) + job.files[fn], source_context, tenant) + config_projects_config.extend(incdata, tenant) else: incdata = TenantParser._parseUntrustedProjectLayout( - job.files[fn], source_context) - untrusted_projects_config.extend(incdata) - new_project_unparsed_config[project].extend(incdata) + job.files[fn], source_context, tenant) + untrusted_projects_config.extend(incdata, tenant) + new_project_unparsed_config[project].extend( + incdata, tenant) if branch in new_project_unparsed_branch_config.get( project, {}): new_project_unparsed_branch_config[project][branch].\ - extend(incdata) + extend(incdata, tenant) # Now that we've sucessfully loaded all of the configuration, # cache the unparsed data on the project objects. for project, data in new_project_unparsed_config.items(): @@ -1540,18 +1541,18 @@ class TenantParser(object): return config_projects_config, untrusted_projects_config @staticmethod - def _parseConfigProjectLayout(data, source_context): + def _parseConfigProjectLayout(data, source_context, tenant): # This is the top-level configuration for a tenant. config = model.UnparsedTenantConfig() with early_configuration_exceptions(source_context): - config.extend(safe_load_yaml(data, source_context)) + config.extend(safe_load_yaml(data, source_context), tenant) return config @staticmethod - def _parseUntrustedProjectLayout(data, source_context): + def _parseUntrustedProjectLayout(data, source_context, tenant): config = model.UnparsedTenantConfig() with early_configuration_exceptions(source_context): - config.extend(safe_load_yaml(data, source_context)) + config.extend(safe_load_yaml(data, source_context), tenant) if config.pipelines: with configuration_exceptions('pipeline', config.pipelines[0]): raise PipelineNotPermittedError() @@ -1753,7 +1754,7 @@ class ConfigLoader(object): else: incdata = project.unparsed_branch_config.get(branch) if incdata: - config.extend(incdata) + config.extend(incdata, tenant) continue # Otherwise, do not use the cached config (even if the # files are empty as that likely means they were deleted). @@ -1782,12 +1783,12 @@ class ConfigLoader(object): if trusted: incdata = TenantParser._parseConfigProjectLayout( - data, source_context) + data, source_context, tenant) else: incdata = TenantParser._parseUntrustedProjectLayout( - data, source_context) + data, source_context, tenant) - config.extend(incdata) + config.extend(incdata, tenant) def createDynamicLayout(self, tenant, files, include_config_projects=False, diff --git a/zuul/driver/git/__init__.py b/zuul/driver/git/__init__.py index 0faa0365a..1fe43f643 100644 --- a/zuul/driver/git/__init__.py +++ b/zuul/driver/git/__init__.py @@ -15,6 +15,7 @@ from zuul.driver import Driver, ConnectionInterface, SourceInterface from zuul.driver.git import gitconnection from zuul.driver.git import gitsource +from zuul.driver.git import gittrigger class GitDriver(Driver, ConnectionInterface, SourceInterface): @@ -23,9 +24,15 @@ class GitDriver(Driver, ConnectionInterface, SourceInterface): def getConnection(self, name, config): return gitconnection.GitConnection(self, name, config) + def getTrigger(self, connection, config=None): + return gittrigger.GitTrigger(self, connection, config) + def getSource(self, connection): return gitsource.GitSource(self, connection) + def getTriggerSchema(self): + return gittrigger.getSchema() + def getRequireSchema(self): return {} diff --git a/zuul/driver/git/gitconnection.py b/zuul/driver/git/gitconnection.py index f93824d2f..03b24cadc 100644 --- a/zuul/driver/git/gitconnection.py +++ b/zuul/driver/git/gitconnection.py @@ -13,12 +13,119 @@ # License for the specific language governing permissions and limitations # under the License. +import os +import git +import time import logging import urllib +import threading import voluptuous as v from zuul.connection import BaseConnection +from zuul.driver.git.gitmodel import GitTriggerEvent, EMPTY_GIT_REF +from zuul.model import Ref, Branch + + +class GitWatcher(threading.Thread): + log = logging.getLogger("connection.git.GitWatcher") + + def __init__(self, git_connection, baseurl, poll_delay): + threading.Thread.__init__(self) + self.daemon = True + self.git_connection = git_connection + self.baseurl = baseurl + self.poll_delay = poll_delay + self._stopped = False + self.projects_refs = self.git_connection.projects_refs + + def compareRefs(self, project, refs): + partial_events = [] + # Fetch previous refs state + base_refs = self.projects_refs.get(project) + # Create list of created refs + rcreateds = set(refs.keys()) - set(base_refs.keys()) + # Create list of deleted refs + rdeleteds = set(base_refs.keys()) - set(refs.keys()) + # Create the list of updated refs + updateds = {} + for ref, sha in refs.items(): + if ref in base_refs and base_refs[ref] != sha: + updateds[ref] = sha + for ref in rcreateds: + event = { + 'ref': ref, + 'branch_created': True, + 'oldrev': EMPTY_GIT_REF, + 'newrev': refs[ref] + } + partial_events.append(event) + for ref in rdeleteds: + event = { + 'ref': ref, + 'branch_deleted': True, + 'oldrev': base_refs[ref], + 'newrev': EMPTY_GIT_REF + } + partial_events.append(event) + for ref, sha in updateds.items(): + event = { + 'ref': ref, + 'branch_updated': True, + 'oldrev': base_refs[ref], + 'newrev': sha + } + partial_events.append(event) + events = [] + for pevent in partial_events: + event = GitTriggerEvent() + event.type = 'ref-updated' + event.project_hostname = self.git_connection.canonical_hostname + event.project_name = project + for attr in ('ref', 'oldrev', 'newrev', 'branch_created', + 'branch_deleted', 'branch_updated'): + if attr in pevent: + setattr(event, attr, pevent[attr]) + events.append(event) + return events + + def _run(self): + self.log.debug("Walk through projects refs for connection: %s" % + self.git_connection.connection_name) + try: + for project in self.git_connection.projects: + refs = self.git_connection.lsRemote(project) + self.log.debug("Read refs %s for project %s" % (refs, project)) + if not self.projects_refs.get(project): + # State for this project does not exist yet so add it. + # No event will be triggered in this loop as + # projects_refs['project'] and refs are equal + self.projects_refs[project] = refs + events = self.compareRefs(project, refs) + self.projects_refs[project] = refs + # Send events to the scheduler + for event in events: + self.log.debug("Handling event: %s" % event) + # Force changes cache update before passing + # the event to the scheduler + self.git_connection.getChange(event) + self.git_connection.logEvent(event) + # Pass the event to the scheduler + self.git_connection.sched.addEvent(event) + except Exception as e: + self.log.debug("Unexpected issue in _run loop: %s" % str(e)) + + def run(self): + while not self._stopped: + if not self.git_connection.w_pause: + self._run() + # Polling wait delay + else: + self.log.debug("Watcher is on pause") + time.sleep(self.poll_delay) + + def stop(self): + self._stopped = True class GitConnection(BaseConnection): @@ -32,6 +139,8 @@ class GitConnection(BaseConnection): raise Exception('baseurl is required for git connections in ' '%s' % self.connection_name) self.baseurl = self.connection_config.get('baseurl') + self.poll_timeout = float( + self.connection_config.get('poll_delay', 3600 * 2)) self.canonical_hostname = self.connection_config.get( 'canonical_hostname') if not self.canonical_hostname: @@ -40,7 +149,10 @@ class GitConnection(BaseConnection): self.canonical_hostname = r.hostname else: self.canonical_hostname = 'localhost' + self.w_pause = False self.projects = {} + self.projects_refs = {} + self._change_cache = {} def getProject(self, name): return self.projects.get(name) @@ -48,15 +160,97 @@ class GitConnection(BaseConnection): def addProject(self, project): self.projects[project.name] = project + def getChangeFilesUpdated(self, project_name, branch, tosha): + job = self.sched.merger.getFilesChanges( + self.connection_name, project_name, branch, tosha) + self.log.debug("Waiting for fileschanges job %s" % job) + job.wait() + if not job.updated: + raise Exception("Fileschanges job %s failed" % job) + self.log.debug("Fileschanges job %s got changes on files %s" % + (job, job.files)) + return job.files + + def lsRemote(self, project): + refs = {} + client = git.cmd.Git() + output = client.ls_remote( + os.path.join(self.baseurl, project)) + for line in output.splitlines(): + sha, ref = line.split('\t') + if ref.startswith('refs/'): + refs[ref] = sha + return refs + + def maintainCache(self, relevant): + remove = {} + for branch, refschange in self._change_cache.items(): + for ref, change in refschange.items(): + if change not in relevant: + remove.setdefault(branch, []).append(ref) + for branch, refs in remove.items(): + for ref in refs: + del self._change_cache[branch][ref] + if not self._change_cache[branch]: + del self._change_cache[branch] + + def getChange(self, event, refresh=False): + if event.ref and event.ref.startswith('refs/heads/'): + branch = event.ref[len('refs/heads/'):] + change = self._change_cache.get(branch, {}).get(event.newrev) + if change: + return change + project = self.getProject(event.project_name) + change = Branch(project) + change.branch = branch + for attr in ('ref', 'oldrev', 'newrev'): + setattr(change, attr, getattr(event, attr)) + change.url = "" + change.files = self.getChangeFilesUpdated( + event.project_name, change.branch, event.oldrev) + self._change_cache.setdefault(branch, {})[event.newrev] = change + elif event.ref: + # catch-all ref (ie, not a branch or head) + project = self.getProject(event.project_name) + change = Ref(project) + for attr in ('ref', 'oldrev', 'newrev'): + setattr(change, attr, getattr(event, attr)) + change.url = "" + else: + self.log.warning("Unable to get change for %s" % (event,)) + change = None + return change + def getProjectBranches(self, project, tenant): - # TODO(jeblair): implement; this will need to handle local or - # remote git urls. - return ['master'] + refs = self.lsRemote(project.name) + branches = [ref[len('refs/heads/'):] for ref in + refs if ref.startswith('refs/heads/')] + return branches def getGitUrl(self, project): url = '%s/%s' % (self.baseurl, project.name) return url + def onLoad(self): + self.log.debug("Starting Git Watcher") + self._start_watcher_thread() + + def onStop(self): + self.log.debug("Stopping Git Watcher") + self._stop_watcher_thread() + + def _stop_watcher_thread(self): + if self.watcher_thread: + self.watcher_thread.stop() + self.watcher_thread.join() + + def _start_watcher_thread(self): + self.watcher_thread = GitWatcher( + self, + self.baseurl, + self.poll_timeout) + self.watcher_thread.start() + def getSchema(): git_connection = v.Any(str, v.Schema(dict)) diff --git a/zuul/driver/git/gitmodel.py b/zuul/driver/git/gitmodel.py new file mode 100644 index 000000000..5d12b36da --- /dev/null +++ b/zuul/driver/git/gitmodel.py @@ -0,0 +1,86 @@ +# Copyright 2017 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import re + +from zuul.model import TriggerEvent +from zuul.model import EventFilter + + +EMPTY_GIT_REF = '0' * 40 # git sha of all zeros, used during creates/deletes + + +class GitTriggerEvent(TriggerEvent): + """Incoming event from an external system.""" + + def __repr__(self): + ret = '<GitTriggerEvent %s %s' % (self.type, + self.project_name) + + if self.branch: + ret += " %s" % self.branch + ret += " oldrev:%s" % self.oldrev + ret += " newrev:%s" % self.newrev + ret += '>' + + return ret + + +class GitEventFilter(EventFilter): + def __init__(self, trigger, types=[], refs=[], + ignore_deletes=True): + + super().__init__(trigger) + + self._refs = refs + self.types = types + self.refs = [re.compile(x) for x in refs] + self.ignore_deletes = ignore_deletes + + def __repr__(self): + ret = '<GitEventFilter' + + if self.types: + ret += ' types: %s' % ', '.join(self.types) + if self._refs: + ret += ' refs: %s' % ', '.join(self._refs) + if self.ignore_deletes: + ret += ' ignore_deletes: %s' % self.ignore_deletes + ret += '>' + + return ret + + def matches(self, event, change): + # event types are ORed + matches_type = False + for etype in self.types: + if etype == event.type: + matches_type = True + if self.types and not matches_type: + return False + + # refs are ORed + matches_ref = False + if event.ref is not None: + for ref in self.refs: + if ref.match(event.ref): + matches_ref = True + if self.refs and not matches_ref: + return False + if self.ignore_deletes and event.newrev == EMPTY_GIT_REF: + # If the updated ref has an empty git sha (all 0s), + # then the ref is being deleted + return False + + return True diff --git a/zuul/driver/git/gitsource.py b/zuul/driver/git/gitsource.py index 8d85c082f..78ae04ee7 100644 --- a/zuul/driver/git/gitsource.py +++ b/zuul/driver/git/gitsource.py @@ -36,7 +36,7 @@ class GitSource(BaseSource): raise NotImplemented() def getChange(self, event, refresh=False): - raise NotImplemented() + return self.connection.getChange(event, refresh) def getProject(self, name): p = self.connection.getProject(name) diff --git a/zuul/driver/git/gittrigger.py b/zuul/driver/git/gittrigger.py new file mode 100644 index 000000000..28852307e --- /dev/null +++ b/zuul/driver/git/gittrigger.py @@ -0,0 +1,49 @@ +# Copyright 2017 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import voluptuous as v +from zuul.trigger import BaseTrigger +from zuul.driver.git.gitmodel import GitEventFilter +from zuul.driver.util import scalar_or_list, to_list + + +class GitTrigger(BaseTrigger): + name = 'git' + log = logging.getLogger("zuul.GitTrigger") + + def getEventFilters(self, trigger_conf): + efilters = [] + for trigger in to_list(trigger_conf): + f = GitEventFilter( + trigger=self, + types=to_list(trigger['event']), + refs=to_list(trigger.get('ref')), + ignore_deletes=trigger.get( + 'ignore-deletes', True) + ) + efilters.append(f) + + return efilters + + +def getSchema(): + git_trigger = { + v.Required('event'): + scalar_or_list(v.Any('ref-updated')), + 'ref': scalar_or_list(str), + 'ignore-deletes': bool, + } + + return git_trigger diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py index f987f4712..62dd45ce7 100644 --- a/zuul/driver/github/githubconnection.py +++ b/zuul/driver/github/githubconnection.py @@ -24,6 +24,7 @@ import re import cachecontrol from cachecontrol.cache import DictCache +from cachecontrol.heuristics import BaseHeuristic import iso8601 import jwt import requests @@ -137,7 +138,6 @@ class GithubEventConnector(threading.Thread): """Move events from GitHub into the scheduler""" log = logging.getLogger("zuul.GithubEventConnector") - delay = 10.0 def __init__(self, connection): super(GithubEventConnector, self).__init__() @@ -153,14 +153,6 @@ class GithubEventConnector(threading.Thread): ts, json_body, event_type = self.connection.getEvent() if self._stopped: return - # Github can produce inconsistent data immediately after an - # event, So ensure that we do not deliver the event to Zuul - # until at least a certain amount of time has passed. Note - # that if we receive several events in succession, we will - # only need to delay for the first event. In essence, Zuul - # should always be a constant number of seconds behind Github. - now = time.time() - time.sleep(max((ts + self.delay) - now, 0.0)) # If there's any installation mapping information in the body then # update the project mapping before any requests are made. @@ -431,9 +423,26 @@ class GithubConnection(BaseConnection): # NOTE(jamielennox): Better here would be to cache to memcache or file # or something external - but zuul already sucks at restarting so in # memory probably doesn't make this much worse. + + # NOTE(tobiash): Unlike documented cachecontrol doesn't priorize + # the etag caching but doesn't even re-request until max-age was + # elapsed. + # + # Thus we need to add a custom caching heuristic which simply drops + # the cache-control header containing max-age. This way we force + # cachecontrol to only rely on the etag headers. + # + # http://cachecontrol.readthedocs.io/en/latest/etags.html + # http://cachecontrol.readthedocs.io/en/latest/custom_heuristics.html + class NoAgeHeuristic(BaseHeuristic): + def update_headers(self, response): + if 'cache-control' in response.headers: + del response.headers['cache-control'] + self.cache_adapter = cachecontrol.CacheControlAdapter( DictCache(), - cache_etags=True) + cache_etags=True, + heuristic=NoAgeHeuristic()) # The regex is based on the connection host. We do not yet support # cross-connection dependency gathering diff --git a/zuul/executor/server.py b/zuul/executor/server.py index 7a93f896b..5a710a62d 100644 --- a/zuul/executor/server.py +++ b/zuul/executor/server.py @@ -931,6 +931,10 @@ class AnsibleJob(object): if username: host_vars['ansible_user'] = username + connection_type = node.get('connection_type') + if connection_type: + host_vars['ansible_connection'] = connection_type + host_keys = [] for key in node.get('host_keys'): if port != 22: @@ -1706,6 +1710,7 @@ class ExecutorServer(object): self.merger_worker.registerFunction("merger:merge") self.merger_worker.registerFunction("merger:cat") self.merger_worker.registerFunction("merger:refstate") + self.merger_worker.registerFunction("merger:fileschanges") def register_work(self): if self._running: @@ -1859,6 +1864,9 @@ class ExecutorServer(object): elif job.name == 'merger:refstate': self.log.debug("Got refstate job: %s" % job.unique) self.refstate(job) + elif job.name == 'merger:fileschanges': + self.log.debug("Got fileschanges job: %s" % job.unique) + self.fileschanges(job) else: self.log.error("Unable to handle job %s" % job.name) job.sendWorkFail() @@ -1970,6 +1978,19 @@ class ExecutorServer(object): files=files) job.sendWorkComplete(json.dumps(result)) + def fileschanges(self, job): + args = json.loads(job.arguments) + task = self.update(args['connection'], args['project']) + task.wait() + with self.merger_lock: + files = self.merger.getFilesChanges( + args['connection'], args['project'], + args['branch'], + args['tosha']) + result = dict(updated=True, + files=files) + job.sendWorkComplete(json.dumps(result)) + def refstate(self, job): args = json.loads(job.arguments) with self.merger_lock: diff --git a/zuul/merger/client.py b/zuul/merger/client.py index 2614e5887..c89a6fba8 100644 --- a/zuul/merger/client.py +++ b/zuul/merger/client.py @@ -131,6 +131,15 @@ class MergeClient(object): job = self.submitJob('merger:cat', data, None, precedence) return job + def getFilesChanges(self, connection_name, project_name, branch, + tosha=None, precedence=zuul.model.PRECEDENCE_HIGH): + data = dict(connection=connection_name, + project=project_name, + branch=branch, + tosha=tosha) + job = self.submitJob('merger:fileschanges', data, None, precedence) + return job + def onBuildCompleted(self, job): data = getJobData(job) merged = data.get('merged', False) diff --git a/zuul/merger/merger.py b/zuul/merger/merger.py index 06ec4b2b9..bd4ca58ee 100644 --- a/zuul/merger/merger.py +++ b/zuul/merger/merger.py @@ -314,6 +314,18 @@ class Repo(object): 'utf-8') return ret + def getFilesChanges(self, branch, tosha=None): + repo = self.createRepoObject() + files = set() + head = repo.heads[branch].commit + files.update(set(head.stats.files.keys())) + if tosha: + for cmt in head.iter_parents(): + if cmt.hexsha == tosha: + break + files.update(set(cmt.stats.files.keys())) + return list(files) + def deleteRemote(self, remote): repo = self.createRepoObject() repo.delete_remote(repo.remotes[remote]) @@ -581,3 +593,8 @@ class Merger(object): def getFiles(self, connection_name, project_name, branch, files, dirs=[]): repo = self.getRepo(connection_name, project_name) return repo.getFiles(files, dirs, branch=branch) + + def getFilesChanges(self, connection_name, project_name, branch, + tosha=None): + repo = self.getRepo(connection_name, project_name) + return repo.getFilesChanges(branch, tosha) diff --git a/zuul/merger/server.py b/zuul/merger/server.py index 576d41ed5..aa04fc206 100644 --- a/zuul/merger/server.py +++ b/zuul/merger/server.py @@ -81,6 +81,7 @@ class MergeServer(object): self.worker.registerFunction("merger:merge") self.worker.registerFunction("merger:cat") self.worker.registerFunction("merger:refstate") + self.worker.registerFunction("merger:fileschanges") def stop(self): self.log.debug("Stopping") @@ -117,6 +118,9 @@ class MergeServer(object): elif job.name == 'merger:refstate': self.log.debug("Got refstate job: %s" % job.unique) self.refstate(job) + elif job.name == 'merger:fileschanges': + self.log.debug("Got fileschanges job: %s" % job.unique) + self.fileschanges(job) else: self.log.error("Unable to handle job %s" % job.name) job.sendWorkFail() @@ -158,3 +162,12 @@ class MergeServer(object): result = dict(updated=True, files=files) job.sendWorkComplete(json.dumps(result)) + + def fileschanges(self, job): + args = json.loads(job.arguments) + self.merger.updateRepo(args['connection'], args['project']) + files = self.merger.getFilesChanges( + args['connection'], args['project'], args['branch'], args['tosha']) + result = dict(updated=True, + files=files) + job.sendWorkComplete(json.dumps(result)) diff --git a/zuul/model.py b/zuul/model.py index 77770b793..cc2fea7e2 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -384,6 +384,7 @@ class Node(object): self.private_ipv4 = None self.public_ipv6 = None self.connection_port = 22 + self.connection_type = None self._keys = [] self.az = None self.provider = None @@ -2255,7 +2256,7 @@ class TenantProjectConfig(object): class ProjectConfig(object): - # Represents a project cofiguration + # Represents a project configuration def __init__(self, name, source_context=None): self.name = name # If this is a template, it will have a source_context, but @@ -2400,7 +2401,7 @@ class UnparsedTenantConfig(object): r.semaphores = copy.deepcopy(self.semaphores) return r - def extend(self, conf, tenant=None): + def extend(self, conf, tenant): if isinstance(conf, UnparsedTenantConfig): self.pragmas.extend(conf.pragmas) self.pipelines.extend(conf.pipelines) @@ -2408,16 +2409,14 @@ class UnparsedTenantConfig(object): self.project_templates.extend(conf.project_templates) for k, v in conf.projects.items(): name = k - # If we have the tenant add the projects to - # the according canonical name instead of the given project - # name. If it is not found, it's ok to add this to the given - # name. We also don't need to throw the + # Add the projects to the according canonical name instead of + # the given project name. If it is not found, it's ok to add + # this to the given name. We also don't need to throw the # ProjectNotFoundException here as semantic validation occurs # later where it will fail then. - if tenant is not None: - trusted, project = tenant.getProject(k) - if project is not None: - name = project.canonical_name + trusted, project = tenant.getProject(k) + if project is not None: + name = project.canonical_name self.projects.setdefault(name, []).extend(v) self.nodesets.extend(conf.nodesets) self.secrets.extend(conf.secrets) @@ -2434,7 +2433,12 @@ class UnparsedTenantConfig(object): raise ConfigItemMultipleKeysError() key, value = list(item.items())[0] if key == 'project': - name = value['name'] + name = value.get('name') + if not name: + # There is no name defined so implicitly add the name + # of the project where it is defined. + name = value['_source_context'].project.canonical_name + value['name'] = name self.projects.setdefault(name, []).append(value) elif key == 'job': self.jobs.append(value) @@ -2643,11 +2647,11 @@ class Layout(object): repr(variant), change) item.debug("Pipeline variant {variant} matched".format( variant=repr(variant)), indent=2) - else: - self.log.debug("Pipeline variant %s did not match %s", - repr(variant), change) - item.debug("Pipeline variant {variant} did not match".format( - variant=repr(variant)), indent=2) + else: + self.log.debug("Pipeline variant %s did not match %s", + repr(variant), change) + item.debug("Pipeline variant {variant} did not match". + format(variant=repr(variant)), indent=2) if not matched: # A change must match at least one project pipeline # job variant. |