diff options
44 files changed, 674 insertions, 225 deletions
diff --git a/.gitignore b/.gitignore index d6a7477e4..a2dd0a38b 100644 --- a/.gitignore +++ b/.gitignore @@ -7,9 +7,12 @@ .testrepository .tox .venv +.coverage AUTHORS build/* ChangeLog doc/build/* zuul/versioninfo dist/ +cover/ +htmlcov/ diff --git a/doc/source/zuul.rst b/doc/source/zuul.rst index a7dfb4482..f07a85906 100644 --- a/doc/source/zuul.rst +++ b/doc/source/zuul.rst @@ -52,6 +52,16 @@ server just set **server** to 127.0.0.1. Port on which the Gearman server is listening. ``port=4730`` (optional) +**ssl_ca** + Optional: An openssl file containing a set of concatenated “certification authority” certificates + in PEM formet. + +**ssl_cert** + Optional: An openssl file containing the client public certificate in PEM format. + +**ssl_key** + Optional: An openssl file containing the client private key in PEM format. + gearman_server """""""""""""" @@ -70,6 +80,16 @@ than connecting to an external one. Path to log config file for internal Gearman server. ``log_config=/etc/zuul/gearman-logging.yaml`` +**ssl_ca** + Optional: An openssl file containing a set of concatenated “certification authority” certificates + in PEM formet. + +**ssl_cert** + Optional: An openssl file containing the server public certificate in PEM format. + +**ssl_key** + Optional: An openssl file containing the server private key in PEM format. + webapp """""" @@ -128,11 +148,6 @@ layout.yaml to be useful. optional value and ``1`` is used by default. ``status_expiry=1`` -**job_name_in_report** - Boolean value (``true`` or ``false``) that indicates whether the - job name should be included in the report (normally only the URL - is included). Defaults to ``false``. Used by zuul-server only. - ``job_name_in_report=true`` merger """""" diff --git a/etc/zuul.conf-sample b/etc/zuul.conf-sample index 1065cecf4..1aee1fa18 100644 --- a/etc/zuul.conf-sample +++ b/etc/zuul.conf-sample @@ -1,8 +1,14 @@ [gearman] server=127.0.0.1 +;ssl_ca=/path/to/ca.pem +;ssl_cert=/path/to/client.pem +;ssl_key=/path/to/client.key [gearman_server] start=true +;ssl_ca=/path/to/ca.pem +;ssl_cert=/path/to/server.pem +;ssl_key=/path/to/server.key [zuul] layout_config=/etc/zuul/layout.yaml diff --git a/tests/base.py b/tests/base.py index 76f7e03b5..31d6f0d7d 100755 --- a/tests/base.py +++ b/tests/base.py @@ -776,21 +776,6 @@ class FakeGithubPullRequest(object): repo = self._getRepo() return repo.references[self._getPRReference()].commit.hexsha - def setStatus(self, sha, state, url, description, context, user='zuul'): - # Since we're bypassing github API, which would require a user, we - # hard set the user as 'zuul' here. - # insert the status at the top of the list, to simulate that it - # is the most recent set status - self.statuses[sha].insert(0, ({ - 'state': state, - 'url': url, - 'description': description, - 'context': context, - 'creator': { - 'login': user - } - })) - def addReview(self, user, state, granted_on=None): gh_time_format = '%Y-%m-%dT%H:%M:%SZ' # convert the timestamp to a str format that would be returned @@ -882,6 +867,7 @@ class FakeGithubConnection(githubconnection.GithubConnection): self.connection_name = connection_name self.pr_number = 0 self.pull_requests = [] + self.statuses = {} self.upstream_root = upstream_root self.merge_failure = False self.merge_not_allowed_count = 0 @@ -949,7 +935,8 @@ class FakeGithubConnection(githubconnection.GithubConnection): 'repo': { 'full_name': pr.project } - } + }, + 'files': pr.files } return data @@ -960,10 +947,6 @@ class FakeGithubConnection(githubconnection.GithubConnection): pr = prs[0] return self.getPull(pr.project, pr.number) - def getPullFileNames(self, project, number): - pr = self.pull_requests[number - 1] - return pr.files - def _getPullReviews(self, owner, project, number): pr = self.pull_requests[number - 1] return pr.reviews @@ -1014,25 +997,24 @@ class FakeGithubConnection(githubconnection.GithubConnection): pull_request.merge_message = commit_message def getCommitStatuses(self, project, sha): - owner, proj = project.split('/') - for pr in self.pull_requests: - pr_owner, pr_project = pr.project.split('/') - # This is somewhat risky, if the same commit exists in multiple - # PRs, we might grab the wrong one that doesn't have a status - # that is expected to be there. Maybe re-work this so that there - # is a global registry of commit statuses like with github. - if (pr_owner == owner and pr_project == proj and - sha in pr.statuses): - return pr.statuses[sha] - - def setCommitStatus(self, project, sha, state, - url='', description='', context=''): - owner, proj = project.split('/') - for pr in self.pull_requests: - pr_owner, pr_project = pr.project.split('/') - if (pr_owner == owner and pr_project == proj and - pr.head_sha == sha): - pr.setStatus(sha, state, url, description, context) + return self.statuses.get(project, {}).get(sha, []) + + def setCommitStatus(self, project, sha, state, url='', description='', + context='default', user='zuul'): + # always insert a status to the front of the list, to represent + # the last status provided for a commit. + # Since we're bypassing github API, which would require a user, we + # default the user as 'zuul' here. + self.statuses.setdefault(project, {}).setdefault(sha, []) + self.statuses[project][sha].insert(0, { + 'state': state, + 'url': url, + 'description': description, + 'context': context, + 'creator': { + 'login': user + } + }) def labelPull(self, project, pr_number, label): pull_request = self.pull_requests[pr_number - 1] @@ -1374,9 +1356,20 @@ class FakeGearmanServer(gear.Server): """ - def __init__(self): + def __init__(self, use_ssl=False): self.hold_jobs_in_queue = False - super(FakeGearmanServer, self).__init__(0) + if use_ssl: + ssl_ca = os.path.join(FIXTURE_DIR, 'gearman/root-ca.pem') + ssl_cert = os.path.join(FIXTURE_DIR, 'gearman/server.pem') + ssl_key = os.path.join(FIXTURE_DIR, 'gearman/server.key') + else: + ssl_ca = None + ssl_cert = None + ssl_key = None + + super(FakeGearmanServer, self).__init__(0, ssl_key=ssl_key, + ssl_cert=ssl_cert, + ssl_ca=ssl_ca) def getJobForConnection(self, connection, peek=False): for queue in [self.high_queue, self.normal_queue, self.low_queue]: @@ -1830,6 +1823,7 @@ class ZuulTestCase(BaseTestCase): config_file = 'zuul.conf' run_ansible = False create_project_keys = False + use_ssl = False def _startMerger(self): self.merge_server = zuul.merger.server.MergeServer(self.config, @@ -1887,11 +1881,22 @@ class ZuulTestCase(BaseTestCase): reload_module(statsd) reload_module(zuul.scheduler) - self.gearman_server = FakeGearmanServer() + self.gearman_server = FakeGearmanServer(self.use_ssl) self.config.set('gearman', 'port', str(self.gearman_server.port)) self.log.info("Gearman server on port %s" % (self.gearman_server.port,)) + if self.use_ssl: + self.log.info('SSL enabled for gearman') + self.config.set( + 'gearman', 'ssl_ca', + os.path.join(FIXTURE_DIR, 'gearman/root-ca.pem')) + self.config.set( + 'gearman', 'ssl_cert', + os.path.join(FIXTURE_DIR, 'gearman/client.pem')) + self.config.set( + 'gearman', 'ssl_key', + os.path.join(FIXTURE_DIR, 'gearman/client.key')) gerritsource.GerritSource.replication_timeout = 1.5 gerritsource.GerritSource.replication_retry_interval = 0.5 @@ -2701,6 +2706,11 @@ class AnsibleZuulTestCase(ZuulTestCase): run_ansible = True +class SSLZuulTestCase(ZuulTestCase): + """ZuulTestCase but using SSL when possible""" + use_ssl = True + + class ZuulDBTestCase(ZuulTestCase): def setup_config(self): super(ZuulDBTestCase, self).setup_config() diff --git a/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml b/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml index 1f8fdf3e1..ce392a401 100644 --- a/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml +++ b/tests/fixtures/config/ansible/git/common-config/playbooks/check-vars.yaml @@ -3,9 +3,9 @@ - name: Assert nodepool variables are valid. assert: that: - - nodepool_az == 'test-az' - - nodepool_region == 'test-region' - - nodepool_provider == 'test-provider' + - nodepool.az == 'test-az' + - nodepool.region == 'test-region' + - nodepool.provider == 'test-provider' - name: Assert zuul-executor variables are valid. assert: diff --git a/tests/fixtures/gearman/README.rst b/tests/fixtures/gearman/README.rst new file mode 100644 index 000000000..a3921ea2e --- /dev/null +++ b/tests/fixtures/gearman/README.rst @@ -0,0 +1,21 @@ +Steps used to create our certs + +# Generate CA cert +$ openssl req -new -newkey rsa:2048 -nodes -keyout root-ca.key -x509 -days 3650 -out root-ca.pem -subj "/C=US/ST=Texas/L=Austin/O=OpenStack Foundation/CN=gearman-ca" + +# Generate server keys +$ CLIENT='server' +$ openssl req -new -newkey rsa:2048 -nodes -keyout $CLIENT.key -out $CLIENT.csr -subj "/C=US/ST=Texas/L=Austin/O=OpenStack Foundation/CN=nodepool-$CLIENT" +$ openssl x509 -req -days 3650 -in $CLIENT.csr -out $CLIENT.pem -CA root-ca.pem -CAkey root-ca.key -CAcreateserial + + +# Generate client keys +$ CLIENT='client' +$ openssl req -new -newkey rsa:2048 -nodes -keyout $CLIENT.key -out $CLIENT.csr -subj "/C=US/ST=Texas/L=Austin/O=OpenStack Foundation/CN=gearman-$CLIENT" +$ openssl x509 -req -days 3650 -in $CLIENT.csr -out $CLIENT.pem -CA root-ca.pem -CAkey root-ca.key -CAcreateserial + + +# Test with geard +# You'll need 2 terminal windows +geard --ssl-ca root-ca.pem --ssl-cert server.pem --ssl-key server.key -d +openssl s_client -connect localhost:4730 -key client.key -cert client.pem -CAfile root-ca.pem diff --git a/tests/fixtures/gearman/client.csr b/tests/fixtures/gearman/client.csr new file mode 100644 index 000000000..fadb857cc --- /dev/null +++ b/tests/fixtures/gearman/client.csr @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICqzCCAZMCAQAwZjELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBVRleGFzMQ8wDQYD +VQQHDAZBdXN0aW4xHTAbBgNVBAoMFE9wZW5TdGFjayBGb3VuZGF0aW9uMRcwFQYD +VQQDDA5nZWFybWFuLWNsaWVudDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC +ggEBALe+ByAkac9cYjeV8lcWXhDxdFqb7Om+6cWSJ/hpM4Z5QyGJ9XHDWyhrmt5W +X2jvE/bAxEWXxWj3v8xR5HbjS6XHBHouQxz+FSDcG1GZjOLK5fwnO5tKG5eLdrAN +WgOqJynJAsA0IuxURI4LiBUnzdi/10VeygwSIHOBLVWfrTZNKiE8siiQIaUAerLT +T8BEUEAUI38UhS4OT83QGUbcCPOkioE5/Q8VVpvlu3eIIEkkacs5293EfUvQRVSG ++GYjSHfFBV7ECX7gu/4nosa/bLfQw7F9O1C2E6QEoUqVNEtURXT0ALlGkUylq6H9 +ctVjoJS9iW8ToMtajW2PZVI/d6MCAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4IBAQBc +v3/Z9Exc7pnbwyL31ZGv+gF0Z1l9CaSobdI3JAMzKxYGK9SxYOAwcwuUL0+zAJEE +VPAaWM0p6eVF6j0d97Q79XsHvIKvyVYFxZ9rYSI+cvAIxhws1b4YtRoPBlY3AajV +u2CQDVos/8JB28X3DpM4MJRua2tnTfAGLCkEp1psAoND+rr5eL7j+naUcPvNMv7Z +WnTbIJYmP/6N+8gGGtAiiibXP3/Z92kFUZZxKNt3YSHfhkGY57/p+d8i3/8B+qeA +/YfohA4hNLPydcw32kzo7865+h3SMdbX7VF8xB9grbZXvkT26rtrFJxWLOf5Vmzi +aGPrVyPIeyVJvW3EeJQ9 +-----END CERTIFICATE REQUEST----- diff --git a/tests/fixtures/gearman/client.key b/tests/fixtures/gearman/client.key new file mode 100644 index 000000000..656cfc70d --- /dev/null +++ b/tests/fixtures/gearman/client.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC3vgcgJGnPXGI3 +lfJXFl4Q8XRam+zpvunFkif4aTOGeUMhifVxw1soa5reVl9o7xP2wMRFl8Vo97/M +UeR240ulxwR6LkMc/hUg3BtRmYziyuX8JzubShuXi3awDVoDqicpyQLANCLsVESO +C4gVJ83Yv9dFXsoMEiBzgS1Vn602TSohPLIokCGlAHqy00/ARFBAFCN/FIUuDk/N +0BlG3AjzpIqBOf0PFVab5bt3iCBJJGnLOdvdxH1L0EVUhvhmI0h3xQVexAl+4Lv+ +J6LGv2y30MOxfTtQthOkBKFKlTRLVEV09AC5RpFMpauh/XLVY6CUvYlvE6DLWo1t +j2VSP3ejAgMBAAECggEAF5cAFzJVm1fDDFvl9yRaA1bcl115dzEZllIDa7Ml+FfN +NJsftfFc3L2j7nOsYC6Bo6ZwDHdF0worx7Gj4VehOLFqc71IxIoicEuR/lH2co+W +I19uGavUCwrOvB+atOm9iXHTNpX6/dh7zLjSSdUIapGGs9NNoWsaW3n0NhAADv51 +SQYemDgG9/vLGPoouUGTBkMNCuI+uHP1C+nRSs/kikebjVqYoDNPm1/ADpccde9p +mntezm9v/xDXzVFD2qQTTve1mDpoy6YLZdY0mT6qUNElwZ+yZHXkBox1tpJ69Uw+ +pGanSMOy/Gj3W5RlX4qTSLLRcSePV8x65MzRwFoxgQKBgQDstP1/sfnS3JBWSW6V +YAN3alXeyb0bH0uA+nfXRzy9GnwlFTAszzwnbKL9xK+hPjJRkDBf8XDyXKQ3FMyi +OVf+H2IkhwErQL01qG4k8E3Hk9wQMvjdO00SaEiLD2uMxX9lRCs9vVlvtmSbGvTH +/RXBFnqYDHeMJxnWZ8Y34chtoQKBgQDGt+cYtoXH79imuyOQ1SORtIQtSGEKcIcg +20o5tCGJfCxLtrKs7n4Yph9IPvMtiA8idPACWU2Q8XV580RABzC7Am8ICGGJSwN8 +PLoWOadEpYYeFOV8Mzfxs/YhdQat6zvGy8sF0O+DER0b1ELfbA1I+FNOuz0y53AJ +MXxOUvQ2wwKBgAFWHEBGTvTDzgTOsVMikaJw9T8mwGyQxqpZv6d1fYBLz/udnQID +wYEvedQY8izk3v/a4osIH+0eXMb61RTtYfPLVZCDOpx15xuQcd6/hJDl4s4sm38U +QKEj+ZTfZ2oKC2gU9HGKyiB5VSQTCOLAKQlICTUmjN47skelmlbibXFBAoGBAIHn +UoELQGU1W3GTQGq7imcDlKxtdlJ2wT8vW1RhdtMDg4lzQ1ZdCb1fS2/VBu8q1In3 +27YNXvFzhxJTfrhEewylSKP9ppUznnGm2RcSVVBAzG35xxLsAJRWyn2QnO8wqYEJ +VAzXSttpYpgAqD6Zyg17mCoNqLIQLWM1IEerXs41AoGAGdswRmzQ2oHF0f01yZaq +rxGtLOuTyHzmwi8vA4qZj/9Baht9ihVJiqxTAg/CuA3sTM7DxAJ6P5h6mHsVM6bh +tPVruBdPIOg4XchcXory1Pa8wSHsPkEnj2NnrZRhvcv86vFxDkhu601nv+AGHj1D +szjDKeH4IP8fjbf/utRxo3w= +-----END PRIVATE KEY----- diff --git a/tests/fixtures/gearman/client.pem b/tests/fixtures/gearman/client.pem new file mode 100644 index 000000000..aac9d8dbc --- /dev/null +++ b/tests/fixtures/gearman/client.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDRDCCAiwCCQDnKP1tRJr+2DANBgkqhkiG9w0BAQsFADBiMQswCQYDVQQGEwJV +UzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEdMBsGA1UECgwUT3Bl +blN0YWNrIEZvdW5kYXRpb24xEzARBgNVBAMMCmdlYXJtYW4tY2EwHhcNMTcwNjE0 +MTQwNzAwWhcNMjcwNjEyMTQwNzAwWjBmMQswCQYDVQQGEwJVUzEOMAwGA1UECAwF +VGV4YXMxDzANBgNVBAcMBkF1c3RpbjEdMBsGA1UECgwUT3BlblN0YWNrIEZvdW5k +YXRpb24xFzAVBgNVBAMMDmdlYXJtYW4tY2xpZW50MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEAt74HICRpz1xiN5XyVxZeEPF0Wpvs6b7pxZIn+GkzhnlD +IYn1ccNbKGua3lZfaO8T9sDERZfFaPe/zFHkduNLpccEei5DHP4VINwbUZmM4srl +/Cc7m0obl4t2sA1aA6onKckCwDQi7FREjguIFSfN2L/XRV7KDBIgc4EtVZ+tNk0q +ITyyKJAhpQB6stNPwERQQBQjfxSFLg5PzdAZRtwI86SKgTn9DxVWm+W7d4ggSSRp +yznb3cR9S9BFVIb4ZiNId8UFXsQJfuC7/ieixr9st9DDsX07ULYTpAShSpU0S1RF +dPQAuUaRTKWrof1y1WOglL2JbxOgy1qNbY9lUj93owIDAQABMA0GCSqGSIb3DQEB +CwUAA4IBAQBSYRP7DDGRBs1wwudH2HzaDRNZrhECUq6n45FY3YHkDU5xxi6CA3wD +EA+fvvB95BvqNNCS4UxQMW3k7cgJQrUVBKXj9m5HqE/GVZuI15+bR9i7vc5USoen +nfbVhDAvZcrzPhmj/pfnXKwgeE7PhG55mrJvJgSmxmK2wTcRRIQ6dfoj3OIJJHEY +kW3oK8I+9r5Tufxbg+CIpZVIuENbRDNGhTPCtzDu3q6DHAEOBKHmwc64W/2c+2QV +CpfPdutVF2reb6CJuGikM8WMh47mksNIyCW832bUvUCDgW4/tqanPqww4lTdU44b +W8gkkWcUmOa6MVCXIzCy7tEbkEDJC2NE +-----END CERTIFICATE----- diff --git a/tests/fixtures/gearman/root-ca.key b/tests/fixtures/gearman/root-ca.key new file mode 100644 index 000000000..3db94c331 --- /dev/null +++ b/tests/fixtures/gearman/root-ca.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDAglCCv7aQxXZg +8wuLq0QuIQbZbK1Y0aFwMaEpOeVyZR/C42nws3hH0BivJZnr5w57fdT2OXFqkAyl +Pw+sF8PcDlSi2wF33rnswz8qYszX5WUgvGnOtcJx8YJhqBqNCLb0wnneJqNQpXPs +CmcsEeBMsCVN9Q1cRMgdjyMBpRfcq7WH5NN+o/n4zClHYZwa3wOyH2ipekl4XTEf +Kz9aq88L3YE/N4dyUWH0UpS+lBem+D0GAarV2IXWqXeMrWce930mBONMhBrgw0X5 +QFrDa0KQn2QRcg9tqlEE9SlAbub/yHUsq7/7q7l6SWl7JBigj4jGw15w98WzSDkJ +a0we1jexAgMBAAECggEAX/HS3IdeHyM7D7CyZWbzcSYmusBuWOEJ29fwYZKoZ248 ++S3MhBl+bhQp6UkNQMSEtEmPlTQl8Z1foBAg6H1jsU43In+SaMLJ2VWqKp7ZRxTe +ZQVimpJ+GbnraG6W5Qmd3bj7chvBs5TyhIbeytkR+EamIQdsJDtnnUvUf6JflSvl +gUZxOvfB7UZQZ2PkMQFleZxlEAvgyk8e4k7AnY2IoTyvw1DIUdP7+7hPInBpWaUn +jJPZzyWyrMDLB+BB7JcdqmO2N5bHudE4iEJwphmdIcHvOFhm/LHfJdZg6+X8lUCP +lIfzp6Uk25nF5/dsoZQcrdBznhW4NfJsIviui+SSAQKBgQDrVI4pW/1fU4AYoOki +jB+RaUaBXkRRV6MYJ/ZUPjAONDtFhDdBDEwunVUpTD8sgVKFnaMzNa/+Jcmy4jsE +Ggj9ZupH05g9Y8w7dYFcfT6VeiPahyml8k/DWWe0qQk0+scH8vuiWcrqIjWRg4WD +8CXJqSkgrCHFCjuJOvxM24d1UQKBgQDRaupcR/c9AGpUHmhFGexwOyXtIR2ObaVf +lEZ9rhrpCRAl5RW0tUmd1vHMNDTdRidfYLFe29h6YQ1afgNcV8JdB51VfurJ+cOF +jbc6FijDag31snIdBnDuV29mazejRm7PSfJjoBnBDNzh3kMed22DsQDlHQmudknH +wUqUWnWEYQKBgG3bYSoJmXRgxJG6vFq2Ux5MqO9HlFjssmRac3HMPh7DX1AKcsjY +9s9j/xdyUqNyE5Xwivki/O+FsGzjk21MwhmZa5DwREeUSQkQx7zncsnQ5N/k7Rpc +zcOB/xmlN3kWAMfDNJkLleBK6/rsDO4Us286msp30KPtLPHZKWKvsMKhAoGAaiER +5nR+Qsb8G+dRFnv9zB7dqKAYt36vyZF+a+EZODJkoZ/IcU1SopA0+DUY+W69M2Pw +X89wlQysVMj58Ql0serS/GoWmQdf5EYermxeejI8IuEtXbJO9ysOhMwfZTqjm5+x +HHYdty1Kn5khUMZblNrWRkaCCo1d9MLrheWWGuECgYEAy5kdeVE8lLliFL39Xrzl +OCJ1rEIAhXrqr6E3PrMlUiQ75dAOiLEl3GGG7UkSHL/0dZv50RRee+4hxI63P2z0 +xPeH2nvrFzknmabOWxtOpw+H0qGOYto9VcvseFPNKTV2O5wxdfaYgLEOXt8ipaLD +OVvm6yN1bP1Gxi6vdVppKwk= +-----END PRIVATE KEY----- diff --git a/tests/fixtures/gearman/root-ca.pem b/tests/fixtures/gearman/root-ca.pem new file mode 100644 index 000000000..defedd6c4 --- /dev/null +++ b/tests/fixtures/gearman/root-ca.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDlzCCAn+gAwIBAgIJAPmWfgTknq1hMA0GCSqGSIb3DQEBCwUAMGIxCzAJBgNV +BAYTAlVTMQ4wDAYDVQQIDAVUZXhhczEPMA0GA1UEBwwGQXVzdGluMR0wGwYDVQQK +DBRPcGVuU3RhY2sgRm91bmRhdGlvbjETMBEGA1UEAwwKZ2Vhcm1hbi1jYTAeFw0x +NzA2MTQxNDA1NDNaFw0yNzA2MTIxNDA1NDNaMGIxCzAJBgNVBAYTAlVTMQ4wDAYD +VQQIDAVUZXhhczEPMA0GA1UEBwwGQXVzdGluMR0wGwYDVQQKDBRPcGVuU3RhY2sg +Rm91bmRhdGlvbjETMBEGA1UEAwwKZ2Vhcm1hbi1jYTCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAMCCUIK/tpDFdmDzC4urRC4hBtlsrVjRoXAxoSk55XJl +H8LjafCzeEfQGK8lmevnDnt91PY5cWqQDKU/D6wXw9wOVKLbAXfeuezDPypizNfl +ZSC8ac61wnHxgmGoGo0ItvTCed4mo1Clc+wKZywR4EywJU31DVxEyB2PIwGlF9yr +tYfk036j+fjMKUdhnBrfA7IfaKl6SXhdMR8rP1qrzwvdgT83h3JRYfRSlL6UF6b4 +PQYBqtXYhdapd4ytZx73fSYE40yEGuDDRflAWsNrQpCfZBFyD22qUQT1KUBu5v/I +dSyrv/uruXpJaXskGKCPiMbDXnD3xbNIOQlrTB7WN7ECAwEAAaNQME4wHQYDVR0O +BBYEFDIaceZ/LY42aNSV0hisgSEcnjlMMB8GA1UdIwQYMBaAFDIaceZ/LY42aNSV +0hisgSEcnjlMMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAKN60Jnx +NPSkDlqrKtcojX3+oVPC5MQctysZXmjjkGzHSAVKeonQ+gN/glfRc0qq/PuzvHej +a2Mk9CirL2VzBgp1d/sGtOijqI0Otn706SBuQl1PEAzcmTyQt7TuhUnVcV22xBwy +ONIuXVT5eh8MhUdrlqZKXX9U49sjmHCheJFFVqFmy0twlqf9YikC0CNxiWa/jDhj +bxi73kxgZTb2RPjwYUWbESfyNCq0H+N2BmSz7Fgc2Ah/wvhXGdx1udaDVgzDqFIR +lMGswkzmd76JpJdN0Rce7lmRoE8E6BqDShvoEGiGo3IbuOUwn5JRKFMUPhN6mv7N +c49ykHzcCgc1wdY= +-----END CERTIFICATE----- diff --git a/tests/fixtures/gearman/root-ca.srl b/tests/fixtures/gearman/root-ca.srl new file mode 100644 index 000000000..0ce584a80 --- /dev/null +++ b/tests/fixtures/gearman/root-ca.srl @@ -0,0 +1 @@ +E728FD6D449AFED8 diff --git a/tests/fixtures/gearman/server.csr b/tests/fixtures/gearman/server.csr new file mode 100644 index 000000000..bbb03d208 --- /dev/null +++ b/tests/fixtures/gearman/server.csr @@ -0,0 +1,17 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICrDCCAZQCAQAwZzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBVRleGFzMQ8wDQYD +VQQHDAZBdXN0aW4xHTAbBgNVBAoMFE9wZW5TdGFjayBGb3VuZGF0aW9uMRgwFgYD +VQQDDA9ub2RlcG9vbC1zZXJ2ZXIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQCzoKkaauTNBRry1Y5YCNG38IrxW0AH5TP5XdTF/q+Qu1p9onRsACiSZX8Y +YAo/y6jVbZ3WKihVfVIQw9xrPTCoA0AwMtI8fiK70YwSuGg6gqBBCr8NXOaYsYFJ +k2Vk+8utlNSmLYlcSTKZR0HbhWNmjH9lj5WngL0XPSbcoogtvet92111qGfBZrg+ +86B3XJh2/6PCru9YmufqlooFog7Q4Qo6Bnz7Dh+h2QjtDmGSFz0dQ9PqP8Jgh3LS +fWRk5TrjGsthKszRTZCQDSXc1XcwAqfO21eufP9oTpfc0zTdAOC1tspdP/632q6B +0Gf8sSEnMpKmwuGUH3z2ZCY6DSE1AgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEA +NPZ0BNt9vjNM9cNHCgL8rYdB9UnsnkcQ5R/XRV1W+tQlj9QjpvcGH5c3PJ6Ol1Qd +x8o19aomLb/IMz8bnRmzLxWggKQHxLwU3UKjHBiV1aqI/ieka22IqKYkjeYUAyxC +ZLytynIZRVt0MB/lo7Z2bjctGHSiZ9tkTsgjawE3hotnZ3BOEOkV42099bLLGdcz +Jq433DsbwThKC0WijeHR4FZEj3H7Gj07PNAlfyM0KeyrZodtcIwvgA4NyBB8mPoV +dARn5C8hOtDCWzRPba46h9mTzF8D87pdvmZce6k/bBGJfY+YvOpwBXsO3xhCDxqP +p9gAs6m+qbxsrwvRRrtn6Q== +-----END CERTIFICATE REQUEST----- diff --git a/tests/fixtures/gearman/server.key b/tests/fixtures/gearman/server.key new file mode 100644 index 000000000..c1707b079 --- /dev/null +++ b/tests/fixtures/gearman/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCzoKkaauTNBRry +1Y5YCNG38IrxW0AH5TP5XdTF/q+Qu1p9onRsACiSZX8YYAo/y6jVbZ3WKihVfVIQ +w9xrPTCoA0AwMtI8fiK70YwSuGg6gqBBCr8NXOaYsYFJk2Vk+8utlNSmLYlcSTKZ +R0HbhWNmjH9lj5WngL0XPSbcoogtvet92111qGfBZrg+86B3XJh2/6PCru9Ymufq +looFog7Q4Qo6Bnz7Dh+h2QjtDmGSFz0dQ9PqP8Jgh3LSfWRk5TrjGsthKszRTZCQ +DSXc1XcwAqfO21eufP9oTpfc0zTdAOC1tspdP/632q6B0Gf8sSEnMpKmwuGUH3z2 +ZCY6DSE1AgMBAAECggEAaG06YhVKtrYFGK92dU+LPHgnDnGSJATn1kzqacDKqEWD +Mg7DyBW/gHxpCu6qhrQLjyiO3fbcQ/b7Qqva9K06IDLjmiGxf2GFJ9OGr0ttrLZM +HAP3VflwRczL8M4z4CVSH7OqfIF0naYgOGPosYo2Y2PCnHSA+EQrqdrvQM1shcot +8lW368VqlAm8ONgh8z4ZLSDswECgJzWleOSsTBIT0qJ6fXIwnN7akM8Bdyy/dPDD +PnPvAu1N9KgwrzxKY9WthJ1alKiFQm4Po/TZZApALOtR8zCN4EmDG9izKdfU5FIL +ZWpVDp0US7a8rbj2e0kf0loRg2bsR2eoJPL7JjJycQKBgQDiHjKnwximtfjqLTtg +ZOHIL4tTlmeTLNq7ZW69BuJSfI7FTa20piHjny+g3mTvxnCQa/BTSpb6VRHPFcYV +dVQzdAX6ZMvBZ3YMp9FkY+S9RrjEyimNU9kvJJQBnC1ujen3YuXj6ENFzcmGkvzR +LZFx3dmFEzfDxOOqzdFTHscGuwKBgQDLXaVBH54yq1fDrXDLG/eEtQsNNyCujIV4 +gp1Z54L34htoDS98dx0L0qZGBEys8I0dGJd9kUBVNu53zDeiJSGW4tHYXQaUpxJH +0wZDHo59mw3aGvVZ5YP+4uukuNHcX6cUYi2HAv0vwet46L3Kb/utDyyStp1QZw9s +eucOLGkQzwKBgG3j0yZo0FAk28WjGdos7PWG9aU30TpbcCnmj7zZ3Z/M3O3SZHsI +yit/L3x02IUW4Zmue2tfMqSSN0d3A39mN/eRiV45IjTp/RsFa+PoEEBUYHNy9GK0 +vzYEBtIJfqLd4TjTHXp3ZEpGSoxWXvuhs6+s64ua3V0NEL/vqq1EpeEFAoGAHa/i +8tnJvz3SBwenoo7HmEDRhzFX/QMYbNosXDZ2oPcJ5yudlf7RZ6ttiGUSSGCpSOkR +HEx65rWpJCXUrT/cYmlkFsCluEeXXJLKpDuus1lSMVekH2Zo2WmI2rf8Mr5n5ora +eI4QJcuaM0FOi2HDjKTdbeFon5cb4ksitaf4AnMCgYB24KyMuOHBAuVlnuf3PSfr +u3ZxqmcUX0D2BoK+1lw3lgzfQO26Qw5VtkjDBnIPL67IUYRZX2YvXsJPWaRRrF72 +yEqFXDWKbcE+Tl0LxLj6mLW5RKJP8LTybaIBgkyUaLtzTRr+TfK29CC8/FzWGiTf +oJQozL3TAlvjoadEPrLnjg== +-----END PRIVATE KEY----- diff --git a/tests/fixtures/gearman/server.pem b/tests/fixtures/gearman/server.pem new file mode 100644 index 000000000..1c85fad38 --- /dev/null +++ b/tests/fixtures/gearman/server.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDRTCCAi0CCQDnKP1tRJr+1zANBgkqhkiG9w0BAQsFADBiMQswCQYDVQQGEwJV +UzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEdMBsGA1UECgwUT3Bl +blN0YWNrIEZvdW5kYXRpb24xEzARBgNVBAMMCmdlYXJtYW4tY2EwHhcNMTcwNjE0 +MTQwNjM1WhcNMjcwNjEyMTQwNjM1WjBnMQswCQYDVQQGEwJVUzEOMAwGA1UECAwF +VGV4YXMxDzANBgNVBAcMBkF1c3RpbjEdMBsGA1UECgwUT3BlblN0YWNrIEZvdW5k +YXRpb24xGDAWBgNVBAMMD25vZGVwb29sLXNlcnZlcjCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBALOgqRpq5M0FGvLVjlgI0bfwivFbQAflM/ld1MX+r5C7 +Wn2idGwAKJJlfxhgCj/LqNVtndYqKFV9UhDD3Gs9MKgDQDAy0jx+IrvRjBK4aDqC +oEEKvw1c5pixgUmTZWT7y62U1KYtiVxJMplHQduFY2aMf2WPlaeAvRc9JtyiiC29 +633bXXWoZ8FmuD7zoHdcmHb/o8Ku71ia5+qWigWiDtDhCjoGfPsOH6HZCO0OYZIX +PR1D0+o/wmCHctJ9ZGTlOuMay2EqzNFNkJANJdzVdzACp87bV658/2hOl9zTNN0A +4LW2yl0//rfaroHQZ/yxIScykqbC4ZQffPZkJjoNITUCAwEAATANBgkqhkiG9w0B +AQsFAAOCAQEAlqcjSBG96JnKcSlw4ntxJiSGja5iuMi3yVpQS8G3ak6i8eGYlqMH +SCWC96ZfXr/KjVyF3AsD554e54pEAywcFLH4QzZoceWc5L2etfTCa9cInQsiNpvV +CfvVADRX4Ib7ozb4MJFJFy5OWnhPO6CcknA2KdTergKIichBmR0LvuUZEblwHOcg +HAwxpZirNofs/i+aXnIgKAIC97WY1S+8SL5cEfdR0Sd9SpbCLVgSdyGhxm0NE2ls +38jQhwYIVkpYYJd/jsyGtiHCDT4rkSEJlRWYfLXfSkyjtiERASqs/NEgrnbkgp/l +Sa2wc5cjntNzls2ey7bkpZbgwOvGQVjS7w== +-----END CERTIFICATE----- diff --git a/tests/fixtures/zuul-connections-gerrit-and-github.conf b/tests/fixtures/zuul-connections-gerrit-and-github.conf index bd05c7597..69e7f8b95 100644 --- a/tests/fixtures/zuul-connections-gerrit-and-github.conf +++ b/tests/fixtures/zuul-connections-gerrit-and-github.conf @@ -3,7 +3,6 @@ server=127.0.0.1 [zuul] tenant_config=config/multi-driver/main.yaml -job_name_in_report=true [merger] git_dir=/tmp/zuul-test/git diff --git a/tests/fixtures/zuul-connections-merger.conf b/tests/fixtures/zuul-connections-merger.conf index 7a1bc42b2..4499493b0 100644 --- a/tests/fixtures/zuul-connections-merger.conf +++ b/tests/fixtures/zuul-connections-merger.conf @@ -2,7 +2,6 @@ server=127.0.0.1 [zuul] -job_name_in_report=true status_url=http://zuul.example.com/status [merger] diff --git a/tests/fixtures/zuul-connections-multiple-gerrits.conf b/tests/fixtures/zuul-connections-multiple-gerrits.conf index d1522ec76..43b00ef3e 100644 --- a/tests/fixtures/zuul-connections-multiple-gerrits.conf +++ b/tests/fixtures/zuul-connections-multiple-gerrits.conf @@ -3,7 +3,6 @@ server=127.0.0.1 [zuul] tenant_config=main.yaml -job_name_in_report=true [merger] git_dir=/tmp/zuul-test/merger-git diff --git a/tests/fixtures/zuul-connections-same-gerrit.conf b/tests/fixtures/zuul-connections-same-gerrit.conf index 8ddd0f1fe..8a998cfae 100644 --- a/tests/fixtures/zuul-connections-same-gerrit.conf +++ b/tests/fixtures/zuul-connections-same-gerrit.conf @@ -3,7 +3,6 @@ server=127.0.0.1 [zuul] tenant_config=config/zuul-connections-same-gerrit/main.yaml -job_name_in_report=true [merger] git_dir=/tmp/zuul-test/merger-git diff --git a/tests/fixtures/zuul-git-driver.conf b/tests/fixtures/zuul-git-driver.conf index 499b56431..b6d347391 100644 --- a/tests/fixtures/zuul-git-driver.conf +++ b/tests/fixtures/zuul-git-driver.conf @@ -3,7 +3,6 @@ server=127.0.0.1 [zuul] tenant_config=config/zuul-connections-same-gerrit/main.yaml -job_name_in_report=true [merger] git_dir=/tmp/zuul-test/git diff --git a/tests/fixtures/zuul-github-driver.conf b/tests/fixtures/zuul-github-driver.conf index dfa813d36..dc28f986b 100644 --- a/tests/fixtures/zuul-github-driver.conf +++ b/tests/fixtures/zuul-github-driver.conf @@ -2,7 +2,6 @@ server=127.0.0.1 [zuul] -job_name_in_report=true status_url=http://zuul.example.com/status/#{change.number},{change.patchset} [merger] diff --git a/tests/fixtures/zuul-push-reqs.conf b/tests/fixtures/zuul-push-reqs.conf index 661ac7937..c5272aae9 100644 --- a/tests/fixtures/zuul-push-reqs.conf +++ b/tests/fixtures/zuul-push-reqs.conf @@ -2,7 +2,6 @@ server=127.0.0.1 [zuul] -job_name_in_report=true status_url=http://zuul.example.com/status [merger] diff --git a/tests/fixtures/zuul-sql-driver-bad.conf b/tests/fixtures/zuul-sql-driver-bad.conf index a4df735ec..1f1b75f12 100644 --- a/tests/fixtures/zuul-sql-driver-bad.conf +++ b/tests/fixtures/zuul-sql-driver-bad.conf @@ -3,7 +3,6 @@ server=127.0.0.1 [zuul] layout_config=layout-connections-multiple-voters.yaml -job_name_in_report=true [merger] git_dir=/tmp/zuul-test/merger-git diff --git a/tests/fixtures/zuul-sql-driver.conf b/tests/fixtures/zuul-sql-driver.conf index 42ab1bac8..6fdd08162 100644 --- a/tests/fixtures/zuul-sql-driver.conf +++ b/tests/fixtures/zuul-sql-driver.conf @@ -4,7 +4,6 @@ server=127.0.0.1 [zuul] tenant_config=main.yaml url_pattern=http://logs.example.com/{change.number}/{change.patchset}/{pipeline.name}/{job.name}/{build.number} -job_name_in_report=true [merger] git_dir=/tmp/zuul-test/merger-git diff --git a/tests/fixtures/zuul.conf b/tests/fixtures/zuul.conf index cd80a4547..c4cfe7070 100644 --- a/tests/fixtures/zuul.conf +++ b/tests/fixtures/zuul.conf @@ -3,7 +3,6 @@ server=127.0.0.1 [zuul] tenant_config=main.yaml -job_name_in_report=true [merger] git_dir=/tmp/zuul-test/merger-git diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py index 4979087b2..ba8e497ae 100644 --- a/tests/unit/test_github_driver.py +++ b/tests/unit/test_github_driver.py @@ -248,16 +248,18 @@ class TestGithubDriver(ZuulTestCase): @simple_layout('layouts/reporting-github.yaml', driver='github') def test_reporting(self): + project = 'org/project' # pipeline reports pull status both on start and success self.executor_server.hold_jobs_in_build = True - A = self.fake_github.openFakePullRequest('org/project', 'master', 'A') + A = self.fake_github.openFakePullRequest(project, 'master', 'A') self.fake_github.emitEvent(A.getPullRequestOpenedEvent()) self.waitUntilSettled() # We should have a status container for the head sha - self.assertIn(A.head_sha, A.statuses.keys()) + statuses = self.fake_github.statuses[project][A.head_sha] + self.assertIn(A.head_sha, self.fake_github.statuses[project].keys()) # We should only have one status for the head sha - self.assertEqual(1, len(A.statuses[A.head_sha])) - check_status = A.statuses[A.head_sha][0] + self.assertEqual(1, len(statuses)) + check_status = statuses[0] check_url = ('http://zuul.example.com/status/#%s,%s' % (A.number, A.head_sha)) self.assertEqual('tenant-one/check', check_status['context']) @@ -270,8 +272,9 @@ class TestGithubDriver(ZuulTestCase): self.executor_server.release() self.waitUntilSettled() # We should only have two statuses for the head sha - self.assertEqual(2, len(A.statuses[A.head_sha])) - check_status = A.statuses[A.head_sha][0] + statuses = self.fake_github.statuses[project][A.head_sha] + self.assertEqual(2, len(statuses)) + check_status = statuses[0] check_url = ('http://zuul.example.com/status/#%s,%s' % (A.number, A.head_sha)) self.assertEqual('tenant-one/check', check_status['context']) @@ -286,7 +289,8 @@ class TestGithubDriver(ZuulTestCase): self.fake_github.emitEvent( A.getCommentAddedEvent('reporting check')) self.waitUntilSettled() - self.assertEqual(2, len(A.statuses[A.head_sha])) + statuses = self.fake_github.statuses[project][A.head_sha] + self.assertEqual(2, len(statuses)) # comments increased by one for the start message self.assertEqual(2, len(A.comments)) self.assertThat(A.comments[1], @@ -295,8 +299,9 @@ class TestGithubDriver(ZuulTestCase): self.executor_server.release() self.waitUntilSettled() # pipeline reports success status - self.assertEqual(3, len(A.statuses[A.head_sha])) - report_status = A.statuses[A.head_sha][0] + statuses = self.fake_github.statuses[project][A.head_sha] + self.assertEqual(3, len(statuses)) + report_status = statuses[0] self.assertEqual('tenant-one/reporting', report_status['context']) self.assertEqual('success', report_status['state']) self.assertEqual(2, len(A.comments)) diff --git a/tests/unit/test_github_requirements.py b/tests/unit/test_github_requirements.py index 5dd6e80e4..301ea2fbe 100644 --- a/tests/unit/test_github_requirements.py +++ b/tests/unit/test_github_requirements.py @@ -25,7 +25,8 @@ class TestGithubRequirements(ZuulTestCase): @simple_layout('layouts/requirements-github.yaml', driver='github') def test_pipeline_require_status(self): "Test pipeline requirement: status" - A = self.fake_github.openFakePullRequest('org/project1', 'master', 'A') + project = 'org/project1' + A = self.fake_github.openFakePullRequest(project, 'master', 'A') # A comment event that we will keep submitting to trigger comment = A.getCommentAddedEvent('test me') self.fake_github.emitEvent(comment) @@ -34,13 +35,15 @@ class TestGithubRequirements(ZuulTestCase): self.assertEqual(len(self.history), 0) # An error status should not cause it to be enqueued - A.setStatus(A.head_sha, 'error', 'null', 'null', 'check') + self.fake_github.setCommitStatus(project, A.head_sha, 'error', + context='check') self.fake_github.emitEvent(comment) self.waitUntilSettled() self.assertEqual(len(self.history), 0) # A success status goes in - A.setStatus(A.head_sha, 'success', 'null', 'null', 'check') + self.fake_github.setCommitStatus(project, A.head_sha, 'success', + context='check') self.fake_github.emitEvent(comment) self.waitUntilSettled() self.assertEqual(len(self.history), 1) @@ -49,7 +52,8 @@ class TestGithubRequirements(ZuulTestCase): @simple_layout('layouts/requirements-github.yaml', driver='github') def test_trigger_require_status(self): "Test trigger requirement: status" - A = self.fake_github.openFakePullRequest('org/project1', 'master', 'A') + project = 'org/project1' + A = self.fake_github.openFakePullRequest(project, 'master', 'A') # A comment event that we will keep submitting to trigger comment = A.getCommentAddedEvent('trigger me') self.fake_github.emitEvent(comment) @@ -58,13 +62,15 @@ class TestGithubRequirements(ZuulTestCase): self.assertEqual(len(self.history), 0) # An error status should not cause it to be enqueued - A.setStatus(A.head_sha, 'error', 'null', 'null', 'check') + self.fake_github.setCommitStatus(project, A.head_sha, 'error', + context='check') self.fake_github.emitEvent(comment) self.waitUntilSettled() self.assertEqual(len(self.history), 0) # A success status goes in - A.setStatus(A.head_sha, 'success', 'null', 'null', 'check') + self.fake_github.setCommitStatus(project, A.head_sha, 'success', + context='check') self.fake_github.emitEvent(comment) self.waitUntilSettled() self.assertEqual(len(self.history), 1) @@ -73,10 +79,12 @@ class TestGithubRequirements(ZuulTestCase): @simple_layout('layouts/requirements-github.yaml', driver='github') def test_trigger_on_status(self): "Test trigger on: status" - A = self.fake_github.openFakePullRequest('org/project2', 'master', 'A') + project = 'org/project2' + A = self.fake_github.openFakePullRequest(project, 'master', 'A') # An error status should not cause it to be enqueued - A.setStatus(A.head_sha, 'error', 'null', 'null', 'check') + self.fake_github.setCommitStatus(project, A.head_sha, 'error', + context='check') self.fake_github.emitEvent(A.getCommitStatusEvent('check', state='error')) self.waitUntilSettled() @@ -84,7 +92,8 @@ class TestGithubRequirements(ZuulTestCase): # A success status from unknown user should not cause it to be # enqueued - A.setStatus(A.head_sha, 'success', 'null', 'null', 'check', user='foo') + self.fake_github.setCommitStatus(project, A.head_sha, 'success', + context='check', user='foo') self.fake_github.emitEvent(A.getCommitStatusEvent('check', state='success', user='foo')) @@ -92,7 +101,8 @@ class TestGithubRequirements(ZuulTestCase): self.assertEqual(len(self.history), 0) # A success status from zuul goes in - A.setStatus(A.head_sha, 'success', 'null', 'null', 'check') + self.fake_github.setCommitStatus(project, A.head_sha, 'success', + context='check') self.fake_github.emitEvent(A.getCommitStatusEvent('check')) self.waitUntilSettled() self.assertEqual(len(self.history), 1) @@ -100,7 +110,8 @@ class TestGithubRequirements(ZuulTestCase): # An error status for a different context should not cause it to be # enqueued - A.setStatus(A.head_sha, 'error', 'null', 'null', 'gate') + self.fake_github.setCommitStatus(project, A.head_sha, 'error', + context='gate') self.fake_github.emitEvent(A.getCommitStatusEvent('gate', state='error')) self.waitUntilSettled() diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py index 839007d93..1124beb6d 100755 --- a/tests/unit/test_scheduler.py +++ b/tests/unit/test_scheduler.py @@ -35,12 +35,36 @@ import zuul.rpcclient import zuul.model from tests.base import ( + SSLZuulTestCase, ZuulTestCase, repack_repo, simple_layout, ) +class TestSchedulerSSL(SSLZuulTestCase): + tenant_config_file = 'config/single-tenant/main.yaml' + + def test_jobs_executed(self): + "Test that jobs are executed and a change is merged" + + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + A.addApproval('code-review', 2) + self.fake_gerrit.addEvent(A.addApproval('approved', 1)) + self.waitUntilSettled() + self.assertEqual(self.getJobFromHistory('project-merge').result, + 'SUCCESS') + self.assertEqual(self.getJobFromHistory('project-test1').result, + 'SUCCESS') + self.assertEqual(self.getJobFromHistory('project-test2').result, + 'SUCCESS') + self.assertEqual(A.data['status'], 'MERGED') + self.assertEqual(A.reported, 2) + self.assertEqual(self.getJobFromHistory('project-test1').node, + 'label1') + self.assertIsNone(self.getJobFromHistory('project-test2').node) + + class TestScheduler(ZuulTestCase): tenant_config_file = 'config/single-tenant/main.yaml' diff --git a/zuul/ansible/callback/zuul_stream.py b/zuul/ansible/callback/zuul_stream.py index 260f4ab8f..c064f3e58 100644 --- a/zuul/ansible/callback/zuul_stream.py +++ b/zuul/ansible/callback/zuul_stream.py @@ -14,10 +14,10 @@ # along with Ansible. If not, see <http://www.gnu.org/licenses/>. import datetime -import multiprocessing import logging import os import socket +import threading import time import uuid @@ -103,14 +103,14 @@ class CallbackModule(default.CallbackModule): self._log = logging.getLogger('zuul.executor.ansible') def _read_log(self, host, ip, log_id, task_name): - self._log.debug("[%s] Starting to log %s for task %s" - % (host, log_id, task_name)) + self._display.display("[%s] Starting to log %s for task %s" + % (host, log_id, task_name)) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) while True: try: s.connect((ip, LOG_STREAM_PORT)) except Exception: - self._log.debug("[%s] Waiting on logger" % host) + self._display.vvv("[%s] Waiting on logger" % host) time.sleep(0.1) continue msg = "%s\n" % log_id @@ -119,7 +119,8 @@ class CallbackModule(default.CallbackModule): if "[Zuul] Task exit code" in line: return else: - ts, ln = line.strip().split(' | ', 1) + ts, ln = line.split(' | ', 1) + ln = ln.strip() self._log.info("%s | %s | %s " % (ts, host, ln)) @@ -161,14 +162,22 @@ class CallbackModule(default.CallbackModule): 'ansible_host', play_vars[host].get( 'ansible_inventory_host')) self._host_dict[host] = ip - self._streamer = multiprocessing.Process( + self._streamer = threading.Thread( target=self._read_log, args=(host, ip, log_id, task_name)) self._streamer.daemon = True self._streamer.start() - def v2_runner_on_failed(self, result, ignore_errors=False): + def _stop_streamer(self): if self._streamer: - self._streamer.join() + self._streamer.join(30) + if self._streamer.is_alive(): + msg = "{now} | [Zuul] Log Stream did not terminate".format( + now=datetime.datetime.now()) + self._log.info(msg) + self._display.display("WARNING: Streamer could not join") + + def v2_runner_on_failed(self, result, ignore_errors=False): + self._stop_streamer() if result._task.action in ('command', 'shell'): zuul_filter_result(result._result) self._handle_exception(result._result) @@ -194,8 +203,7 @@ class CallbackModule(default.CallbackModule): if result._task.action in ('include', 'include_role'): return - if self._streamer: - self._streamer.join() + self._stop_streamer() if result._result.get('changed', False): status = 'changed' diff --git a/zuul/ansible/library/command.py b/zuul/ansible/library/command.py index 4b3a30fcf..39a961eaf 100644 --- a/zuul/ansible/library/command.py +++ b/zuul/ansible/library/command.py @@ -19,6 +19,10 @@ # You should have received a copy of the GNU General Public License # along with this software. If not, see <http://www.gnu.org/licenses/>. +ANSIBLE_METADATA = {'metadata_version': '1.0', + 'status': ['stableinterface'], + 'supported_by': 'core'} + # flake8: noqa # This file shares a significant chunk of code with an upstream ansible # function, run_command. The goal is to not have to fork quite so much @@ -34,7 +38,7 @@ module: command short_description: Executes a command on a remote node version_added: historical description: - - The M(command) module takes the command name followed by a list of space-delimited arguments. + - The C(command) module takes the command name followed by a list of space-delimited arguments. - The given command will be executed on all selected nodes. It will not be processed through the shell, so variables like C($HOME) and operations like C("<"), C(">"), C("|"), C(";") and C("&") will not work (use the M(shell) @@ -76,30 +80,33 @@ options: - if command warnings are on in ansible.cfg, do not warn about this particular line if set to no/false. required: false notes: - - If you want to run a command through the shell (say you are using C(<), - C(>), C(|), etc), you actually want the M(shell) module instead. The - M(command) module is much more secure as it's not affected by the user's - environment. - - " C(creates), C(removes), and C(chdir) can be specified after the command. For instance, if you only want to run a command if a certain file does not exist, use this." + - If you want to run a command through the shell (say you are using C(<), C(>), C(|), etc), you actually want the M(shell) module instead. + The C(command) module is much more secure as it's not affected by the user's environment. + - " C(creates), C(removes), and C(chdir) can be specified after the command. + For instance, if you only want to run a command if a certain file does not exist, use this." author: - Ansible Core Team - Michael DeHaan ''' EXAMPLES = ''' -# Example from Ansible Playbooks. -- command: /sbin/shutdown -t now +- name: return motd to registered var + command: cat /etc/motd + register: mymotd -# Run the command if the specified file does not exist. -- command: /usr/bin/make_database.sh arg1 arg2 creates=/path/to/database +- name: Run the command if the specified file does not exist. + command: /usr/bin/make_database.sh arg1 arg2 creates=/path/to/database -# You can also use the 'args' form to provide the options. This command -# will change the working directory to somedir/ and will only run when -# /path/to/database doesn't exist. -- command: /usr/bin/make_database.sh arg1 arg2 +# You can also use the 'args' form to provide the options. +- name: This command will change the working directory to somedir/ and will only run when /path/to/database doesn't exist. + command: /usr/bin/make_database.sh arg1 arg2 args: chdir: somedir/ creates: /path/to/database + +- name: safely use templated variable to run command. Always use the quote filter to avoid injection issues. + command: cat {{ myfile|quote }} + register: myoutput ''' import datetime @@ -116,10 +123,19 @@ import traceback import threading from ansible.module_utils.basic import AnsibleModule, heuristic_log_sanitize -from ansible.module_utils.basic import get_exception -# ZUUL: Hardcode python2 until we're on ansible 2.2 -from ast import literal_eval - +from ansible.module_utils.pycompat24 import get_exception, literal_eval +from ansible.module_utils.six import ( + PY2, + PY3, + b, + binary_type, + integer_types, + iteritems, + string_types, + text_type, +) +from ansible.module_utils.six.moves import map, reduce +from ansible.module_utils._text import to_native, to_bytes, to_text LOG_STREAM_FILE = '/tmp/console-{log_uuid}.log' PASSWD_ARG_RE = re.compile(r'^[-]{0,2}pass[-]?(word|wd)?') @@ -166,7 +182,7 @@ def follow(fd, log_uuid): # Taken from ansible/module_utils/basic.py ... forking the method for now # so that we can dive in and figure out how to make appropriate hook points -def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, executable=None, data=None, binary_data=False, path_prefix=None, cwd=None, use_unsafe_shell=False, prompt_regex=None, environ_update=None): +def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, executable=None, data=None, binary_data=False, path_prefix=None, cwd=None, use_unsafe_shell=False, prompt_regex=None, environ_update=None, umask=None, encoding='utf-8', errors='surrogate_or_strict'): ''' Execute a command, returns rc, stdout, and stderr. @@ -188,7 +204,27 @@ def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, ex :kw prompt_regex: Regex string (not a compiled regex) which can be used to detect prompts in the stdout which would otherwise cause the execution to hang (especially if no input data is specified) - :kwarg environ_update: dictionary to *update* os.environ with + :kw environ_update: dictionary to *update* os.environ with + :kw umask: Umask to be used when running the command. Default None + :kw encoding: Since we return native strings, on python3 we need to + know the encoding to use to transform from bytes to text. If you + want to always get bytes back, use encoding=None. The default is + "utf-8". This does not affect transformation of strings given as + args. + :kw errors: Since we return native strings, on python3 we need to + transform stdout and stderr from bytes to text. If the bytes are + undecodable in the ``encoding`` specified, then use this error + handler to deal with them. The default is ``surrogate_or_strict`` + which means that the bytes will be decoded using the + surrogateescape error handler if available (available on all + python3 versions we support) otherwise a UnicodeError traceback + will be raised. This does not affect transformations of strings + given as args. + :returns: A 3-tuple of return code (integer), stdout (native string), + and stderr (native string). On python2, stdout and stderr are both + byte strings. On python3, stdout and stderr are text strings converted + according to the encoding and errors parameters. If you want byte + strings on python3, use encoding=None to turn decoding to text off. ''' shell = False @@ -196,13 +232,15 @@ def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, ex if use_unsafe_shell: args = " ".join([pipes.quote(x) for x in args]) shell = True - elif isinstance(args, (str, unicode)) and use_unsafe_shell: + elif isinstance(args, (binary_type, text_type)) and use_unsafe_shell: shell = True - elif isinstance(args, (str, unicode)): + elif isinstance(args, (binary_type, text_type)): # On python2.6 and below, shlex has problems with text type - # ZUUL: Hardcode python2 until we're on ansible 2.2 - if isinstance(args, unicode): - args = args.encode('utf-8') + # On python3, shlex needs a text type. + if PY2: + args = to_bytes(args, errors='surrogate_or_strict') + elif PY3: + args = to_text(args, errors='surrogateescape') args = shlex.split(args) else: msg = "Argument 'args' to run_command must be list or string" @@ -210,6 +248,11 @@ def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, ex prompt_re = None if prompt_regex: + if isinstance(prompt_regex, text_type): + if PY3: + prompt_regex = to_bytes(prompt_regex, errors='surrogateescape') + elif PY2: + prompt_regex = to_bytes(prompt_regex, errors='surrogate_or_strict') try: prompt_re = re.compile(prompt_regex, re.MULTILINE) except re.error: @@ -217,7 +260,7 @@ def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, ex # expand things like $HOME and ~ if not shell: - args = [ os.path.expanduser(os.path.expandvars(x)) for x in args if x is not None ] + args = [os.path.expanduser(os.path.expandvars(x)) for x in args if x is not None] rc = 0 msg = None @@ -245,9 +288,9 @@ def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, ex # Clean out python paths set by ansiballz if 'PYTHONPATH' in os.environ: pypaths = os.environ['PYTHONPATH'].split(':') - pypaths = [x for x in pypaths \ - if not x.endswith('/ansible_modlib.zip') \ - and not x.endswith('/debug_dir')] + pypaths = [x for x in pypaths + if not x.endswith('/ansible_modlib.zip') and + not x.endswith('/debug_dir')] os.environ['PYTHONPATH'] = ':'.join(pypaths) if not os.environ['PYTHONPATH']: del os.environ['PYTHONPATH'] @@ -256,8 +299,13 @@ def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, ex # in reporting later, which strips out things like # passwords from the args list to_clean_args = args - # ZUUL: Hardcode python2 until we're on ansible 2.2 - if isinstance(args, (unicode, str)): + if PY2: + if isinstance(args, text_type): + to_clean_args = to_bytes(args) + else: + if isinstance(args, binary_type): + to_clean_args = to_text(args) + if isinstance(args, (text_type, binary_type)): to_clean_args = shlex.split(to_clean_args) clean_args = [] @@ -291,34 +339,36 @@ def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, ex stderr=subprocess.STDOUT, ) - if cwd and os.path.isdir(cwd): - kwargs['cwd'] = cwd - # store the pwd prev_dir = os.getcwd() # make sure we're in the right working directory if cwd and os.path.isdir(cwd): + cwd = os.path.abspath(os.path.expanduser(cwd)) + kwargs['cwd'] = cwd try: os.chdir(cwd) except (OSError, IOError): e = get_exception() self.fail_json(rc=e.errno, msg="Could not open %s, %s" % (cwd, str(e))) - try: + old_umask = None + if umask: + old_umask = os.umask(umask) + try: if self._debug: - if isinstance(args, list): - running = ' '.join(args) - else: - running = args - self.log('Executing: ' + running) + self.log('Executing: ' + clean_args) + cmd = subprocess.Popen(args, **kwargs) + # ZUUL: Replaced the excution loop with the zuul_runner run function cmd = subprocess.Popen(args, **kwargs) t = threading.Thread(target=follow, args=(cmd.stdout, zuul_log_id)) t.daemon = True t.start() + ret = cmd.wait() + # Give the thread that is writing the console log up to 10 seconds # to catch up and exit. If it hasn't done so by then, it is very # likely stuck in readline() because it spawed a child that is @@ -334,19 +384,21 @@ def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, ex # we can't close stdout (attempting to do so raises an # exception) , so this is disabled. # cmd.stdout.close() + # cmd.stderr.close() # ZUUL: stdout and stderr are in the console log file # ZUUL: return the saved log lines so we can ship them back - stdout = ''.join(_log_lines) - stderr = '' + stdout = b('').join(_log_lines) + stderr = b('') rc = cmd.returncode except (OSError, IOError): e = get_exception() - self.fail_json(rc=e.errno, msg=str(e), cmd=clean_args) + self.log("Error Executing CMD:%s Exception:%s" % (clean_args, to_native(e))) + self.fail_json(rc=e.errno, msg=to_native(e), cmd=clean_args) except Exception: - e = get_exception() - self.fail_json(rc=257, msg=str(e), exception=traceback.format_exc(), cmd=clean_args) + self.log("Error Executing CMD:%s Exception:%s" % (clean_args, to_native(traceback.format_exc()))) + self.fail_json(rc=257, msg=to_native(e), exception=traceback.format_exc(), cmd=clean_args) # Restore env settings for key, val in old_env_vals.items(): @@ -355,6 +407,9 @@ def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, ex else: os.environ[key] = val + if old_umask: + os.umask(old_umask) + if rc != 0 and check_rc: msg = heuristic_log_sanitize(stderr.rstrip(), self.no_log_values) self.fail_json(cmd=clean_args, rc=rc, stdout=stdout, stderr=stderr, msg=msg) @@ -362,6 +417,9 @@ def zuul_run_command(self, args, zuul_log_id, check_rc=False, close_fds=True, ex # reset the pwd os.chdir(prev_dir) + if encoding is not None: + return (rc, to_native(stdout, encoding=encoding, errors=errors), + to_native(stderr, encoding=encoding, errors=errors)) return (rc, stdout, stderr) @@ -392,24 +450,24 @@ def main(): # hence don't copy this one if you are looking to build others! module = AnsibleModule( argument_spec=dict( - _raw_params = dict(), - _uses_shell = dict(type='bool', default=False), - chdir = dict(type='path'), - executable = dict(), - creates = dict(type='path'), - removes = dict(type='path'), - warn = dict(type='bool', default=True), - environ = dict(type='dict', default=None), - zuul_log_id = dict(type='str'), + _raw_params = dict(), + _uses_shell = dict(type='bool', default=False), + chdir = dict(type='path'), + executable = dict(), + creates = dict(type='path'), + removes = dict(type='path'), + warn = dict(type='bool', default=True), + environ = dict(type='dict', default=None), + zuul_log_id = dict(type='str'), ) ) shell = module.params['_uses_shell'] chdir = module.params['chdir'] executable = module.params['executable'] - args = module.params['_raw_params'] - creates = module.params['creates'] - removes = module.params['removes'] + args = module.params['_raw_params'] + creates = module.params['creates'] + removes = module.params['removes'] warn = module.params['warn'] environ = module.params['environ'] zuul_log_id = module.params['zuul_log_id'] @@ -434,9 +492,9 @@ def main(): ) if removes: - # do not run the command if the line contains removes=filename - # and the filename does not exist. This allows idempotence - # of command executions. + # do not run the command if the line contains removes=filename + # and the filename does not exist. This allows idempotence + # of command executions. if not glob.glob(removes): module.exit_json( cmd=args, @@ -453,20 +511,20 @@ def main(): args = shlex.split(args) startd = datetime.datetime.now() - rc, out, err = zuul_run_command(module, args, zuul_log_id, executable=executable, use_unsafe_shell=shell, environ_update=environ) + rc, out, err = zuul_run_command(module, args, zuul_log_id, executable=executable, use_unsafe_shell=shell, encoding=None, environ_update=environ) endd = datetime.datetime.now() delta = endd - startd if out is None: - out = '' + out = b('') if err is None: - err = '' + err = b('') module.exit_json( cmd = args, - stdout = out.rstrip("\r\n"), - stderr = err.rstrip("\r\n"), + stdout = out.rstrip(b("\r\n")), + stderr = err.rstrip(b("\r\n")), rc = rc, start = str(startd), end = str(endd), diff --git a/zuul/cmd/client.py b/zuul/cmd/client.py index 3f67a3802..d8250c51d 100644 --- a/zuul/cmd/client.py +++ b/zuul/cmd/client.py @@ -125,6 +125,18 @@ class Client(zuul.cmd.ZuulApp): self.port = self.config.get('gearman', 'port') else: self.port = 4730 + if self.config.has_option('gearman', 'ssl_key'): + self.ssl_key = self.config.get('gearman', 'ssl_key') + else: + self.ssl_key = None + if self.config.has_option('gearman', 'ssl_cert'): + self.ssl_cert = self.config.get('gearman', 'ssl_cert') + else: + self.ssl_cert = None + if self.config.has_option('gearman', 'ssl_ca'): + self.ssl_ca = self.config.get('gearman', 'ssl_ca') + else: + self.ssl_ca = None if self.args.func(): sys.exit(0) @@ -132,7 +144,8 @@ class Client(zuul.cmd.ZuulApp): sys.exit(1) def enqueue(self): - client = zuul.rpcclient.RPCClient(self.server, self.port) + client = zuul.rpcclient.RPCClient( + self.server, self.port, self.ssl_key, self.ssl_cert, self.ssl_ca) r = client.enqueue(tenant=self.args.tenant, pipeline=self.args.pipeline, project=self.args.project, @@ -141,7 +154,8 @@ class Client(zuul.cmd.ZuulApp): return r def enqueue_ref(self): - client = zuul.rpcclient.RPCClient(self.server, self.port) + client = zuul.rpcclient.RPCClient( + self.server, self.port, self.ssl_key, self.ssl_cert, self.ssl_ca) r = client.enqueue_ref(tenant=self.args.tenant, pipeline=self.args.pipeline, project=self.args.project, @@ -152,14 +166,16 @@ class Client(zuul.cmd.ZuulApp): return r def promote(self): - client = zuul.rpcclient.RPCClient(self.server, self.port) + client = zuul.rpcclient.RPCClient( + self.server, self.port, self.ssl_key, self.ssl_cert, self.ssl_ca) r = client.promote(tenant=self.args.tenant, pipeline=self.args.pipeline, change_ids=self.args.changes) return r def show_running_jobs(self): - client = zuul.rpcclient.RPCClient(self.server, self.port) + client = zuul.rpcclient.RPCClient( + self.server, self.port, self.ssl_key, self.ssl_cert, self.ssl_ca) running_items = client.get_running_jobs() if len(running_items) == 0: diff --git a/zuul/cmd/scheduler.py b/zuul/cmd/scheduler.py index 5328bba5c..d16eb17dd 100755 --- a/zuul/cmd/scheduler.py +++ b/zuul/cmd/scheduler.py @@ -102,7 +102,22 @@ class Scheduler(zuul.cmd.ZuulApp): host = self.config.get('gearman_server', 'listen_address') else: host = None + if self.config.has_option('gearman_server', 'ssl_key'): + ssl_key = self.config.get('gearman_server', 'ssl_key') + else: + ssl_key = None + if self.config.has_option('gearman_server', 'ssl_cert'): + ssl_cert = self.config.get('gearman_server', 'ssl_cert') + else: + ssl_cert = None + if self.config.has_option('gearman_server', 'ssl_ca'): + ssl_ca = self.config.get('gearman_server', 'ssl_ca') + else: + ssl_ca = None zuul.lib.gearserver.GearServer(4730, + ssl_key=ssl_key, + ssl_cert=ssl_cert, + ssl_ca=ssl_ca, host=host, statsd_host=statsd_host, statsd_port=statsd_port, diff --git a/zuul/configloader.py b/zuul/configloader.py index 029b8406e..5e0fe65df 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -327,6 +327,7 @@ class JobParser(object): 'dependencies': to_list(str), 'allowed-projects': to_list(str), 'override-branch': str, + 'description': str, } return vs.Schema(job) diff --git a/zuul/driver/bubblewrap/__init__.py b/zuul/driver/bubblewrap/__init__.py index 083a384e8..e3a986611 100644 --- a/zuul/driver/bubblewrap/__init__.py +++ b/zuul/driver/bubblewrap/__init__.py @@ -152,6 +152,8 @@ class BubblewrapDriver(Driver, WrapperInterface): def main(args=None): + logging.basicConfig(level=logging.DEBUG) + driver = BubblewrapDriver() parser = argparse.ArgumentParser() diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py index 659d88b1b..f9f1c27d3 100644 --- a/zuul/driver/github/githubconnection.py +++ b/zuul/driver/github/githubconnection.py @@ -118,6 +118,12 @@ class GithubWebhookListener(): event = None if event: + if event.change_number: + project = self.connection.source.getProject(event.project_name) + self.connection._getChange(project, + event.change_number, + event.patch_number, + refresh=True) event.project_hostname = self.connection.canonical_hostname self.connection.logEvent(event) self.connection.sched.addEvent(event) @@ -247,7 +253,7 @@ class GithubWebhookListener(): raise webob.exc.HTTPUnauthorized( 'Please specify a X-Hub-Signature header with secret.') - payload_signature = 'sha1=' + hmac.new(secret, + payload_signature = 'sha1=' + hmac.new(secret.encode('utf-8'), body, hashlib.sha1).hexdigest() @@ -463,28 +469,20 @@ class GithubConnection(BaseConnection): if change not in relevant: del self._change_cache[key] - def getChange(self, event): + def getChange(self, event, refresh=False): """Get the change representing an event.""" project = self.source.getProject(event.project_name) if event.change_number: - change = PullRequest(event.project_name) - change.project = project - change.number = event.change_number + change = self._getChange(project, event.change_number, + event.patch_number, refresh=refresh) change.refspec = event.refspec change.branch = event.branch change.url = event.change_url change.updated_at = self._ghTimestampToDate(event.updated_at) - change.patchset = event.patch_number - change.files = self.getPullFileNames(project, change.number) - change.title = event.title - change.status = self._get_statuses(project, event.patch_number) - change.reviews = self.getPullReviews(project, change.number) change.source_event = event - change.open = self.getPullOpen(event.project_name, change.number) - change.is_current_patchset = self.getIsCurrent(event.project_name, - change.number, - event.patch_number) + change.is_current_patchset = (change.pr.get('head').get('sha') == + event.patch_number) elif event.ref: change = Ref(project) change.ref = event.ref @@ -497,6 +495,38 @@ class GithubConnection(BaseConnection): change = Ref(project) return change + def _getChange(self, project, number, patchset, refresh=False): + key = '%s/%s/%s' % (project.name, number, patchset) + change = self._change_cache.get(key) + if change and not refresh: + return change + if not change: + change = PullRequest(project.name) + change.project = project + change.number = number + change.patchset = patchset + self._change_cache[key] = change + try: + self._updateChange(change) + except Exception: + if key in self._change_cache: + del self._change_cache[key] + raise + return change + + def _updateChange(self, change): + self.log.info("Updating %s" % (change,)) + change.pr = self.getPull(change.project.name, change.number) + change.files = change.pr.get('files') + change.title = change.pr.get('title') + change.open = change.pr.get('state') == 'open' + change.status = self._get_statuses(change.project, + change.patchset) + change.reviews = self.getPullReviews(change.project, + change.number) + + return change + def getGitUrl(self, project): if self.git_ssh_key: return 'ssh://git@%s/%s.git' % (self.git_host, project) @@ -535,7 +565,15 @@ class GithubConnection(BaseConnection): def getPull(self, project_name, number): github = self.getGithubClient(project_name) owner, proj = project_name.split('/') - pr = github.pull_request(owner, proj, number).as_dict() + for retry in range(5): + probj = github.pull_request(owner, proj, number) + if probj is not None: + break + self.log.warning("Pull request #%s of %s/%s returned None!" % ( + number, owner, proj)) + time.sleep(1) + pr = probj.as_dict() + pr['files'] = [f.filename for f in probj.files()] log_rate_limit(self.log, github) return pr @@ -578,14 +616,6 @@ class GithubConnection(BaseConnection): return None return pulls.pop() - def getPullFileNames(self, project, number): - github = self.getGithubClient(project) - owner, proj = project.name.split('/') - filenames = [f.filename for f in - github.pull_request(owner, proj, number).files()] - log_rate_limit(self.log, github) - return filenames - def getPullReviews(self, project, number): owner, proj = project.name.split('/') @@ -723,14 +753,6 @@ class GithubConnection(BaseConnection): pull_request.remove_label(label) log_rate_limit(self.log, github) - def getPullOpen(self, project, number): - pr = self.getPull(project, number) - return pr.get('state') == 'open' - - def getIsCurrent(self, project, number, sha): - pr = self.getPull(project, number) - return pr.get('head').get('sha') == sha - def getPushedFileNames(self, event): files = set() for c in event.commits: diff --git a/zuul/driver/github/githubsource.py b/zuul/driver/github/githubsource.py index 1350b1095..1c2f72718 100644 --- a/zuul/driver/github/githubsource.py +++ b/zuul/driver/github/githubsource.py @@ -58,8 +58,8 @@ class GithubSource(BaseSource): """Called after configuration has been processed.""" pass - def getChange(self, event): - return self.connection.getChange(event) + def getChange(self, event, refresh=False): + return self.connection.getChange(event, refresh) def getProject(self, name): p = self.connection.getProject(name) @@ -87,10 +87,6 @@ class GithubSource(BaseSource): """Get the git-web url for a project.""" return self.connection.getGitwebUrl(project, sha) - def getPullFiles(self, project, number): - """Get filenames of the pull request""" - return self.connection.getPullFileNames(project, number) - def _ghTimestampToDate(self, timestamp): return time.strptime(timestamp, '%Y-%m-%dT%H:%M:%SZ') diff --git a/zuul/executor/client.py b/zuul/executor/client.py index f6961f302..6ecb27cc9 100644 --- a/zuul/executor/client.py +++ b/zuul/executor/client.py @@ -119,9 +119,21 @@ class ExecutorClient(object): port = config.get('gearman', 'port') else: port = 4730 + if self.config.has_option('gearman', 'ssl_key'): + ssl_key = self.config.get('gearman', 'ssl_key') + else: + ssl_key = None + if self.config.has_option('gearman', 'ssl_cert'): + ssl_cert = self.config.get('gearman', 'ssl_cert') + else: + ssl_cert = None + if self.config.has_option('gearman', 'ssl_ca'): + ssl_ca = self.config.get('gearman', 'ssl_ca') + else: + ssl_ca = None self.gearman = ZuulGearmanClient(self) - self.gearman.addServer(server, port) + self.gearman.addServer(server, port, ssl_key, ssl_cert, ssl_ca) self.cleanup_thread = GearmanCleanup(self) self.cleanup_thread.start() diff --git a/zuul/executor/server.py b/zuul/executor/server.py index c498fa463..a29eeb9e1 100644 --- a/zuul/executor/server.py +++ b/zuul/executor/server.py @@ -459,10 +459,22 @@ class ExecutorServer(object): port = self.config.get('gearman', 'port') else: port = 4730 + if self.config.has_option('gearman', 'ssl_key'): + ssl_key = self.config.get('gearman', 'ssl_key') + else: + ssl_key = None + if self.config.has_option('gearman', 'ssl_cert'): + ssl_cert = self.config.get('gearman', 'ssl_cert') + else: + ssl_cert = None + if self.config.has_option('gearman', 'ssl_ca'): + ssl_ca = self.config.get('gearman', 'ssl_ca') + else: + ssl_ca = None self.merger_worker = ExecutorMergeWorker(self, 'Zuul Executor Merger') - self.merger_worker.addServer(server, port) + self.merger_worker.addServer(server, port, ssl_key, ssl_cert, ssl_ca) self.executor_worker = gear.TextWorker('Zuul Executor Server') - self.executor_worker.addServer(server, port) + self.executor_worker.addServer(server, port, ssl_key, ssl_cert, ssl_ca) self.log.debug("Waiting for server") self.merger_worker.waitForServer() self.executor_worker.waitForServer() @@ -681,6 +693,13 @@ class AnsibleJob(object): RESULT_UNREACHABLE = 3 RESULT_ABORTED = 4 + RESULT_MAP = { + RESULT_NORMAL: 'RESULT_NORMAL', + RESULT_TIMED_OUT: 'RESULT_TIMED_OUT', + RESULT_UNREACHABLE: 'RESULT_UNREACHABLE', + RESULT_ABORTED: 'RESULT_ABORTED', + } + def __init__(self, executor_server, job): logger = logging.getLogger("zuul.AnsibleJob") self.log = AnsibleJobLogAdapter(logger, {'job': job.unique}) @@ -903,9 +922,10 @@ class AnsibleJob(object): host_vars = dict( ansible_host=ip, ansible_user=self.executor_server.default_username, - nodepool_az=node.get('az'), - nodepool_provider=node.get('provider'), - nodepool_region=node.get('region')) + nodepool=dict( + az=node.get('az'), + provider=node.get('provider'), + region=node.get('region'))) host_keys = [] for key in node.get('host_keys'): @@ -1170,7 +1190,6 @@ class AnsibleJob(object): self.jobdir.root) config.write('remote_tmp = %s/.ansible/remote_tmp\n' % self.jobdir.root) - config.write('private_key_file = %s\n' % self.private_key_file) config.write('retry_files_enabled = False\n') config.write('gathering = explicit\n') config.write('library = %s\n' @@ -1282,11 +1301,13 @@ class AnsibleJob(object): for line in iter(self.proc.stdout.readline, b''): line = line[:1024].rstrip() self.log.debug("Ansible output: %s" % (line,)) + self.log.debug("Ansible output terminated") ret = self.proc.wait() + self.log.debug("Ansible exit code: %s" % (ret,)) finally: if timeout: watchdog.stop() - self.log.debug("Ansible exit code: %s" % (ret,)) + self.log.debug("Stopped watchdog") with self.proc_lock: self.proc = None @@ -1317,5 +1338,8 @@ class AnsibleJob(object): if success is not None: cmd.extend(['-e', 'success=%s' % str(bool(success))]) - return self.runAnsible( + result, code = self.runAnsible( cmd=cmd, timeout=timeout, trusted=playbook.trusted) + self.log.debug("Ansible complete, result %s code %s" % ( + self.RESULT_MAP[result], code)) + return result, code diff --git a/zuul/lib/log_streamer.py b/zuul/lib/log_streamer.py index 59d52404d..669572350 100644 --- a/zuul/lib/log_streamer.py +++ b/zuul/lib/log_streamer.py @@ -144,6 +144,11 @@ class RequestHandler(ss.BaseRequestHandler): else: break + # See if the file has been removed, meaning we should stop + # streaming it. + if not os.path.exists(log.path): + return False + # At this point, we are waiting for more data to be written time.sleep(0.5) @@ -159,16 +164,6 @@ class RequestHandler(ss.BaseRequestHandler): if not ret: return False - # See if the file has been truncated - try: - st = os.stat(log.path) - if (st.st_ino != log.stat.st_ino or - st.st_size < log.size): - return True - except Exception: - return True - log.size = st.st_size - class CustomForkingTCPServer(ss.ForkingTCPServer): ''' diff --git a/zuul/merger/client.py b/zuul/merger/client.py index c98f20e4f..4054df6f3 100644 --- a/zuul/merger/client.py +++ b/zuul/merger/client.py @@ -79,9 +79,21 @@ class MergeClient(object): port = self.config.get('gearman', 'port') else: port = 4730 + if self.config.has_option('gearman', 'ssl_key'): + ssl_key = self.config.get('gearman', 'ssl_key') + else: + ssl_key = None + if self.config.has_option('gearman', 'ssl_cert'): + ssl_cert = self.config.get('gearman', 'ssl_cert') + else: + ssl_cert = None + if self.config.has_option('gearman', 'ssl_ca'): + ssl_ca = self.config.get('gearman', 'ssl_ca') + else: + ssl_ca = None self.log.debug("Connecting to gearman at %s:%s" % (server, port)) self.gearman = MergeGearmanClient(self) - self.gearman.addServer(server, port) + self.gearman.addServer(server, port, ssl_key, ssl_cert, ssl_ca) self.log.debug("Waiting for gearman") self.gearman.waitForServer() self.jobs = set() diff --git a/zuul/merger/server.py b/zuul/merger/server.py index 1a32f9615..7d7e771f2 100644 --- a/zuul/merger/server.py +++ b/zuul/merger/server.py @@ -54,8 +54,20 @@ class MergeServer(object): port = self.config.get('gearman', 'port') else: port = 4730 + if self.config.has_option('gearman', 'ssl_key'): + ssl_key = self.config.get('gearman', 'ssl_key') + else: + ssl_key = None + if self.config.has_option('gearman', 'ssl_cert'): + ssl_cert = self.config.get('gearman', 'ssl_cert') + else: + ssl_cert = None + if self.config.has_option('gearman', 'ssl_ca'): + ssl_ca = self.config.get('gearman', 'ssl_ca') + else: + ssl_ca = None self.worker = gear.TextWorker('Zuul Merger') - self.worker.addServer(server, port) + self.worker.addServer(server, port, ssl_key, ssl_cert, ssl_ca) self.log.debug("Waiting for server") self.worker.waitForServer() self.log.debug("Registering") diff --git a/zuul/reporter/__init__.py b/zuul/reporter/__init__.py index dc99c8bd2..89830d5fe 100644 --- a/zuul/reporter/__init__.py +++ b/zuul/reporter/__init__.py @@ -142,11 +142,7 @@ class BaseReporter(object): elapsed = ' in %ds' % (s) else: elapsed = '' - name = '' - if config.has_option('zuul', 'job_name_in_report'): - if config.getboolean('zuul', - 'job_name_in_report'): - name = job.name + ' ' + name = job.name + ' ' ret += '- %s%s : %s%s%s\n' % (name, url, result, elapsed, voting) return ret diff --git a/zuul/rpcclient.py b/zuul/rpcclient.py index d9809926d..6f0d34b0e 100644 --- a/zuul/rpcclient.py +++ b/zuul/rpcclient.py @@ -26,10 +26,10 @@ class RPCFailure(Exception): class RPCClient(object): log = logging.getLogger("zuul.RPCClient") - def __init__(self, server, port): + def __init__(self, server, port, ssl_key=None, ssl_cert=None, ssl_ca=None): self.log.debug("Connecting to gearman at %s:%s" % (server, port)) self.gearman = gear.Client() - self.gearman.addServer(server, port) + self.gearman.addServer(server, port, ssl_key, ssl_cert, ssl_ca) self.log.debug("Waiting for gearman") self.gearman.waitForServer() diff --git a/zuul/rpclistener.py b/zuul/rpclistener.py index 6508e848a..0079ab8c4 100644 --- a/zuul/rpclistener.py +++ b/zuul/rpclistener.py @@ -38,9 +38,23 @@ class RPCListener(object): port = self.config.get('gearman', 'port') else: port = 4730 + if self.config.has_option('gearman', 'ssl_key'): + ssl_key = self.config.get('gearman', 'ssl_key') + else: + ssl_key = None + if self.config.has_option('gearman', 'ssl_cert'): + ssl_cert = self.config.get('gearman', 'ssl_cert') + else: + ssl_cert = None + if self.config.has_option('gearman', 'ssl_ca'): + ssl_ca = self.config.get('gearman', 'ssl_ca') + else: + ssl_ca = None self.worker = gear.TextWorker('Zuul RPC Listener') - self.worker.addServer(server, port) + self.worker.addServer(server, port, ssl_key, ssl_cert, ssl_ca) + self.log.debug("Waiting for server") self.worker.waitForServer() + self.log.debug("Registering") self.register() self.thread = threading.Thread(target=self.run) self.thread.daemon = True |