diff options
36 files changed, 428 insertions, 107 deletions
diff --git a/README.rst b/README.rst index 52b89dfb6..8d0066530 100644 --- a/README.rst +++ b/README.rst @@ -10,6 +10,14 @@ preparation for the third major version of Zuul. We call this effort The latest documentation for Zuul v3 is published at: https://docs.openstack.org/infra/zuul/feature/zuulv3/ +If you are looking for the Edge routing service named Zuul that is +related to Netflix, it can be found here: +https://github.com/Netflix/zuul + +If you are looking for the Javascript testing tool named Zuul, it +can be found here: +https://github.com/defunctzombie/zuul + Contributing ------------ diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst index 3bec28afd..18bbfa3f4 100644 --- a/doc/source/admin/components.rst +++ b/doc/source/admin/components.rst @@ -408,7 +408,7 @@ The following sections of ``zuul.conf`` are used by the executor: Path to command socket file for the executor process. .. attr:: finger_port - :default: 79 + :default: 7900 Port to use for finger log streamer. @@ -451,13 +451,6 @@ The following sections of ``zuul.conf`` are used by the executor: SSH private key file to be used when logging into worker nodes. - .. attr:: user - :default: zuul - - User ID for the zuul-executor process. In normal operation as a - daemon, the executor should be started as the ``root`` user, but - it will drop privileges to this user during startup. - .. _admin_sitewide_variables: .. attr:: variables diff --git a/doc/source/index.rst b/doc/source/index.rst index 677e9584c..6e1b52e21 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -12,6 +12,14 @@ are installing or operating a Zuul system, you will also find the :doc:`admin/index` useful. If you want help make Zuul itself better, take a look at the :doc:`developer/index`. +If you are looking for the Edge routing service named Zuul that is +related to Netflix, it can be found here: +https://github.com/Netflix/zuul + +If you are looking for the Javascript testing tool named Zuul, it +can be found here: +https://github.com/defunctzombie/zuul + Contents: .. toctree:: diff --git a/doc/source/user/config.rst b/doc/source/user/config.rst index fff673b55..525cb3892 100644 --- a/doc/source/user/config.rst +++ b/doc/source/user/config.rst @@ -539,6 +539,13 @@ Here is an example of two job definitions: specified in a project's pipeline, set this attribute to ``true``. + .. attr:: protected + :default: false + + When set to ``true`` only jobs defined in the same project may inherit + from this job. Once this is set to ``true`` it cannot be reset to + ``false``. + .. attr:: success-message :default: SUCCESS diff --git a/requirements.txt b/requirements.txt index 39a2b0268..193c64e71 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,5 +25,6 @@ cryptography>=1.6 cachecontrol pyjwt iso8601 +yarl>=0.11,<1.0 aiohttp uvloop;python_version>='3.5' diff --git a/tests/base.py b/tests/base.py index 64f66579c..c4492426f 100755 --- a/tests/base.py +++ b/tests/base.py @@ -1619,6 +1619,8 @@ class FakeNodepool(object): data['username'] = 'fakeuser' if 'windows' in node_type: data['connection_type'] = 'winrm' + if 'network' in node_type: + data['connection_type'] = 'network_cli' data = json.dumps(data).encode('utf8') path = self.client.create(path, data, diff --git a/tests/fixtures/config/inventory/git/common-config/zuul.yaml b/tests/fixtures/config/inventory/git/common-config/zuul.yaml index 36789a321..f592eb48b 100644 --- a/tests/fixtures/config/inventory/git/common-config/zuul.yaml +++ b/tests/fixtures/config/inventory/git/common-config/zuul.yaml @@ -40,6 +40,8 @@ label: fakeuser-label - name: windows label: windows-label + - name: network + label: network-label - job: name: base diff --git a/tests/fixtures/config/protected/git/common-config/zuul.yaml b/tests/fixtures/config/protected/git/common-config/zuul.yaml new file mode 100644 index 000000000..c941573e6 --- /dev/null +++ b/tests/fixtures/config/protected/git/common-config/zuul.yaml @@ -0,0 +1,16 @@ +- pipeline: + name: check + manager: independent + trigger: + gerrit: + - event: patchset-created + success: + gerrit: + Verified: 1 + failure: + gerrit: + Verified: -1 + +- job: + name: base + parent: null diff --git a/tests/fixtures/config/protected/git/org_project/playbooks/job-protected.yaml b/tests/fixtures/config/protected/git/org_project/playbooks/job-protected.yaml new file mode 100644 index 000000000..f679dceae --- /dev/null +++ b/tests/fixtures/config/protected/git/org_project/playbooks/job-protected.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/protected/git/org_project/zuul.yaml b/tests/fixtures/config/protected/git/org_project/zuul.yaml new file mode 100644 index 000000000..95f33df6f --- /dev/null +++ b/tests/fixtures/config/protected/git/org_project/zuul.yaml @@ -0,0 +1,9 @@ +- job: + name: job-protected + protected: true + run: playbooks/job-protected.yaml + +- project: + name: org/project + check: + jobs: [] diff --git a/tests/fixtures/config/protected/git/org_project1/README b/tests/fixtures/config/protected/git/org_project1/README new file mode 100644 index 000000000..9daeafb98 --- /dev/null +++ b/tests/fixtures/config/protected/git/org_project1/README @@ -0,0 +1 @@ +test diff --git a/tests/fixtures/config/protected/git/org_project1/playbooks/job-child-notok.yaml b/tests/fixtures/config/protected/git/org_project1/playbooks/job-child-notok.yaml new file mode 100644 index 000000000..f679dceae --- /dev/null +++ b/tests/fixtures/config/protected/git/org_project1/playbooks/job-child-notok.yaml @@ -0,0 +1,2 @@ +- hosts: all + tasks: [] diff --git a/tests/fixtures/config/protected/git/org_project1/playbooks/placeholder b/tests/fixtures/config/protected/git/org_project1/playbooks/placeholder new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/fixtures/config/protected/git/org_project1/playbooks/placeholder diff --git a/tests/fixtures/config/protected/main.yaml b/tests/fixtures/config/protected/main.yaml new file mode 100644 index 000000000..5f57245cc --- /dev/null +++ b/tests/fixtures/config/protected/main.yaml @@ -0,0 +1,9 @@ +- tenant: + name: tenant-one + source: + gerrit: + config-projects: + - common-config + untrusted-projects: + - org/project + - org/project1 diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py index 197b5256d..c45da94cb 100644 --- a/tests/unit/test_connection.py +++ b/tests/unit/test_connection.py @@ -115,7 +115,7 @@ class TestSQLConnection(ZuulDBTestCase): self.assertEqual('check', buildset0['pipeline']) self.assertEqual('org/project', buildset0['project']) self.assertEqual(1, buildset0['change']) - self.assertEqual(1, buildset0['patchset']) + self.assertEqual('1', buildset0['patchset']) self.assertEqual('SUCCESS', buildset0['result']) self.assertEqual('Build succeeded.', buildset0['message']) self.assertEqual('tenant-one', buildset0['tenant']) @@ -141,7 +141,7 @@ class TestSQLConnection(ZuulDBTestCase): self.assertEqual('check', buildset1['pipeline']) self.assertEqual('org/project', buildset1['project']) self.assertEqual(2, buildset1['change']) - self.assertEqual(1, buildset1['patchset']) + self.assertEqual('1', buildset1['patchset']) self.assertEqual('FAILURE', buildset1['result']) self.assertEqual('Build failed.', buildset1['message']) @@ -194,7 +194,7 @@ class TestSQLConnection(ZuulDBTestCase): self.assertEqual('check', buildsets_resultsdb[0]['pipeline']) self.assertEqual('org/project', buildsets_resultsdb[0]['project']) self.assertEqual(1, buildsets_resultsdb[0]['change']) - self.assertEqual(1, buildsets_resultsdb[0]['patchset']) + self.assertEqual('1', buildsets_resultsdb[0]['patchset']) self.assertEqual('SUCCESS', buildsets_resultsdb[0]['result']) self.assertEqual('Build succeeded.', buildsets_resultsdb[0]['message']) @@ -215,7 +215,7 @@ class TestSQLConnection(ZuulDBTestCase): self.assertEqual( 'org/project', buildsets_resultsdb_failures[0]['project']) self.assertEqual(2, buildsets_resultsdb_failures[0]['change']) - self.assertEqual(1, buildsets_resultsdb_failures[0]['patchset']) + self.assertEqual('1', buildsets_resultsdb_failures[0]['patchset']) self.assertEqual('FAILURE', buildsets_resultsdb_failures[0]['result']) self.assertEqual( 'Build failed.', buildsets_resultsdb_failures[0]['message']) diff --git a/tests/unit/test_github_driver.py b/tests/unit/test_github_driver.py index 6ab1a2650..3942b0be8 100644 --- a/tests/unit/test_github_driver.py +++ b/tests/unit/test_github_driver.py @@ -50,6 +50,12 @@ class TestGithubDriver(ZuulTestCase): self.assertEqual(str(A.head_sha), zuulvars['patchset']) self.assertEqual('master', zuulvars['branch']) self.assertEqual(1, len(A.comments)) + self.assertThat( + A.comments[0], + MatchesRegex('.*\[project-test1 \]\(.*\).*', re.DOTALL)) + self.assertThat( + A.comments[0], + MatchesRegex('.*\[project-test2 \]\(.*\).*', re.DOTALL)) self.assertEqual(2, len(self.history)) # test_pull_unmatched_branch_event(self): diff --git a/tests/unit/test_inventory.py b/tests/unit/test_inventory.py index be504475a..b7e35ebd2 100644 --- a/tests/unit/test_inventory.py +++ b/tests/unit/test_inventory.py @@ -37,6 +37,12 @@ class TestInventory(ZuulTestCase): inv_path = os.path.join(build.jobdir.root, 'ansible', 'inventory.yaml') return yaml.safe_load(open(inv_path, 'r')) + def _get_setup_inventory(self, name): + build = self.getBuildByName(name) + setup_inv_path = os.path.join(build.jobdir.root, 'ansible', + 'setup-inventory.yaml') + return yaml.safe_load(open(setup_inv_path, 'r')) + def test_single_inventory(self): inventory = self._get_build_inventory('single-inventory') @@ -131,3 +137,23 @@ class TestInventory(ZuulTestCase): self.executor_server.release() self.waitUntilSettled() + + def test_setup_inventory(self): + + setup_inventory = self._get_setup_inventory('hostvars-inventory') + inventory = self._get_build_inventory('hostvars-inventory') + + self.assertIn('all', inventory) + self.assertIn('hosts', inventory['all']) + + self.assertIn('default', setup_inventory['all']['hosts']) + self.assertIn('fakeuser', setup_inventory['all']['hosts']) + self.assertIn('windows', setup_inventory['all']['hosts']) + self.assertNotIn('network', setup_inventory['all']['hosts']) + self.assertIn('default', inventory['all']['hosts']) + self.assertIn('fakeuser', inventory['all']['hosts']) + self.assertIn('windows', inventory['all']['hosts']) + self.assertIn('network', inventory['all']['hosts']) + + self.executor_server.release() + self.waitUntilSettled() diff --git a/tests/unit/test_streaming.py b/tests/unit/test_streaming.py index 59dd8b016..b999106c8 100644 --- a/tests/unit/test_streaming.py +++ b/tests/unit/test_streaming.py @@ -41,13 +41,13 @@ class TestLogStreamer(tests.base.BaseTestCase): def startStreamer(self, port, root=None): if not root: root = tempfile.gettempdir() - return zuul.lib.log_streamer.LogStreamer(None, self.host, port, root) + return zuul.lib.log_streamer.LogStreamer(self.host, port, root) def test_start_stop(self): - port = 7900 - streamer = self.startStreamer(port) + streamer = self.startStreamer(0) self.addCleanup(streamer.stop) + port = streamer.server.socket.getsockname()[1] s = socket.create_connection((self.host, port)) s.close() @@ -77,8 +77,9 @@ class TestStreaming(tests.base.AnsibleZuulTestCase): def startStreamer(self, port, build_uuid, root=None): if not root: root = tempfile.gettempdir() - self.streamer = zuul.lib.log_streamer.LogStreamer(None, self.host, + self.streamer = zuul.lib.log_streamer.LogStreamer(self.host, port, root) + port = self.streamer.server.socket.getsockname()[1] s = socket.create_connection((self.host, port)) self.addCleanup(s.close) @@ -129,10 +130,9 @@ class TestStreaming(tests.base.AnsibleZuulTestCase): # Create a thread to stream the log. We need this to be happening # before we create the flag file to tell the job to complete. - port = 7901 streamer_thread = threading.Thread( target=self.startStreamer, - args=(port, build.uuid, self.executor_server.jobdir_root,) + args=(0, build.uuid, self.executor_server.jobdir_root,) ) streamer_thread.start() self.addCleanup(self.stopStreamer) @@ -209,7 +209,7 @@ class TestStreaming(tests.base.AnsibleZuulTestCase): def test_websocket_streaming(self): # Start the finger streamer daemon streamer = zuul.lib.log_streamer.LogStreamer( - None, self.host, 0, self.executor_server.jobdir_root) + self.host, 0, self.executor_server.jobdir_root) self.addCleanup(streamer.stop) # Need to set the streaming port before submitting the job @@ -294,7 +294,7 @@ class TestStreaming(tests.base.AnsibleZuulTestCase): def test_finger_gateway(self): # Start the finger streamer daemon streamer = zuul.lib.log_streamer.LogStreamer( - None, self.host, 0, self.executor_server.jobdir_root) + self.host, 0, self.executor_server.jobdir_root) self.addCleanup(streamer.stop) finger_port = streamer.server.socket.getsockname()[1] diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py index 2779e6e66..163a58b90 100755 --- a/tests/unit/test_v3.py +++ b/tests/unit/test_v3.py @@ -73,6 +73,110 @@ class TestMultipleTenants(AnsibleZuulTestCase): "not affect tenant one") +class TestProtected(ZuulTestCase): + + tenant_config_file = 'config/protected/main.yaml' + + def test_protected_ok(self): + # test clean usage of final parent job + in_repo_conf = textwrap.dedent( + """ + - job: + name: job-protected + protected: true + run: playbooks/job-protected.yaml + + - project: + name: org/project + check: + jobs: + - job-child-ok + + - job: + name: job-child-ok + parent: job-protected + + - project: + name: org/project + check: + jobs: + - job-child-ok + + """) + + file_dict = {'zuul.yaml': in_repo_conf} + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', + files=file_dict) + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertEqual(A.reported, 1) + self.assertEqual(A.patchsets[-1]['approvals'][0]['value'], '1') + + def test_protected_reset(self): + # try to reset protected flag + in_repo_conf = textwrap.dedent( + """ + - job: + name: job-protected + protected: true + run: playbooks/job-protected.yaml + + - job: + name: job-child-reset-protected + parent: job-protected + protected: false + + - project: + name: org/project + check: + jobs: + - job-child-reset-protected + + """) + + file_dict = {'zuul.yaml': in_repo_conf} + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A', + files=file_dict) + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + # The second patch tried to override some variables. + # Thus it should fail. + self.assertEqual(A.reported, 1) + self.assertEqual(A.patchsets[-1]['approvals'][0]['value'], '-1') + self.assertIn('Unable to reset protected attribute', A.messages[0]) + + def test_protected_inherit_not_ok(self): + # try to inherit from a protected job in different project + in_repo_conf = textwrap.dedent( + """ + - job: + name: job-child-notok + run: playbooks/job-child-notok.yaml + parent: job-protected + + - project: + name: org/project1 + check: + jobs: + - job-child-notok + + """) + + file_dict = {'zuul.yaml': in_repo_conf} + A = self.fake_gerrit.addFakeChange('org/project1', 'master', 'A', + files=file_dict) + self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) + self.waitUntilSettled() + + self.assertEqual(A.reported, 1) + self.assertEqual(A.patchsets[-1]['approvals'][0]['value'], '-1') + self.assertIn( + "which is defined in review.example.com/org/project is protected " + "and cannot be inherited from other projects.", A.messages[0]) + + class TestFinal(ZuulTestCase): tenant_config_file = 'config/final/main.yaml' diff --git a/tools/encrypt_secret.py b/tools/encrypt_secret.py index c0ee9be64..4cb166631 100755 --- a/tools/encrypt_secret.py +++ b/tools/encrypt_secret.py @@ -46,6 +46,8 @@ def main(): # TODO(jeblair): Throw a fit if SSL is not used. parser.add_argument('project', help="The name of the project.") + parser.add_argument('--strip', action='store_true', default=False, + help="Strip whitespace from beginning/end of input.") parser.add_argument('--infile', default=None, help="A filename whose contents will be encrypted. " @@ -68,6 +70,8 @@ def main(): plaintext = sys.stdin.read() plaintext = plaintext.encode("utf-8") + if args.strip: + plaintext = plaintext.strip() pubkey_file = tempfile.NamedTemporaryFile(delete=False) try: diff --git a/tools/github-debugging.py b/tools/github-debugging.py index 171627ab9..101fd1118 100644..100755 --- a/tools/github-debugging.py +++ b/tools/github-debugging.py @@ -1,55 +1,71 @@ -import github3 +#!/usr/bin/env python3 + import logging -import time + +from zuul.driver.github.githubconnection import GithubConnection +from zuul.driver.github import GithubDriver +from zuul.model import Change, Project # This is a template with boilerplate code for debugging github issues # TODO: for real use override the following variables -url = 'https://example.com' +server = 'github.com' api_token = 'xxxx' -org = 'org' -project = 'project' -pull_nr = 3 +org = 'example' +repo = 'sandbox' +pull_nr = 8 + + +def configure_logging(context): + stream_handler = logging.StreamHandler() + logger = logging.getLogger(context) + logger.addHandler(stream_handler) + logger.setLevel(logging.DEBUG) + + +# uncomment for more logging +# configure_logging('urllib3') +# configure_logging('github3') +# configure_logging('cachecontrol') + + +# This is all that's needed for getting a usable github connection +def create_connection(server, api_token): + driver = GithubDriver() + connection_config = { + 'server': server, + 'api_token': api_token, + } + conn = GithubConnection(driver, 'github', connection_config) + conn._authenticateGithubAPI() + return conn -# Send the logs to stderr as well -stream_handler = logging.StreamHandler() +def get_change(connection: GithubConnection, + org: str, + repo: str, + pull: int) -> Change: + p = Project("%s/%s" % (org, repo), connection.source) + github = connection.getGithubClient(p) + pr = github.pull_request(org, repo, pull) + sha = pr.head.sha + return conn._getChange(p, pull, sha, True) -logger_urllib3 = logging.getLogger('requests.packages.logger_urllib3') -# logger_urllib3.addHandler(stream_handler) -logger_urllib3.setLevel(logging.DEBUG) -logger = logging.getLogger('github3') -# logger.addHandler(stream_handler) -logger.setLevel(logging.DEBUG) +# create github connection +conn = create_connection(server, api_token) -github = github3.GitHubEnterprise(url) +# Now we can do anything we want with the connection, e.g. check canMerge for +# a pull request. +change = get_change(conn, org, repo, pull_nr) +print(conn.canMerge(change, {'cc/gate2'})) -# This is the currently broken cache adapter, enable or replace it to debug -# caching -# import cachecontrol -# from cachecontrol.cache import DictCache -# cache_adapter = cachecontrol.CacheControlAdapter( -# DictCache(), -# cache_etags=True) +# Or just use the github object. +# github = conn.getGithubClient() # -# github.session.mount('http://', cache_adapter) -# github.session.mount('https://', cache_adapter) - - -github.login(token=api_token) - -i = 0 -while True: - pr = github.pull_request(org, project, pull_nr) - prdict = pr.as_dict() - issue = pr.issue() - labels = list(issue.labels()) - print(labels) - i += 1 - print(i) - time.sleep(1) +# repository = github.repository(org, repo) +# print(repository.as_dict()) diff --git a/zuul/cmd/__init__.py b/zuul/cmd/__init__.py index 236fd9f44..07d4a8d08 100755 --- a/zuul/cmd/__init__.py +++ b/zuul/cmd/__init__.py @@ -181,8 +181,9 @@ class ZuulDaemonApp(ZuulApp): else: # Exercise the pidfile before we do anything else (including # logging or daemonizing) - with daemon.DaemonContext(pidfile=pid): + with pid: pass + with daemon.DaemonContext(pidfile=pid): self.run() diff --git a/zuul/cmd/executor.py b/zuul/cmd/executor.py index ade9715c2..aa7a106b5 100755 --- a/zuul/cmd/executor.py +++ b/zuul/cmd/executor.py @@ -14,10 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. -import grp import logging import os -import pwd import sys import signal import tempfile @@ -51,9 +49,10 @@ class Executor(zuul.cmd.ZuulDaemonApp): if self.args.command: self.args.nodaemon = True - def exit_handler(self): + def exit_handler(self, signum, frame): self.executor.stop() self.executor.join() + sys.exit(0) def start_log_streamer(self): pipe_read, pipe_write = os.pipe() @@ -64,7 +63,7 @@ class Executor(zuul.cmd.ZuulDaemonApp): self.log.info("Starting log streamer") streamer = zuul.lib.log_streamer.LogStreamer( - self.user, '::', self.finger_port, self.job_dir) + '::', self.finger_port, self.job_dir) # Keep running until the parent dies: pipe_read = os.fdopen(pipe_read) @@ -76,22 +75,6 @@ class Executor(zuul.cmd.ZuulDaemonApp): os.close(pipe_read) self.log_streamer_pid = child_pid - def change_privs(self): - ''' - Drop our privileges to the zuul user. - ''' - if os.getuid() != 0: - return - pw = pwd.getpwnam(self.user) - # get a list of supplementary groups for the target user, and make sure - # we set them when dropping privileges. - groups = [g.gr_gid for g in grp.getgrall() if self.user in g.gr_mem] - os.setgroups(groups) - os.setgid(pw.pw_gid) - os.setuid(pw.pw_uid) - os.chdir(pw.pw_dir) - os.umask(0o022) - def run(self): if self.args.command in zuul.executor.server.COMMANDS: self.send_command(self.args.command) @@ -99,8 +82,6 @@ class Executor(zuul.cmd.ZuulDaemonApp): self.configure_connections(source_only=True) - self.user = get_default(self.config, 'executor', 'user', 'zuul') - if self.config.has_option('executor', 'job_dir'): self.job_dir = os.path.expanduser( self.config.get('executor', 'job_dir')) @@ -120,7 +101,6 @@ class Executor(zuul.cmd.ZuulDaemonApp): ) self.start_log_streamer() - self.change_privs() ExecutorServer = zuul.executor.server.ExecutorServer self.executor = ExecutorServer(self.config, self.connections, @@ -132,13 +112,13 @@ class Executor(zuul.cmd.ZuulDaemonApp): signal.signal(signal.SIGUSR2, zuul.cmd.stack_dump_handler) if self.args.nodaemon: + signal.signal(signal.SIGTERM, self.exit_handler) while True: try: signal.pause() except KeyboardInterrupt: print("Ctrl + C: asking executor to exit nicely...\n") - self.exit_handler() - sys.exit(0) + self.exit_handler(signal.SIGINT, None) else: self.executor.join() diff --git a/zuul/cmd/merger.py b/zuul/cmd/merger.py index 7db1beeaf..2916a0b3c 100755 --- a/zuul/cmd/merger.py +++ b/zuul/cmd/merger.py @@ -42,9 +42,10 @@ class Merger(zuul.cmd.ZuulDaemonApp): if self.args.command: self.args.nodaemon = True - def exit_handler(self): + def exit_handler(self, signum, frame): self.merger.stop() self.merger.join() + sys.exit(0) def run(self): # See comment at top of file about zuul imports @@ -64,13 +65,13 @@ class Merger(zuul.cmd.ZuulDaemonApp): signal.signal(signal.SIGUSR2, zuul.cmd.stack_dump_handler) if self.args.nodaemon: + signal.signal(signal.SIGTERM, self.exit_handler) while True: try: signal.pause() except KeyboardInterrupt: print("Ctrl + C: asking merger to exit nicely...\n") - self.exit_handler() - sys.exit(0) + self.exit_handler(signal.SIGINT, None) else: self.merger.join() diff --git a/zuul/cmd/scheduler.py b/zuul/cmd/scheduler.py index 7722d6e9c..d94e9ea17 100755 --- a/zuul/cmd/scheduler.py +++ b/zuul/cmd/scheduler.py @@ -61,14 +61,11 @@ class Scheduler(zuul.cmd.ZuulDaemonApp): self.log.exception("Reconfiguration failed:") signal.signal(signal.SIGHUP, self.reconfigure_handler) - def exit_handler(self): + def exit_handler(self, signum, frame): self.sched.exit() self.sched.join() self.stop_gear_server() - - def term_handler(self, signum, frame): - self.stop_gear_server() - os._exit(0) + sys.exit(0) def start_gear_server(self): pipe_read, pipe_write = os.pipe() @@ -180,13 +177,13 @@ class Scheduler(zuul.cmd.ZuulDaemonApp): signal.signal(signal.SIGHUP, self.reconfigure_handler) if self.args.nodaemon: + signal.signal(signal.SIGTERM, self.exit_handler) while True: try: signal.pause() except KeyboardInterrupt: print("Ctrl + C: asking scheduler to exit nicely...\n") - self.exit_handler() - sys.exit(0) + self.exit_handler(signal.SIGINT, None) else: self.sched.join() diff --git a/zuul/configloader.py b/zuul/configloader.py index 3a7e9b970..d62237043 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -474,6 +474,7 @@ class JobParser(object): # Attributes of a job that can also be used in Project and ProjectTemplate job_attributes = {'parent': vs.Any(str, None), 'final': bool, + 'protected': bool, 'failure-message': str, 'success-message': str, 'failure-url': str, @@ -513,6 +514,7 @@ class JobParser(object): simple_attributes = [ 'final', + 'protected', 'timeout', 'workspace', 'voting', diff --git a/zuul/driver/github/githubreporter.py b/zuul/driver/github/githubreporter.py index 505757fa2..848ae1b3a 100644 --- a/zuul/driver/github/githubreporter.py +++ b/zuul/driver/github/githubreporter.py @@ -75,6 +75,14 @@ class GithubReporter(BaseReporter): msg = self._formatItemReportMergeFailure(item) self.addPullComment(item, msg) + def _formatItemReportJobs(self, item): + # Return the list of jobs portion of the report + ret = '' + jobs_fields = self._getItemReportJobsFields(item) + for job_fields in jobs_fields: + ret += '- [%s](%s) : %s%s%s%s\n' % job_fields + return ret + def addPullComment(self, item, comment=None): message = comment or self._formatItemReport(item) project = item.change.project.name diff --git a/zuul/driver/sql/alembic.ini b/zuul/driver/sql/alembic.ini new file mode 100644 index 000000000..e94d496e1 --- /dev/null +++ b/zuul/driver/sql/alembic.ini @@ -0,0 +1,2 @@ +[alembic] +script_location = alembic diff --git a/zuul/driver/sql/alembic/versions/19d3a3ebfe1d_change_patchset_to_string.py b/zuul/driver/sql/alembic/versions/19d3a3ebfe1d_change_patchset_to_string.py new file mode 100644 index 000000000..505a1ed73 --- /dev/null +++ b/zuul/driver/sql/alembic/versions/19d3a3ebfe1d_change_patchset_to_string.py @@ -0,0 +1,29 @@ +"""Change patchset to string + +Revision ID: 19d3a3ebfe1d +Revises: cfc0dc45f341 +Create Date: 2018-01-10 07:42:16.546751 + +""" + +# revision identifiers, used by Alembic. +revision = '19d3a3ebfe1d' +down_revision = 'cfc0dc45f341' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + +BUILDSET_TABLE = 'zuul_buildset' + + +def upgrade(table_prefix=''): + op.alter_column(table_prefix + BUILDSET_TABLE, + 'patchset', + type_=sa.String(255), + existing_nullable=True) + + +def downgrade(): + raise Exception("Downgrades not supported") diff --git a/zuul/driver/sql/alembic/versions/cfc0dc45f341_change_patchset_to_string.py b/zuul/driver/sql/alembic/versions/cfc0dc45f341_change_patchset_to_string.py new file mode 100644 index 000000000..3fde8e545 --- /dev/null +++ b/zuul/driver/sql/alembic/versions/cfc0dc45f341_change_patchset_to_string.py @@ -0,0 +1,30 @@ +"""Change patchset to string + +Revision ID: cfc0dc45f341 +Revises: ba4cdce9b18c +Create Date: 2018-01-09 16:44:31.506958 + +""" + +# revision identifiers, used by Alembic. +revision = 'cfc0dc45f341' +down_revision = 'ba4cdce9b18c' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + +BUILDSET_TABLE = 'zuul_buildset' + + +def upgrade(table_prefix=''): + op.alter_column(table_prefix + BUILDSET_TABLE, + 'patchset', + sa.String(255), + existing_nullable=True, + existing_type=sa.Integer) + + +def downgrade(): + raise Exception("Downgrades not supported") diff --git a/zuul/driver/sql/sqlconnection.py b/zuul/driver/sql/sqlconnection.py index 285d0c23f..715d72bba 100644 --- a/zuul/driver/sql/sqlconnection.py +++ b/zuul/driver/sql/sqlconnection.py @@ -92,7 +92,7 @@ class SQLConnection(BaseConnection): sa.Column('pipeline', sa.String(255)), sa.Column('project', sa.String(255)), sa.Column('change', sa.Integer, nullable=True), - sa.Column('patchset', sa.Integer, nullable=True), + sa.Column('patchset', sa.String(255), nullable=True), sa.Column('ref', sa.String(255)), sa.Column('oldrev', sa.String(255)), sa.Column('newrev', sa.String(255)), diff --git a/zuul/executor/server.py b/zuul/executor/server.py index 5a710a62d..a8ab8c45e 100644 --- a/zuul/executor/server.py +++ b/zuul/executor/server.py @@ -44,7 +44,8 @@ from zuul.lib import commandsocket BUFFER_LINES_FOR_SYNTAX = 200 COMMANDS = ['stop', 'pause', 'unpause', 'graceful', 'verbose', 'unverbose', 'keep', 'nokeep'] -DEFAULT_FINGER_PORT = 79 +DEFAULT_FINGER_PORT = 7900 +BLACKLISTED_ANSIBLE_CONNECTION_TYPES = ['network_cli'] class StopException(Exception): @@ -347,6 +348,8 @@ class JobDir(object): pass self.known_hosts = os.path.join(ssh_dir, 'known_hosts') self.inventory = os.path.join(self.ansible_root, 'inventory.yaml') + self.setup_inventory = os.path.join(self.ansible_root, + 'setup-inventory.yaml') self.logging_json = os.path.join(self.ansible_root, 'logging.json') self.playbooks = [] # The list of candidate playbooks self.playbook = None # A pointer to the candidate we have chosen @@ -493,6 +496,26 @@ def _copy_ansible_files(python_module, target_dir): shutil.copy(os.path.join(library_path, fn), target_dir) +def make_setup_inventory_dict(nodes): + + hosts = {} + for node in nodes: + if (node['host_vars']['ansible_connection'] in + BLACKLISTED_ANSIBLE_CONNECTION_TYPES): + continue + + for name in node['name']: + hosts[name] = node['host_vars'] + + inventory = { + 'all': { + 'hosts': hosts, + } + } + + return inventory + + def make_inventory_dict(nodes, groups, all_vars): hosts = {} @@ -1157,8 +1180,13 @@ class AnsibleJob(object): result_data_file=self.jobdir.result_data_file) nodes = self.getHostList(args) + setup_inventory = make_setup_inventory_dict(nodes) inventory = make_inventory_dict(nodes, args['groups'], all_vars) + with open(self.jobdir.setup_inventory, 'w') as setup_inventory_yaml: + setup_inventory_yaml.write( + yaml.safe_dump(setup_inventory, default_flow_style=False)) + with open(self.jobdir.inventory, 'w') as inventory_yaml: inventory_yaml.write( yaml.safe_dump(inventory, default_flow_style=False)) @@ -1423,6 +1451,7 @@ class AnsibleJob(object): verbose = '-v' cmd = ['ansible', '*', verbose, '-m', 'setup', + '-i', self.jobdir.setup_inventory, '-a', 'gather_subset=!all'] result, code = self.runAnsible( diff --git a/zuul/lib/log_streamer.py b/zuul/lib/log_streamer.py index c778812a6..f96f44279 100644 --- a/zuul/lib/log_streamer.py +++ b/zuul/lib/log_streamer.py @@ -157,12 +157,11 @@ class LogStreamer(object): Class implementing log streaming over the finger daemon port. ''' - def __init__(self, user, host, port, jobdir_root): + def __init__(self, host, port, jobdir_root): self.log = logging.getLogger('zuul.log_streamer') self.log.debug("LogStreamer starting on port %s", port) self.server = LogStreamerServer((host, port), RequestHandler, - user=user, jobdir_root=jobdir_root) # We start the actual serving within a thread so we can return to diff --git a/zuul/lib/streamer_utils.py b/zuul/lib/streamer_utils.py index 43bc28626..3d2d561b9 100644 --- a/zuul/lib/streamer_utils.py +++ b/zuul/lib/streamer_utils.py @@ -74,7 +74,7 @@ class CustomThreadingTCPServer(socketserver.ThreadingTCPServer): address_family = socket.AF_INET6 def __init__(self, *args, **kwargs): - self.user = kwargs.pop('user') + self.user = kwargs.pop('user', None) self.pid_file = kwargs.pop('pid_file', None) socketserver.ThreadingTCPServer.__init__(self, *args, **kwargs) diff --git a/zuul/model.py b/zuul/model.py index bac9e4cc8..29c5a9d7e 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -845,6 +845,7 @@ class Job(object): semaphore=None, attempts=3, final=False, + protected=None, roles=(), required_projects={}, allowed_projects=None, @@ -862,6 +863,7 @@ class Job(object): inheritance_path=(), parent_data=None, description=None, + protected_origin=None, ) self.inheritable_attributes = {} @@ -1039,12 +1041,21 @@ class Job(object): for k in self.execution_attributes: if (other._get(k) is not None and - k not in set(['final'])): + k not in set(['final', 'protected'])): if self.final: raise Exception("Unable to modify final job %s attribute " "%s=%s with variant %s" % ( repr(self), k, other._get(k), repr(other))) + if self.protected_origin: + # this is a protected job, check origin of job definition + this_origin = self.protected_origin + other_origin = other.source_context.project.canonical_name + if this_origin != other_origin: + raise Exception("Job %s which is defined in %s is " + "protected and cannot be inherited " + "from other projects." + % (repr(self), this_origin)) if k not in set(['pre_run', 'run', 'post_run', 'roles', 'variables', 'required_projects']): # TODO(jeblair): determine if deepcopy is required @@ -1055,6 +1066,17 @@ class Job(object): if other.final != self.attributes['final']: self.final = other.final + # Protected may only be set to true + if other.protected is not None: + # don't allow to reset protected flag + if not other.protected and self.protected_origin: + raise Exception("Unable to reset protected attribute of job" + " %s by job %s" % ( + repr(self), repr(other))) + if not self.protected_origin: + self.protected_origin = \ + other.source_context.project.canonical_name + # We must update roles before any playbook contexts if other._get('roles') is not None: self.addRoles(other.roles) diff --git a/zuul/reporter/__init__.py b/zuul/reporter/__init__.py index ecf88553a..1bff5cb94 100644 --- a/zuul/reporter/__init__.py +++ b/zuul/reporter/__init__.py @@ -109,12 +109,10 @@ class BaseReporter(object, metaclass=abc.ABCMeta): else: return self._formatItemReport(item) - def _formatItemReportJobs(self, item): - # Return the list of jobs portion of the report - ret = '' - + def _getItemReportJobsFields(self, item): + # Extract the report elements from an item config = self.connection.sched.config - + jobs_fields = [] for job in item.getJobs(): build = item.current_build_set.getBuild(job.name) (result, url) = item.formatJobResult(job) @@ -147,6 +145,13 @@ class BaseReporter(object, metaclass=abc.ABCMeta): else: error = '' name = job.name + ' ' - ret += '- %s%s : %s%s%s%s\n' % (name, url, result, error, - elapsed, voting) + jobs_fields.append((name, url, result, error, elapsed, voting)) + return jobs_fields + + def _formatItemReportJobs(self, item): + # Return the list of jobs portion of the report + ret = '' + jobs_fields = self._getItemReportJobsFields(item) + for job_fields in jobs_fields: + ret += '- %s%s : %s%s%s%s\n' % job_fields return ret |