diff options
-rw-r--r-- | doc/source/admin/components.rst | 9 | ||||
-rw-r--r-- | requirements.txt | 1 | ||||
-rwxr-xr-x | tests/base.py | 2 | ||||
-rw-r--r-- | tests/fixtures/config/inventory/git/common-config/zuul.yaml | 2 | ||||
-rw-r--r-- | tests/unit/test_inventory.py | 26 | ||||
-rw-r--r-- | tests/unit/test_streaming.py | 16 | ||||
-rwxr-xr-x | zuul/cmd/__init__.py | 3 | ||||
-rwxr-xr-x | zuul/cmd/executor.py | 23 | ||||
-rw-r--r-- | zuul/driver/sql/alembic.ini | 2 | ||||
-rw-r--r-- | zuul/driver/sql/alembic/versions/cfc0dc45f341_change_patchset_to_string.py | 30 | ||||
-rw-r--r-- | zuul/executor/server.py | 31 | ||||
-rw-r--r-- | zuul/lib/log_streamer.py | 3 | ||||
-rw-r--r-- | zuul/lib/streamer_utils.py | 2 |
13 files changed, 107 insertions, 43 deletions
diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst index 3bec28afd..18bbfa3f4 100644 --- a/doc/source/admin/components.rst +++ b/doc/source/admin/components.rst @@ -408,7 +408,7 @@ The following sections of ``zuul.conf`` are used by the executor: Path to command socket file for the executor process. .. attr:: finger_port - :default: 79 + :default: 7900 Port to use for finger log streamer. @@ -451,13 +451,6 @@ The following sections of ``zuul.conf`` are used by the executor: SSH private key file to be used when logging into worker nodes. - .. attr:: user - :default: zuul - - User ID for the zuul-executor process. In normal operation as a - daemon, the executor should be started as the ``root`` user, but - it will drop privileges to this user during startup. - .. _admin_sitewide_variables: .. attr:: variables diff --git a/requirements.txt b/requirements.txt index 39a2b0268..193c64e71 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,5 +25,6 @@ cryptography>=1.6 cachecontrol pyjwt iso8601 +yarl>=0.11,<1.0 aiohttp uvloop;python_version>='3.5' diff --git a/tests/base.py b/tests/base.py index e688abd02..e2bfb4214 100755 --- a/tests/base.py +++ b/tests/base.py @@ -1618,6 +1618,8 @@ class FakeNodepool(object): data['username'] = 'fakeuser' if 'windows' in node_type: data['connection_type'] = 'winrm' + if 'network' in node_type: + data['connection_type'] = 'network_cli' data = json.dumps(data).encode('utf8') path = self.client.create(path, data, diff --git a/tests/fixtures/config/inventory/git/common-config/zuul.yaml b/tests/fixtures/config/inventory/git/common-config/zuul.yaml index 36789a321..f592eb48b 100644 --- a/tests/fixtures/config/inventory/git/common-config/zuul.yaml +++ b/tests/fixtures/config/inventory/git/common-config/zuul.yaml @@ -40,6 +40,8 @@ label: fakeuser-label - name: windows label: windows-label + - name: network + label: network-label - job: name: base diff --git a/tests/unit/test_inventory.py b/tests/unit/test_inventory.py index be504475a..b7e35ebd2 100644 --- a/tests/unit/test_inventory.py +++ b/tests/unit/test_inventory.py @@ -37,6 +37,12 @@ class TestInventory(ZuulTestCase): inv_path = os.path.join(build.jobdir.root, 'ansible', 'inventory.yaml') return yaml.safe_load(open(inv_path, 'r')) + def _get_setup_inventory(self, name): + build = self.getBuildByName(name) + setup_inv_path = os.path.join(build.jobdir.root, 'ansible', + 'setup-inventory.yaml') + return yaml.safe_load(open(setup_inv_path, 'r')) + def test_single_inventory(self): inventory = self._get_build_inventory('single-inventory') @@ -131,3 +137,23 @@ class TestInventory(ZuulTestCase): self.executor_server.release() self.waitUntilSettled() + + def test_setup_inventory(self): + + setup_inventory = self._get_setup_inventory('hostvars-inventory') + inventory = self._get_build_inventory('hostvars-inventory') + + self.assertIn('all', inventory) + self.assertIn('hosts', inventory['all']) + + self.assertIn('default', setup_inventory['all']['hosts']) + self.assertIn('fakeuser', setup_inventory['all']['hosts']) + self.assertIn('windows', setup_inventory['all']['hosts']) + self.assertNotIn('network', setup_inventory['all']['hosts']) + self.assertIn('default', inventory['all']['hosts']) + self.assertIn('fakeuser', inventory['all']['hosts']) + self.assertIn('windows', inventory['all']['hosts']) + self.assertIn('network', inventory['all']['hosts']) + + self.executor_server.release() + self.waitUntilSettled() diff --git a/tests/unit/test_streaming.py b/tests/unit/test_streaming.py index 59dd8b016..b999106c8 100644 --- a/tests/unit/test_streaming.py +++ b/tests/unit/test_streaming.py @@ -41,13 +41,13 @@ class TestLogStreamer(tests.base.BaseTestCase): def startStreamer(self, port, root=None): if not root: root = tempfile.gettempdir() - return zuul.lib.log_streamer.LogStreamer(None, self.host, port, root) + return zuul.lib.log_streamer.LogStreamer(self.host, port, root) def test_start_stop(self): - port = 7900 - streamer = self.startStreamer(port) + streamer = self.startStreamer(0) self.addCleanup(streamer.stop) + port = streamer.server.socket.getsockname()[1] s = socket.create_connection((self.host, port)) s.close() @@ -77,8 +77,9 @@ class TestStreaming(tests.base.AnsibleZuulTestCase): def startStreamer(self, port, build_uuid, root=None): if not root: root = tempfile.gettempdir() - self.streamer = zuul.lib.log_streamer.LogStreamer(None, self.host, + self.streamer = zuul.lib.log_streamer.LogStreamer(self.host, port, root) + port = self.streamer.server.socket.getsockname()[1] s = socket.create_connection((self.host, port)) self.addCleanup(s.close) @@ -129,10 +130,9 @@ class TestStreaming(tests.base.AnsibleZuulTestCase): # Create a thread to stream the log. We need this to be happening # before we create the flag file to tell the job to complete. - port = 7901 streamer_thread = threading.Thread( target=self.startStreamer, - args=(port, build.uuid, self.executor_server.jobdir_root,) + args=(0, build.uuid, self.executor_server.jobdir_root,) ) streamer_thread.start() self.addCleanup(self.stopStreamer) @@ -209,7 +209,7 @@ class TestStreaming(tests.base.AnsibleZuulTestCase): def test_websocket_streaming(self): # Start the finger streamer daemon streamer = zuul.lib.log_streamer.LogStreamer( - None, self.host, 0, self.executor_server.jobdir_root) + self.host, 0, self.executor_server.jobdir_root) self.addCleanup(streamer.stop) # Need to set the streaming port before submitting the job @@ -294,7 +294,7 @@ class TestStreaming(tests.base.AnsibleZuulTestCase): def test_finger_gateway(self): # Start the finger streamer daemon streamer = zuul.lib.log_streamer.LogStreamer( - None, self.host, 0, self.executor_server.jobdir_root) + self.host, 0, self.executor_server.jobdir_root) self.addCleanup(streamer.stop) finger_port = streamer.server.socket.getsockname()[1] diff --git a/zuul/cmd/__init__.py b/zuul/cmd/__init__.py index 236fd9f44..07d4a8d08 100755 --- a/zuul/cmd/__init__.py +++ b/zuul/cmd/__init__.py @@ -181,8 +181,9 @@ class ZuulDaemonApp(ZuulApp): else: # Exercise the pidfile before we do anything else (including # logging or daemonizing) - with daemon.DaemonContext(pidfile=pid): + with pid: pass + with daemon.DaemonContext(pidfile=pid): self.run() diff --git a/zuul/cmd/executor.py b/zuul/cmd/executor.py index ade9715c2..ad7aaa837 100755 --- a/zuul/cmd/executor.py +++ b/zuul/cmd/executor.py @@ -14,10 +14,8 @@ # License for the specific language governing permissions and limitations # under the License. -import grp import logging import os -import pwd import sys import signal import tempfile @@ -64,7 +62,7 @@ class Executor(zuul.cmd.ZuulDaemonApp): self.log.info("Starting log streamer") streamer = zuul.lib.log_streamer.LogStreamer( - self.user, '::', self.finger_port, self.job_dir) + '::', self.finger_port, self.job_dir) # Keep running until the parent dies: pipe_read = os.fdopen(pipe_read) @@ -76,22 +74,6 @@ class Executor(zuul.cmd.ZuulDaemonApp): os.close(pipe_read) self.log_streamer_pid = child_pid - def change_privs(self): - ''' - Drop our privileges to the zuul user. - ''' - if os.getuid() != 0: - return - pw = pwd.getpwnam(self.user) - # get a list of supplementary groups for the target user, and make sure - # we set them when dropping privileges. - groups = [g.gr_gid for g in grp.getgrall() if self.user in g.gr_mem] - os.setgroups(groups) - os.setgid(pw.pw_gid) - os.setuid(pw.pw_uid) - os.chdir(pw.pw_dir) - os.umask(0o022) - def run(self): if self.args.command in zuul.executor.server.COMMANDS: self.send_command(self.args.command) @@ -99,8 +81,6 @@ class Executor(zuul.cmd.ZuulDaemonApp): self.configure_connections(source_only=True) - self.user = get_default(self.config, 'executor', 'user', 'zuul') - if self.config.has_option('executor', 'job_dir'): self.job_dir = os.path.expanduser( self.config.get('executor', 'job_dir')) @@ -120,7 +100,6 @@ class Executor(zuul.cmd.ZuulDaemonApp): ) self.start_log_streamer() - self.change_privs() ExecutorServer = zuul.executor.server.ExecutorServer self.executor = ExecutorServer(self.config, self.connections, diff --git a/zuul/driver/sql/alembic.ini b/zuul/driver/sql/alembic.ini new file mode 100644 index 000000000..e94d496e1 --- /dev/null +++ b/zuul/driver/sql/alembic.ini @@ -0,0 +1,2 @@ +[alembic] +script_location = alembic diff --git a/zuul/driver/sql/alembic/versions/cfc0dc45f341_change_patchset_to_string.py b/zuul/driver/sql/alembic/versions/cfc0dc45f341_change_patchset_to_string.py new file mode 100644 index 000000000..3fde8e545 --- /dev/null +++ b/zuul/driver/sql/alembic/versions/cfc0dc45f341_change_patchset_to_string.py @@ -0,0 +1,30 @@ +"""Change patchset to string + +Revision ID: cfc0dc45f341 +Revises: ba4cdce9b18c +Create Date: 2018-01-09 16:44:31.506958 + +""" + +# revision identifiers, used by Alembic. +revision = 'cfc0dc45f341' +down_revision = 'ba4cdce9b18c' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + +BUILDSET_TABLE = 'zuul_buildset' + + +def upgrade(table_prefix=''): + op.alter_column(table_prefix + BUILDSET_TABLE, + 'patchset', + sa.String(255), + existing_nullable=True, + existing_type=sa.Integer) + + +def downgrade(): + raise Exception("Downgrades not supported") diff --git a/zuul/executor/server.py b/zuul/executor/server.py index 5a710a62d..a8ab8c45e 100644 --- a/zuul/executor/server.py +++ b/zuul/executor/server.py @@ -44,7 +44,8 @@ from zuul.lib import commandsocket BUFFER_LINES_FOR_SYNTAX = 200 COMMANDS = ['stop', 'pause', 'unpause', 'graceful', 'verbose', 'unverbose', 'keep', 'nokeep'] -DEFAULT_FINGER_PORT = 79 +DEFAULT_FINGER_PORT = 7900 +BLACKLISTED_ANSIBLE_CONNECTION_TYPES = ['network_cli'] class StopException(Exception): @@ -347,6 +348,8 @@ class JobDir(object): pass self.known_hosts = os.path.join(ssh_dir, 'known_hosts') self.inventory = os.path.join(self.ansible_root, 'inventory.yaml') + self.setup_inventory = os.path.join(self.ansible_root, + 'setup-inventory.yaml') self.logging_json = os.path.join(self.ansible_root, 'logging.json') self.playbooks = [] # The list of candidate playbooks self.playbook = None # A pointer to the candidate we have chosen @@ -493,6 +496,26 @@ def _copy_ansible_files(python_module, target_dir): shutil.copy(os.path.join(library_path, fn), target_dir) +def make_setup_inventory_dict(nodes): + + hosts = {} + for node in nodes: + if (node['host_vars']['ansible_connection'] in + BLACKLISTED_ANSIBLE_CONNECTION_TYPES): + continue + + for name in node['name']: + hosts[name] = node['host_vars'] + + inventory = { + 'all': { + 'hosts': hosts, + } + } + + return inventory + + def make_inventory_dict(nodes, groups, all_vars): hosts = {} @@ -1157,8 +1180,13 @@ class AnsibleJob(object): result_data_file=self.jobdir.result_data_file) nodes = self.getHostList(args) + setup_inventory = make_setup_inventory_dict(nodes) inventory = make_inventory_dict(nodes, args['groups'], all_vars) + with open(self.jobdir.setup_inventory, 'w') as setup_inventory_yaml: + setup_inventory_yaml.write( + yaml.safe_dump(setup_inventory, default_flow_style=False)) + with open(self.jobdir.inventory, 'w') as inventory_yaml: inventory_yaml.write( yaml.safe_dump(inventory, default_flow_style=False)) @@ -1423,6 +1451,7 @@ class AnsibleJob(object): verbose = '-v' cmd = ['ansible', '*', verbose, '-m', 'setup', + '-i', self.jobdir.setup_inventory, '-a', 'gather_subset=!all'] result, code = self.runAnsible( diff --git a/zuul/lib/log_streamer.py b/zuul/lib/log_streamer.py index c778812a6..f96f44279 100644 --- a/zuul/lib/log_streamer.py +++ b/zuul/lib/log_streamer.py @@ -157,12 +157,11 @@ class LogStreamer(object): Class implementing log streaming over the finger daemon port. ''' - def __init__(self, user, host, port, jobdir_root): + def __init__(self, host, port, jobdir_root): self.log = logging.getLogger('zuul.log_streamer') self.log.debug("LogStreamer starting on port %s", port) self.server = LogStreamerServer((host, port), RequestHandler, - user=user, jobdir_root=jobdir_root) # We start the actual serving within a thread so we can return to diff --git a/zuul/lib/streamer_utils.py b/zuul/lib/streamer_utils.py index 43bc28626..3d2d561b9 100644 --- a/zuul/lib/streamer_utils.py +++ b/zuul/lib/streamer_utils.py @@ -74,7 +74,7 @@ class CustomThreadingTCPServer(socketserver.ThreadingTCPServer): address_family = socket.AF_INET6 def __init__(self, *args, **kwargs): - self.user = kwargs.pop('user') + self.user = kwargs.pop('user', None) self.pid_file = kwargs.pop('pid_file', None) socketserver.ThreadingTCPServer.__init__(self, *args, **kwargs) |