summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--doc/source/admin/components.rst9
-rw-r--r--requirements.txt1
-rwxr-xr-xtests/base.py2
-rw-r--r--tests/fixtures/config/inventory/git/common-config/zuul.yaml2
-rw-r--r--tests/unit/test_inventory.py26
-rw-r--r--tests/unit/test_streaming.py16
-rwxr-xr-xzuul/cmd/__init__.py3
-rwxr-xr-xzuul/cmd/executor.py23
-rw-r--r--zuul/driver/sql/alembic.ini2
-rw-r--r--zuul/driver/sql/alembic/versions/cfc0dc45f341_change_patchset_to_string.py30
-rw-r--r--zuul/executor/server.py31
-rw-r--r--zuul/lib/log_streamer.py3
-rw-r--r--zuul/lib/streamer_utils.py2
13 files changed, 107 insertions, 43 deletions
diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst
index 3bec28afd..18bbfa3f4 100644
--- a/doc/source/admin/components.rst
+++ b/doc/source/admin/components.rst
@@ -408,7 +408,7 @@ The following sections of ``zuul.conf`` are used by the executor:
Path to command socket file for the executor process.
.. attr:: finger_port
- :default: 79
+ :default: 7900
Port to use for finger log streamer.
@@ -451,13 +451,6 @@ The following sections of ``zuul.conf`` are used by the executor:
SSH private key file to be used when logging into worker nodes.
- .. attr:: user
- :default: zuul
-
- User ID for the zuul-executor process. In normal operation as a
- daemon, the executor should be started as the ``root`` user, but
- it will drop privileges to this user during startup.
-
.. _admin_sitewide_variables:
.. attr:: variables
diff --git a/requirements.txt b/requirements.txt
index 39a2b0268..193c64e71 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -25,5 +25,6 @@ cryptography>=1.6
cachecontrol
pyjwt
iso8601
+yarl>=0.11,<1.0
aiohttp
uvloop;python_version>='3.5'
diff --git a/tests/base.py b/tests/base.py
index 64f66579c..c4492426f 100755
--- a/tests/base.py
+++ b/tests/base.py
@@ -1619,6 +1619,8 @@ class FakeNodepool(object):
data['username'] = 'fakeuser'
if 'windows' in node_type:
data['connection_type'] = 'winrm'
+ if 'network' in node_type:
+ data['connection_type'] = 'network_cli'
data = json.dumps(data).encode('utf8')
path = self.client.create(path, data,
diff --git a/tests/fixtures/config/inventory/git/common-config/zuul.yaml b/tests/fixtures/config/inventory/git/common-config/zuul.yaml
index 36789a321..f592eb48b 100644
--- a/tests/fixtures/config/inventory/git/common-config/zuul.yaml
+++ b/tests/fixtures/config/inventory/git/common-config/zuul.yaml
@@ -40,6 +40,8 @@
label: fakeuser-label
- name: windows
label: windows-label
+ - name: network
+ label: network-label
- job:
name: base
diff --git a/tests/unit/test_inventory.py b/tests/unit/test_inventory.py
index be504475a..b7e35ebd2 100644
--- a/tests/unit/test_inventory.py
+++ b/tests/unit/test_inventory.py
@@ -37,6 +37,12 @@ class TestInventory(ZuulTestCase):
inv_path = os.path.join(build.jobdir.root, 'ansible', 'inventory.yaml')
return yaml.safe_load(open(inv_path, 'r'))
+ def _get_setup_inventory(self, name):
+ build = self.getBuildByName(name)
+ setup_inv_path = os.path.join(build.jobdir.root, 'ansible',
+ 'setup-inventory.yaml')
+ return yaml.safe_load(open(setup_inv_path, 'r'))
+
def test_single_inventory(self):
inventory = self._get_build_inventory('single-inventory')
@@ -131,3 +137,23 @@ class TestInventory(ZuulTestCase):
self.executor_server.release()
self.waitUntilSettled()
+
+ def test_setup_inventory(self):
+
+ setup_inventory = self._get_setup_inventory('hostvars-inventory')
+ inventory = self._get_build_inventory('hostvars-inventory')
+
+ self.assertIn('all', inventory)
+ self.assertIn('hosts', inventory['all'])
+
+ self.assertIn('default', setup_inventory['all']['hosts'])
+ self.assertIn('fakeuser', setup_inventory['all']['hosts'])
+ self.assertIn('windows', setup_inventory['all']['hosts'])
+ self.assertNotIn('network', setup_inventory['all']['hosts'])
+ self.assertIn('default', inventory['all']['hosts'])
+ self.assertIn('fakeuser', inventory['all']['hosts'])
+ self.assertIn('windows', inventory['all']['hosts'])
+ self.assertIn('network', inventory['all']['hosts'])
+
+ self.executor_server.release()
+ self.waitUntilSettled()
diff --git a/tests/unit/test_streaming.py b/tests/unit/test_streaming.py
index 59dd8b016..b999106c8 100644
--- a/tests/unit/test_streaming.py
+++ b/tests/unit/test_streaming.py
@@ -41,13 +41,13 @@ class TestLogStreamer(tests.base.BaseTestCase):
def startStreamer(self, port, root=None):
if not root:
root = tempfile.gettempdir()
- return zuul.lib.log_streamer.LogStreamer(None, self.host, port, root)
+ return zuul.lib.log_streamer.LogStreamer(self.host, port, root)
def test_start_stop(self):
- port = 7900
- streamer = self.startStreamer(port)
+ streamer = self.startStreamer(0)
self.addCleanup(streamer.stop)
+ port = streamer.server.socket.getsockname()[1]
s = socket.create_connection((self.host, port))
s.close()
@@ -77,8 +77,9 @@ class TestStreaming(tests.base.AnsibleZuulTestCase):
def startStreamer(self, port, build_uuid, root=None):
if not root:
root = tempfile.gettempdir()
- self.streamer = zuul.lib.log_streamer.LogStreamer(None, self.host,
+ self.streamer = zuul.lib.log_streamer.LogStreamer(self.host,
port, root)
+ port = self.streamer.server.socket.getsockname()[1]
s = socket.create_connection((self.host, port))
self.addCleanup(s.close)
@@ -129,10 +130,9 @@ class TestStreaming(tests.base.AnsibleZuulTestCase):
# Create a thread to stream the log. We need this to be happening
# before we create the flag file to tell the job to complete.
- port = 7901
streamer_thread = threading.Thread(
target=self.startStreamer,
- args=(port, build.uuid, self.executor_server.jobdir_root,)
+ args=(0, build.uuid, self.executor_server.jobdir_root,)
)
streamer_thread.start()
self.addCleanup(self.stopStreamer)
@@ -209,7 +209,7 @@ class TestStreaming(tests.base.AnsibleZuulTestCase):
def test_websocket_streaming(self):
# Start the finger streamer daemon
streamer = zuul.lib.log_streamer.LogStreamer(
- None, self.host, 0, self.executor_server.jobdir_root)
+ self.host, 0, self.executor_server.jobdir_root)
self.addCleanup(streamer.stop)
# Need to set the streaming port before submitting the job
@@ -294,7 +294,7 @@ class TestStreaming(tests.base.AnsibleZuulTestCase):
def test_finger_gateway(self):
# Start the finger streamer daemon
streamer = zuul.lib.log_streamer.LogStreamer(
- None, self.host, 0, self.executor_server.jobdir_root)
+ self.host, 0, self.executor_server.jobdir_root)
self.addCleanup(streamer.stop)
finger_port = streamer.server.socket.getsockname()[1]
diff --git a/zuul/cmd/__init__.py b/zuul/cmd/__init__.py
index 236fd9f44..07d4a8d08 100755
--- a/zuul/cmd/__init__.py
+++ b/zuul/cmd/__init__.py
@@ -181,8 +181,9 @@ class ZuulDaemonApp(ZuulApp):
else:
# Exercise the pidfile before we do anything else (including
# logging or daemonizing)
- with daemon.DaemonContext(pidfile=pid):
+ with pid:
pass
+
with daemon.DaemonContext(pidfile=pid):
self.run()
diff --git a/zuul/cmd/executor.py b/zuul/cmd/executor.py
index ade9715c2..ad7aaa837 100755
--- a/zuul/cmd/executor.py
+++ b/zuul/cmd/executor.py
@@ -14,10 +14,8 @@
# License for the specific language governing permissions and limitations
# under the License.
-import grp
import logging
import os
-import pwd
import sys
import signal
import tempfile
@@ -64,7 +62,7 @@ class Executor(zuul.cmd.ZuulDaemonApp):
self.log.info("Starting log streamer")
streamer = zuul.lib.log_streamer.LogStreamer(
- self.user, '::', self.finger_port, self.job_dir)
+ '::', self.finger_port, self.job_dir)
# Keep running until the parent dies:
pipe_read = os.fdopen(pipe_read)
@@ -76,22 +74,6 @@ class Executor(zuul.cmd.ZuulDaemonApp):
os.close(pipe_read)
self.log_streamer_pid = child_pid
- def change_privs(self):
- '''
- Drop our privileges to the zuul user.
- '''
- if os.getuid() != 0:
- return
- pw = pwd.getpwnam(self.user)
- # get a list of supplementary groups for the target user, and make sure
- # we set them when dropping privileges.
- groups = [g.gr_gid for g in grp.getgrall() if self.user in g.gr_mem]
- os.setgroups(groups)
- os.setgid(pw.pw_gid)
- os.setuid(pw.pw_uid)
- os.chdir(pw.pw_dir)
- os.umask(0o022)
-
def run(self):
if self.args.command in zuul.executor.server.COMMANDS:
self.send_command(self.args.command)
@@ -99,8 +81,6 @@ class Executor(zuul.cmd.ZuulDaemonApp):
self.configure_connections(source_only=True)
- self.user = get_default(self.config, 'executor', 'user', 'zuul')
-
if self.config.has_option('executor', 'job_dir'):
self.job_dir = os.path.expanduser(
self.config.get('executor', 'job_dir'))
@@ -120,7 +100,6 @@ class Executor(zuul.cmd.ZuulDaemonApp):
)
self.start_log_streamer()
- self.change_privs()
ExecutorServer = zuul.executor.server.ExecutorServer
self.executor = ExecutorServer(self.config, self.connections,
diff --git a/zuul/driver/sql/alembic.ini b/zuul/driver/sql/alembic.ini
new file mode 100644
index 000000000..e94d496e1
--- /dev/null
+++ b/zuul/driver/sql/alembic.ini
@@ -0,0 +1,2 @@
+[alembic]
+script_location = alembic
diff --git a/zuul/driver/sql/alembic/versions/cfc0dc45f341_change_patchset_to_string.py b/zuul/driver/sql/alembic/versions/cfc0dc45f341_change_patchset_to_string.py
new file mode 100644
index 000000000..3fde8e545
--- /dev/null
+++ b/zuul/driver/sql/alembic/versions/cfc0dc45f341_change_patchset_to_string.py
@@ -0,0 +1,30 @@
+"""Change patchset to string
+
+Revision ID: cfc0dc45f341
+Revises: ba4cdce9b18c
+Create Date: 2018-01-09 16:44:31.506958
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'cfc0dc45f341'
+down_revision = 'ba4cdce9b18c'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+BUILDSET_TABLE = 'zuul_buildset'
+
+
+def upgrade(table_prefix=''):
+ op.alter_column(table_prefix + BUILDSET_TABLE,
+ 'patchset',
+ sa.String(255),
+ existing_nullable=True,
+ existing_type=sa.Integer)
+
+
+def downgrade():
+ raise Exception("Downgrades not supported")
diff --git a/zuul/executor/server.py b/zuul/executor/server.py
index 5a710a62d..a8ab8c45e 100644
--- a/zuul/executor/server.py
+++ b/zuul/executor/server.py
@@ -44,7 +44,8 @@ from zuul.lib import commandsocket
BUFFER_LINES_FOR_SYNTAX = 200
COMMANDS = ['stop', 'pause', 'unpause', 'graceful', 'verbose',
'unverbose', 'keep', 'nokeep']
-DEFAULT_FINGER_PORT = 79
+DEFAULT_FINGER_PORT = 7900
+BLACKLISTED_ANSIBLE_CONNECTION_TYPES = ['network_cli']
class StopException(Exception):
@@ -347,6 +348,8 @@ class JobDir(object):
pass
self.known_hosts = os.path.join(ssh_dir, 'known_hosts')
self.inventory = os.path.join(self.ansible_root, 'inventory.yaml')
+ self.setup_inventory = os.path.join(self.ansible_root,
+ 'setup-inventory.yaml')
self.logging_json = os.path.join(self.ansible_root, 'logging.json')
self.playbooks = [] # The list of candidate playbooks
self.playbook = None # A pointer to the candidate we have chosen
@@ -493,6 +496,26 @@ def _copy_ansible_files(python_module, target_dir):
shutil.copy(os.path.join(library_path, fn), target_dir)
+def make_setup_inventory_dict(nodes):
+
+ hosts = {}
+ for node in nodes:
+ if (node['host_vars']['ansible_connection'] in
+ BLACKLISTED_ANSIBLE_CONNECTION_TYPES):
+ continue
+
+ for name in node['name']:
+ hosts[name] = node['host_vars']
+
+ inventory = {
+ 'all': {
+ 'hosts': hosts,
+ }
+ }
+
+ return inventory
+
+
def make_inventory_dict(nodes, groups, all_vars):
hosts = {}
@@ -1157,8 +1180,13 @@ class AnsibleJob(object):
result_data_file=self.jobdir.result_data_file)
nodes = self.getHostList(args)
+ setup_inventory = make_setup_inventory_dict(nodes)
inventory = make_inventory_dict(nodes, args['groups'], all_vars)
+ with open(self.jobdir.setup_inventory, 'w') as setup_inventory_yaml:
+ setup_inventory_yaml.write(
+ yaml.safe_dump(setup_inventory, default_flow_style=False))
+
with open(self.jobdir.inventory, 'w') as inventory_yaml:
inventory_yaml.write(
yaml.safe_dump(inventory, default_flow_style=False))
@@ -1423,6 +1451,7 @@ class AnsibleJob(object):
verbose = '-v'
cmd = ['ansible', '*', verbose, '-m', 'setup',
+ '-i', self.jobdir.setup_inventory,
'-a', 'gather_subset=!all']
result, code = self.runAnsible(
diff --git a/zuul/lib/log_streamer.py b/zuul/lib/log_streamer.py
index c778812a6..f96f44279 100644
--- a/zuul/lib/log_streamer.py
+++ b/zuul/lib/log_streamer.py
@@ -157,12 +157,11 @@ class LogStreamer(object):
Class implementing log streaming over the finger daemon port.
'''
- def __init__(self, user, host, port, jobdir_root):
+ def __init__(self, host, port, jobdir_root):
self.log = logging.getLogger('zuul.log_streamer')
self.log.debug("LogStreamer starting on port %s", port)
self.server = LogStreamerServer((host, port),
RequestHandler,
- user=user,
jobdir_root=jobdir_root)
# We start the actual serving within a thread so we can return to
diff --git a/zuul/lib/streamer_utils.py b/zuul/lib/streamer_utils.py
index 43bc28626..3d2d561b9 100644
--- a/zuul/lib/streamer_utils.py
+++ b/zuul/lib/streamer_utils.py
@@ -74,7 +74,7 @@ class CustomThreadingTCPServer(socketserver.ThreadingTCPServer):
address_family = socket.AF_INET6
def __init__(self, *args, **kwargs):
- self.user = kwargs.pop('user')
+ self.user = kwargs.pop('user', None)
self.pid_file = kwargs.pop('pid_file', None)
socketserver.ThreadingTCPServer.__init__(self, *args, **kwargs)
View Lorry Controller Status