diff options
Diffstat (limited to 'zuul/configloader.py')
| -rw-r--r-- | zuul/configloader.py | 80 |
1 files changed, 74 insertions, 6 deletions
diff --git a/zuul/configloader.py b/zuul/configloader.py index 67d4494c4..fe22fe0f8 100644 --- a/zuul/configloader.py +++ b/zuul/configloader.py @@ -437,6 +437,30 @@ def ansible_vars_dict(value): ansible_var_name(key) +def copy_safe_config(conf): + """Return a deep copy of a config dictionary. + + This lets us assign values of a config dictionary to configuration + objects, even if those values are nested dictionaries. This way + we can safely freeze the configuration object (the process of + which mutates dictionaries) without mutating the original + configuration. + + Meanwhile, this does retain the original context information as a + single object (some behaviors rely on mutating the source context + (e.g., pragma)). + + """ + ret = copy.deepcopy(conf) + for key in ( + '_source_context', + '_start_mark', + ): + if key in conf: + ret[key] = conf[key] + return ret + + class PragmaParser(object): pragma = { 'implied-branch-matchers': bool, @@ -452,6 +476,7 @@ class PragmaParser(object): self.pcontext = pcontext def fromYaml(self, conf): + conf = copy_safe_config(conf) self.schema(conf) bm = conf.get('implied-branch-matchers') @@ -512,6 +537,7 @@ class NodeSetParser(object): return vs.Schema(nodeset) def fromYaml(self, conf, anonymous=False): + conf = copy_safe_config(conf) if anonymous: self.anon_schema(conf) self.anonymous = True @@ -599,6 +625,7 @@ class SecretParser(object): return vs.Schema(secret) def fromYaml(self, conf): + conf = copy_safe_config(conf) self.schema(conf) s = model.Secret(conf['name'], conf['_source_context']) s.source_context = conf['_source_context'] @@ -723,6 +750,7 @@ class JobParser(object): def fromYaml(self, conf, project_pipeline=False, name=None, validate=True): + conf = copy_safe_config(conf) if validate: self.schema(conf) @@ -1075,6 +1103,7 @@ class ProjectTemplateParser(object): return vs.Schema(project) def fromYaml(self, conf, validate=True, freeze=True): + conf = copy_safe_config(conf) if validate: self.schema(conf) source_context = conf['_source_context'] @@ -1165,6 +1194,7 @@ class ProjectParser(object): return vs.Schema(project) def fromYaml(self, conf): + conf = copy_safe_config(conf) self.schema(conf) project_name = conf.get('name') @@ -1292,6 +1322,7 @@ class PipelineParser(object): pipeline = {vs.Required('name'): str, vs.Required('manager'): manager, + 'allow-other-connections': bool, 'precedence': precedence, 'supercedes': to_list(str), 'description': str, @@ -1327,10 +1358,13 @@ class PipelineParser(object): return vs.Schema(pipeline) def fromYaml(self, conf): + conf = copy_safe_config(conf) self.schema(conf) pipeline = model.Pipeline(conf['name'], self.pcontext.tenant) pipeline.source_context = conf['_source_context'] pipeline.start_mark = conf['_start_mark'] + pipeline.allow_other_connections = conf.get( + 'allow-other-connections', True) pipeline.description = conf.get('description') pipeline.supercedes = as_list(conf.get('supercedes', [])) @@ -1366,6 +1400,7 @@ class PipelineParser(object): # Make a copy to manipulate for backwards compat. conf_copy = conf.copy() + seen_connections = set() for conf_key, action in self.reporter_actions.items(): reporter_set = [] allowed_reporters = self.pcontext.tenant.allowed_reporters @@ -1379,6 +1414,7 @@ class PipelineParser(object): reporter_name, pipeline, params) reporter.setAction(conf_key) reporter_set.append(reporter) + seen_connections.add(reporter_name) setattr(pipeline, action, reporter_set) # If merge-conflict actions aren't explicit, use the failure actions @@ -1423,11 +1459,13 @@ class PipelineParser(object): source = self.pcontext.connections.getSource(source_name) manager.ref_filters.extend( source.getRequireFilters(require_config)) + seen_connections.add(source_name) for source_name, reject_config in conf.get('reject', {}).items(): source = self.pcontext.connections.getSource(source_name) manager.ref_filters.extend( source.getRejectFilters(reject_config)) + seen_connections.add(source_name) for connection_name, trigger_config in conf.get('trigger').items(): if self.pcontext.tenant.allowed_triggers is not None and \ @@ -1439,7 +1477,9 @@ class PipelineParser(object): manager.event_filters.extend( trigger.getEventFilters(connection_name, conf['trigger'][connection_name])) + seen_connections.add(connection_name) + pipeline.connections = list(seen_connections) # Pipelines don't get frozen return pipeline @@ -1460,6 +1500,7 @@ class SemaphoreParser(object): return vs.Schema(semaphore) def fromYaml(self, conf): + conf = copy_safe_config(conf) self.schema(conf) semaphore = model.Semaphore(conf['name'], conf.get('max', 1)) semaphore.source_context = conf.get('_source_context') @@ -1485,6 +1526,7 @@ class QueueParser: return vs.Schema(queue) def fromYaml(self, conf): + conf = copy_safe_config(conf) self.schema(conf) queue = model.Queue( conf['name'], @@ -1514,6 +1556,7 @@ class AuthorizationRuleParser(object): return vs.Schema(authRule) def fromYaml(self, conf): + conf = copy_safe_config(conf) self.schema(conf) a = model.AuthZRuleTree(conf['name']) @@ -1547,6 +1590,7 @@ class GlobalSemaphoreParser(object): return vs.Schema(semaphore) def fromYaml(self, conf): + conf = copy_safe_config(conf) self.schema(conf) semaphore = model.Semaphore(conf['name'], conf.get('max', 1), global_scope=True) @@ -1567,6 +1611,7 @@ class ApiRootParser(object): return vs.Schema(api_root) def fromYaml(self, conf): + conf = copy_safe_config(conf) self.schema(conf) api_root = model.ApiRoot(conf.get('authentication-realm')) api_root.access_rules = conf.get('access-rules', []) @@ -1688,7 +1733,7 @@ class TenantParser(object): 'disallowed-labels': to_list(str), 'allow-circular-dependencies': bool, 'default-parent': str, - 'default-ansible-version': vs.Any(str, float), + 'default-ansible-version': vs.Any(str, float, int), 'access-rules': to_list(str), 'admin-rules': to_list(str), 'semaphores': to_list(str), @@ -1761,8 +1806,10 @@ class TenantParser(object): for branch_future in as_completed(branch_futures.keys()): tpc = branch_futures[branch_future] - source_context = model.ProjectContext( - tpc.project.canonical_name, tpc.project.name) + trusted, _ = tenant.getProject(tpc.project.canonical_name) + source_context = model.SourceContext( + tpc.project.canonical_name, tpc.project.name, + tpc.project.connection_name, None, None, trusted) with project_configuration_exceptions(source_context, loading_errors): self._getProjectBranches(tenant, tpc, branch_cache_min_ltimes) @@ -1771,8 +1818,8 @@ class TenantParser(object): # Set default ansible version default_ansible_version = conf.get('default-ansible-version') if default_ansible_version is not None: - # The ansible version can be interpreted as float by yaml so make - # sure it's a string. + # The ansible version can be interpreted as float or int + # by yaml so make sure it's a string. default_ansible_version = str(default_ansible_version) ansible_manager.requestVersion(default_ansible_version) else: @@ -1858,6 +1905,9 @@ class TenantParser(object): tpc.branches = static_branches tpc.dynamic_branches = always_dynamic_branches + tpc.merge_modes = tpc.project.source.getProjectMergeModes( + tpc.project, tenant, min_ltime) + def _loadProjectKeys(self, connection_name, project): project.private_secrets_key, project.public_secrets_key = ( self.keystorage.getProjectSecretsKeys( @@ -2209,6 +2259,12 @@ class TenantParser(object): job.source_context.branch) with self.unparsed_config_cache.writeLock( job.source_context.project_canonical_name): + # Prevent files cache ltime from going backward + if files_cache.ltime >= job.ltime: + self.log.info( + "Discarding job %s result since the files cache was " + "updated in the meantime", job) + continue # Since the cat job returns all required config files # for ALL tenants the project is a part of, we can # clear the whole cache and then populate it with the @@ -2577,11 +2633,23 @@ class TenantParser(object): # Set a merge mode if we don't have one for this project. # This can happen if there are only regex project stanzas # but no specific project stanzas. + (trusted, project) = tenant.getProject(project_name) project_metadata = layout.getProjectMetadata(project_name) if project_metadata.merge_mode is None: - (trusted, project) = tenant.getProject(project_name) mode = project.source.getProjectDefaultMergeMode(project) project_metadata.merge_mode = model.MERGER_MAP[mode] + tpc = tenant.project_configs[project.canonical_name] + if tpc.merge_modes is not None: + source_context = model.SourceContext( + project.canonical_name, project.name, + project.connection_name, None, None, trusted) + with project_configuration_exceptions(source_context, + layout.loading_errors): + if project_metadata.merge_mode not in tpc.merge_modes: + mode = model.get_merge_mode_name( + project_metadata.merge_mode) + raise Exception(f'Merge mode {mode} not supported ' + f'by project {project_name}') def _parseLayout(self, tenant, data, loading_errors, layout_uuid=None): # Don't call this method from dynamic reconfiguration because |