diff options
Diffstat (limited to 'chromium/infra/config/lib')
| -rw-r--r-- | chromium/infra/config/lib/args.star | 4 | ||||
| -rw-r--r-- | chromium/infra/config/lib/builders.star | 43 | ||||
| -rw-r--r-- | chromium/infra/config/lib/ci.star | 131 | ||||
| -rw-r--r-- | chromium/infra/config/lib/try.star | 24 |
4 files changed, 140 insertions, 62 deletions
diff --git a/chromium/infra/config/lib/args.star b/chromium/infra/config/lib/args.star index b6d9624dd79..f419c3b189a 100644 --- a/chromium/infra/config/lib/args.star +++ b/chromium/infra/config/lib/args.star @@ -1,3 +1,7 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Library containing utilities for handling args in libraries.""" def _sentinel(tag): diff --git a/chromium/infra/config/lib/builders.star b/chromium/infra/config/lib/builders.star index 6cb88920d00..5997c1c40c1 100644 --- a/chromium/infra/config/lib/builders.star +++ b/chromium/infra/config/lib/builders.star @@ -1,3 +1,7 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Library for defining builders. The `builder` function defined in this module simplifies setting all of the @@ -129,11 +133,14 @@ def xcode_enum(cache_name, cache_path): xcode_cache = struct( x10e1001 = xcode_enum('xcode_ios_10e1001', 'xcode_ios_10e1001.app'), x11a1027 = xcode_enum('xcode_ios_11a1027', 'xcode_ios_11a1027.app'), - x11c505wk = xcode_enum('xcode_ios_11c505wk', 'xcode_ios_11c505wk.app'), x11c29 = xcode_enum('xcode_ios_11c29', 'xcode_ios_11c29.app'), x11m382q = xcode_enum('xcode_ios_11m382q', 'xcode_ios_11m382q.app'), x11e146 = xcode_enum('xcode_ios_11e146', 'xcode_ios_11e146.app'), - x11n605cwk = xcode_enum('xcode_ios_11n605cwk', 'xcode_ios_11n605cwk.app'), + x11e608c = xcode_enum('xcode_ios_11e608c', 'xcode_ios_11e608c.app'), + x11e608cwk = xcode_enum('xcode_ios_11e608cwk', 'xcode_ios_11e608cwk.app'), + x11n700h = xcode_enum('xcode_ios_11n700h', 'xcode_ios_11n700h.app'), + # xcode12 + x12a6159 = xcode_enum('xcode_ios_12a6159', 'xcode_ios_12a6159.app'), ) @@ -182,7 +189,12 @@ def _goma_property(*, goma_backend, goma_debug, goma_enable_ats, goma_jobs, os): return goma_properties or None -def _code_coverage_property(*, use_clang_coverage, use_java_coverage): +def _code_coverage_property( + *, + use_clang_coverage, + use_java_coverage, + coverage_exclude_sources, + coverage_test_types): code_coverage = {} use_clang_coverage = defaults.get_value( @@ -194,6 +206,16 @@ def _code_coverage_property(*, use_clang_coverage, use_java_coverage): if use_java_coverage: code_coverage['use_java_coverage'] = True + coverage_exclude_sources = defaults.get_value('coverage_exclude_sources', + coverage_exclude_sources) + if coverage_exclude_sources: + code_coverage['coverage_exclude_sources'] = coverage_exclude_sources + + coverage_test_types = defaults.get_value('coverage_test_types', + coverage_test_types) + if coverage_test_types: + code_coverage['coverage_test_types'] = coverage_test_types + return code_coverage or None @@ -222,6 +244,8 @@ defaults = args.defaults( ssd = args.COMPUTE, use_clang_coverage = False, use_java_coverage = False, + coverage_exclude_sources = None, + coverage_test_types = None, resultdb_bigquery_exports = [], # Provide vars for bucket and executable so users don't have to @@ -256,6 +280,8 @@ def builder( goma_jobs=args.DEFAULT, use_clang_coverage=args.DEFAULT, use_java_coverage=args.DEFAULT, + coverage_exclude_sources=args.DEFAULT, + coverage_test_types=args.DEFAULT, resultdb_bigquery_exports=args.DEFAULT, **kwargs): """Define a builder. @@ -330,6 +356,12 @@ def builder( * use_java_coverage - a boolean indicating whether java coverage should be used. If True, the 'use_java_coverage" field will be set in the '$build/code_coverage' property. By default, considered False. + * coverage_exclude_sources - a string as the key to find the source file + exclusion pattern in code_coverage recipe module. Will be copied to + '$build/code_coverage' property if set. By default, considered None. + * coverage_test_types - a list of string as test types to process data for + in code_coverage recipe module. Will be copied to '$build/code_coverage' + property. By default, considered None. * resultdb_bigquery_exports - a list of resultdb.export_test_results(...) specifying parameters for exporting test results to BigQuery. By default, do not export. @@ -352,7 +384,8 @@ def builder( + 'use goma_backend, goma_dbug, goma_enable_ats and goma_jobs instead') if '$build/code_coverage' in properties: fail('Setting "$build/code_coverage" property is not supported: ' - + 'use use_clang_coverage and use_java_coverage instead') + + 'use use_clang_coverage, use_java_coverage, coverage_exclude_sources' + + ' and/or coverage_test_types instead') properties = dict(properties) os = defaults.get_value('os', os) @@ -420,6 +453,8 @@ def builder( code_coverage = _code_coverage_property( use_clang_coverage = use_clang_coverage, use_java_coverage = use_java_coverage, + coverage_exclude_sources = coverage_exclude_sources, + coverage_test_types = coverage_test_types, ) if code_coverage != None: properties['$build/code_coverage'] = code_coverage diff --git a/chromium/infra/config/lib/ci.star b/chromium/infra/config/lib/ci.star index e164998e249..98e8c973e0c 100644 --- a/chromium/infra/config/lib/ci.star +++ b/chromium/infra/config/lib/ci.star @@ -1,3 +1,7 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Library for defining CI builders. The `ci_builder` function defined in this module enables defining a CI builder. @@ -21,6 +25,7 @@ defaults = args.defaults( console_view = args.COMPUTE, header = None, main_console_view = None, + cq_mirrors_console_view = None, repo = None, ) @@ -51,31 +56,36 @@ def declare_bucket(milestone_vars): refs = [milestone_vars.ref], ) - ci.main_console_view( - name = milestone_vars.main_console_name, - header = '//chromium-header.textpb', - repo = 'https://chromium.googlesource.com/chromium/src', - refs = [milestone_vars.ref], - title = milestone_vars.main_console_title, - top_level_ordering = [ - 'chromium', - 'chromium.win', - 'chromium.mac', - 'chromium.linux', - 'chromium.chromiumos', - 'chromium.android', - 'chrome', - 'chromium.memory', - 'chromium.dawn', - 'chromium.gpu', - 'chromium.fyi', - 'chromium.android.fyi', - 'chromium.clang', - 'chromium.fuzz', - 'chromium.gpu.fyi', - 'chromium.swangle', - ], - ) + # TODO(gbeaty) Determine what should be in each main console and define it + # separately + for name, title in ( + (milestone_vars.main_console_name, milestone_vars.main_console_title), + (milestone_vars.cq_mirrors_console_name, milestone_vars.cq_mirrors_console_title)): + ci.overview_console_view( + name = name, + header = '//chromium-header.textpb', + repo = 'https://chromium.googlesource.com/chromium/src', + refs = [milestone_vars.ref], + title = title, + top_level_ordering = [ + 'chromium', + 'chromium.win', + 'chromium.mac', + 'chromium.linux', + 'chromium.chromiumos', + 'chromium.android', + 'chrome', + 'chromium.memory', + 'chromium.dawn', + 'chromium.gpu', + 'chromium.fyi', + 'chromium.android.fyi', + 'chromium.clang', + 'chromium.fuzz', + 'chromium.gpu.fyi', + 'chromium.swangle', + ], + ) def set_defaults(milestone_vars, **kwargs): @@ -105,8 +115,8 @@ def _console_view_ordering_graph_key(console_name): return graph.key('@chromium', '', 'console_view_ordering', console_name) -def _main_console_view_ordering_graph_key(console_name): - return graph.key('@chromium', '', 'main_console_view_ordering', console_name) +def _overview_console_view_ordering_graph_key(console_name): + return graph.key('@chromium', '', 'overview_console_view_ordering', console_name) def _console_view_ordering_impl(ctx, *, console_name, ordering): @@ -120,15 +130,15 @@ def _console_view_ordering_impl(ctx, *, console_name, ordering): _console_view_ordering = lucicfg.rule(impl=_console_view_ordering_impl) -def _main_console_view_ordering_impl(ctx, *, console_name, top_level_ordering): - key = _main_console_view_ordering_graph_key(console_name) +def _overview_console_view_ordering_impl(ctx, *, console_name, top_level_ordering): + key = _overview_console_view_ordering_graph_key(console_name) graph.add_node(key, props = { 'top_level_ordering': top_level_ordering, }) graph.add_edge(keys.project(), key) return graph.keyset(key) -_main_console_view_ordering = lucicfg.rule(impl=_main_console_view_ordering_impl) +_overview_console_view_ordering = lucicfg.rule(impl=_overview_console_view_ordering_impl) def _category_join(parent, category): @@ -227,20 +237,20 @@ def _get_console_view_key_fn(console_name): return key_fn -def _get_main_console_view_key_fn(console_name): - """Get the key function for sorting entries of a main_console_view. +def _get_overview_console_view_key_fn(console_name): + """Get the key function for sorting overview_console_view entries. Returns: - The key function used to sort entries of the main_console_view with - the given name or None if the name does not refer to a - main_console_view. + The key function used to sort entries of the overview_console_view + with the given name or None if the name does not refer to an + overview_console_view. """ - main_console_ordering = graph.node( - _main_console_view_ordering_graph_key(console_name)) - if main_console_ordering == None: + overview_console_ordering = graph.node( + _overview_console_view_ordering_graph_key(console_name)) + if overview_console_ordering == None: return None - top_level_ordering = main_console_ordering.props.top_level_ordering + top_level_ordering = overview_console_ordering.props.top_level_ordering def key_fn(b): if not b.category: @@ -267,20 +277,19 @@ def _get_main_console_view_key_fn(console_name): return key_fn -def _sort_consoles(ctx): +def _sort_console_entries(ctx): milo = ctx.output['luci-milo.cfg'] consoles = [] for console in milo.consoles: if not console.builders: continue key_fn = (_get_console_view_key_fn(console.id) - or _get_main_console_view_key_fn(console.id)) + or _get_overview_console_view_key_fn(console.id)) if key_fn: console.builders = sorted(console.builders, key_fn) consoles.append(console) - milo.consoles = sorted(consoles, lambda c: c.id) -lucicfg.generator(_sort_consoles) +lucicfg.generator(_sort_console_entries) def ordering(*, short_names=None, categories=None): @@ -351,15 +360,15 @@ def console_view(*, name, ordering=None, **kwargs): ) -def main_console_view(*, name, top_level_ordering, **kwargs): - """Create a main console view. +def overview_console_view(*, name, top_level_ordering, **kwargs): + """Create an overview console view. - A main console view is a console view that contains a subset of + An overview console view is a console view that contains a subset of entries from other consoles. The entries from each console will have that console's name prepended to the entries' categories and will appear in the same order as they do in that console. The ordering of entries from different consoles is controlled by the - `top_level_ordering` parameter. + `top_level_ordering` parameter. Args: name - The name of the console view. @@ -381,7 +390,7 @@ def main_console_view(*, name, top_level_ordering, **kwargs): **kwargs ) - _main_console_view_ordering( + _overview_console_view_ordering( console_name = name, top_level_ordering = top_level_ordering, ) @@ -410,6 +419,7 @@ def ci_builder( add_to_console_view=args.DEFAULT, console_view=args.DEFAULT, main_console_view=args.DEFAULT, + cq_mirrors_console_view=args.DEFAULT, console_view_entry=None, **kwargs): """Define a CI builder. @@ -430,6 +440,11 @@ def ci_builder( defaults to None. An entry will be added only if `console_view_entry` is provided. Note that `add_to_console_view` has no effect on creating an entry to the main console view. + cq_mirrors_console_view - A string identifying the ID of the CQ + mirrors console view to add an entry to. Supports a module-level + default that defaults to None. An entry will be added only if + `console_view_entry` is provided. Note that `add_to_console_view` + has no effect on creating an entry to the main console view. console_view_entry - A structure providing the details of the entry to add to the console view. See `ci.console_view_entry` for details. """ @@ -472,6 +487,16 @@ def ci_builder( short_name = console_view_entry.short_name, ) + cq_mirrors_console_view = defaults.get_value( + 'cq_mirrors_console_view', cq_mirrors_console_view) + if cq_mirrors_console_view: + luci.console_view_entry( + builder = builder, + console_view = cq_mirrors_console_view, + category = '|'.join([console_view, console_view_entry.category]), + short_name = console_view_entry.short_name, + ) + return ret @@ -639,7 +664,7 @@ def fyi_ios_builder( **kwargs): if not caches: - caches = [builders.xcode_cache.x11c29] + caches = [builders.xcode_cache.x11e146] return fyi_builder( name = name, @@ -796,15 +821,15 @@ def mac_builder( def mac_ios_builder(*, name, caches=None, - executable='recipe:ios/unified_builder_tester', + executable='recipe:chromium', goma_backend=builders.goma.backend.RBE_PROD, properties=None, **kwargs): if not caches: - caches = [builders.xcode_cache.x11c29] + caches = [builders.xcode_cache.x11e146] if not properties: properties = { - 'xcode_build_version': '11c29' + 'xcode_build_version': '11e146' } return mac_builder( @@ -909,7 +934,7 @@ ci = struct( console_view_entry = console_view_entry, declare_bucket = declare_bucket, defaults = defaults, - main_console_view = main_console_view, + overview_console_view = overview_console_view, ordering = ordering, set_defaults = set_defaults, diff --git a/chromium/infra/config/lib/try.star b/chromium/infra/config/lib/try.star index d5d1231f157..93dab87357f 100644 --- a/chromium/infra/config/lib/try.star +++ b/chromium/infra/config/lib/try.star @@ -1,3 +1,7 @@ +# Copyright 2020 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + """Library for defining try builders. The `try_builder` function defined in this module enables defining a builder and @@ -48,6 +52,7 @@ def declare_bucket(milestone_vars): 'service-account-chromeperf', 'service-account-cq', ], + projects = milestone_vars.try_triggering_projects, ), acl.entry( roles = acl.BUILDBUCKET_OWNER, @@ -126,7 +131,7 @@ def _sorted_list_view_impl(ctx, *, console_name): _sorted_list_view = lucicfg.rule(impl=_sorted_list_view_impl) -def _sort_consoles(ctx): +def _sort_console_entries(ctx): milo = ctx.output['luci-milo.cfg'] consoles = [] for console in milo.consoles: @@ -137,9 +142,8 @@ def _sort_consoles(ctx): if node: console.builders = sorted(console.builders, lambda b: b.name) consoles.append(console) - milo.consoles = sorted(consoles, lambda c: c.id) -lucicfg.generator(_sort_consoles) +lucicfg.generator(_sort_console_entries) def list_view(*, name, **kwargs): @@ -288,6 +292,15 @@ def blink_mac_builder(*, name, **kwargs): ) +def chromium_builder(*, name, **kwargs): + return try_builder( + name = name, + goma_backend = builders.goma.backend.RBE_PROD, + mastername = 'tryserver.chromium', + **kwargs + ) + + def chromium_android_builder(*, name, **kwargs): return try_builder( name = name, @@ -368,10 +381,10 @@ def chromium_mac_ios_builder( properties=None, **kwargs): if not caches: - caches = [builders.xcode_cache.x11c29] + caches = [builders.xcode_cache.x11e146] if not properties: properties = { - 'xcode_build_version': '11c29', + 'xcode_build_version': '11e146', } return try_builder( name = name, @@ -500,6 +513,7 @@ try_ = struct( blink_builder = blink_builder, blink_mac_builder = blink_mac_builder, + chromium_builder = chromium_builder, chromium_android_builder = chromium_android_builder, chromium_angle_builder = chromium_angle_builder, chromium_chromiumos_builder = chromium_chromiumos_builder, |