# Copyright 2020 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """Library for defining builders. The `builder` function defined in this module simplifies setting all of the dimensions and many of the properties used for chromium builders by providing direct arguments for them rather than requiring them to appear as part of a dict. This simplifies creating wrapper functions that need to fix or override the default value of specific dimensions or property fields without having to handle merging dictionaries. Can also be accessed through `builders.builder`. The `defaults` struct provides module-level defaults for the arguments to `builder`. Each parameter of `builder` besides `name` and `kwargs` have a corresponding attribute on `defaults` that is a `lucicfg.var` that can be used to set the default value. Additionally, the module-level defaults defined for use with `luci.builder` can be accessed through `defaults`. Finally module-level defaults are provided for the `bucket` and `executable` arguments, removing the need to create a wrapper function just to set those default values for a bucket. Can also be accessed through `builders.defaults`. The `cpu`, `os`, and `goma` module members are structs that provide constants for use with the corresponding arguments to `builder`. Can also be accessed through `builders.cpu`, `builders.os` and `builders.goma` respectively. """ load('./args.star', 'args') ################################################################################ # Constants for use with the builder function # ################################################################################ # The cpu constants to be used with the builder function cpu = struct( X86 = 'x86', X86_64 = 'x86-64', ) # The category for an os: a more generic grouping than specific OS versions that # can be used for computing defaults os_category = struct( ANDROID = 'Android', LINUX = 'Linux', MAC = 'Mac', WINDOWS = 'Windows', ) # The os constants to be used for the os parameter of the builder function # The *_DEFAULT members enable distinguishing between a use that runs the # "current" version of the OS and a use that runs against a specific version # that happens to be the "current" version def os_enum(dimension, category): return struct(dimension=dimension, category=category) os = struct( ANDROID = os_enum('Android', os_category.ANDROID), LINUX_TRUSTY = os_enum('Ubuntu-14.04', os_category.LINUX), LINUX_XENIAL = os_enum('Ubuntu-16.04', os_category.LINUX), LINUX_DEFAULT = os_enum('Ubuntu-16.04', os_category.LINUX), MAC_10_12 = os_enum('Mac-10.12', os_category.MAC), MAC_10_13 = os_enum('Mac-10.13', os_category.MAC), MAC_10_14 = os_enum('Mac-10.14', os_category.MAC), MAC_10_15 = os_enum('Mac-10.15', os_category.MAC), MAC_DEFAULT = os_enum('Mac-10.13', os_category.MAC), MAC_ANY = os_enum('Mac', os_category.MAC), WINDOWS_7 = os_enum('Windows-7', os_category.WINDOWS), WINDOWS_8_1 = os_enum('Windows-8.1', os_category.WINDOWS), WINDOWS_10 = os_enum('Windows-10', os_category.WINDOWS), WINDOWS_DEFAULT = os_enum('Windows-10', os_category.WINDOWS), WINDOWS_ANY = os_enum('Windows', os_category.WINDOWS), ) # The constants to be used for the goma_backend and goma_jobs parameters of the # builder function goma = struct( backend = struct( RBE_PROD = { 'server_host': 'goma.chromium.org', 'rpc_extra_params': '?prod', }, RBE_STAGING = { 'server_host': 'staging-goma.chromium.org', 'rpc_extra_params': '?staging', }, RBE_TOT = { 'server_host': 'staging-goma.chromium.org', 'rpc_extra_params': '?tot', }, ), jobs = struct( J50 = 50, # This is for 4 cores mac. -j40 is too small, especially for clobber # builder. J80 = 80, # This is for tryservers becoming slow and critical path of patch # landing. J150 = 150, # This is for tryservers becoming very slow and critical path of patch # landing. J300 = 300, # CI builders (of which are few) may use high number of concurrent Goma # jobs. # IMPORTANT: when # * bumping number of jobs below, or # * adding this mixin to many builders at once, or # * adding this mixin to a builder with many concurrent builds # get review from Goma team. MANY_JOBS_FOR_CI = 500, # For load testing of the execution backend LOAD_TESTING_J1000 = 1000, LOAD_TESTING_J2000 = 2000, ), ) def xcode_enum(cache_name, cache_path): return swarming.cache(name=cache_name, path=cache_path) # Keep this in-sync with the versions of bots in //ios/build/bots/. xcode_cache = struct( x10e1001 = xcode_enum('xcode_ios_10e1001', 'xcode_ios_10e1001.app'), x11a1027 = xcode_enum('xcode_ios_11a1027', 'xcode_ios_11a1027.app'), x11c29 = xcode_enum('xcode_ios_11c29', 'xcode_ios_11c29.app'), x11m382q = xcode_enum('xcode_ios_11m382q', 'xcode_ios_11m382q.app'), x11e146 = xcode_enum('xcode_ios_11e146', 'xcode_ios_11e146.app'), x11e608c = xcode_enum('xcode_ios_11e608c', 'xcode_ios_11e608c.app'), x11e608cwk = xcode_enum('xcode_ios_11e608cwk', 'xcode_ios_11e608cwk.app'), x11n700h = xcode_enum('xcode_ios_11n700h', 'xcode_ios_11n700h.app'), # xcode12 x12a6159 = xcode_enum('xcode_ios_12a6159', 'xcode_ios_12a6159.app'), ) ################################################################################ # Implementation details # ################################################################################ _DEFAULT_BUILDERLESS_OS_CATEGORIES = [os_category.LINUX] def _chromium_tests_property(*, bucketed_triggers): chromium_tests = {} bucketed_triggers = defaults.get_value('bucketed_triggers', bucketed_triggers) if bucketed_triggers: chromium_tests['bucketed_triggers'] = True return chromium_tests or None def _goma_property(*, goma_backend, goma_debug, goma_enable_ats, goma_jobs, os): goma_properties = {} goma_backend = defaults.get_value('goma_backend', goma_backend) if goma_backend != None: goma_properties.update(goma_backend) goma_debug = defaults.get_value('goma_debug', goma_debug) if goma_debug: goma_properties['debug'] = True goma_enable_ats = defaults.get_value('goma_enable_ats', goma_enable_ats) # TODO(crbug.com/1040754): Remove this flag. if goma_enable_ats == args.COMPUTE: goma_enable_ats = ( os and os.category in (os_category.LINUX, os_category.WINDOWS) and goma_backend in (goma.backend.RBE_TOT, goma.backend.RBE_STAGING, goma.backend.RBE_PROD)) if goma_enable_ats: goma_properties['enable_ats'] = True goma_jobs = defaults.get_value('goma_jobs', goma_jobs) if goma_jobs != None: goma_properties['jobs'] = goma_jobs return goma_properties or None def _code_coverage_property( *, use_clang_coverage, use_java_coverage, coverage_exclude_sources, coverage_test_types): code_coverage = {} use_clang_coverage = defaults.get_value( 'use_clang_coverage', use_clang_coverage) if use_clang_coverage: code_coverage['use_clang_coverage'] = True use_java_coverage = defaults.get_value('use_java_coverage', use_java_coverage) if use_java_coverage: code_coverage['use_java_coverage'] = True coverage_exclude_sources = defaults.get_value('coverage_exclude_sources', coverage_exclude_sources) if coverage_exclude_sources: code_coverage['coverage_exclude_sources'] = coverage_exclude_sources coverage_test_types = defaults.get_value('coverage_test_types', coverage_test_types) if coverage_test_types: code_coverage['coverage_test_types'] = coverage_test_types return code_coverage or None ################################################################################ # Builder defaults and function # ################################################################################ # The module-level defaults to use with the builder function defaults = args.defaults( luci.builder.defaults, # Our custom arguments auto_builder_dimension = args.COMPUTE, builderless = args.COMPUTE, bucketed_triggers = False, configure_kitchen = False, cores = None, cpu = None, goma_backend = None, goma_debug = False, goma_enable_ats = args.COMPUTE, goma_jobs = None, mastername = None, os = None, pool = None, ssd = args.COMPUTE, use_clang_coverage = False, use_java_coverage = False, coverage_exclude_sources = None, coverage_test_types = None, resultdb_bigquery_exports = [], # Provide vars for bucket and executable so users don't have to # unnecessarily make wrapper functions bucket = args.COMPUTE, executable = args.COMPUTE, triggered_by = args.COMPUTE, # Forward on luci.builder.defaults so users have a consistent interface **{a: getattr(luci.builder.defaults, a) for a in dir(luci.builder.defaults)} ) def builder( *, name, bucket=args.DEFAULT, executable=args.DEFAULT, triggered_by=args.DEFAULT, os=args.DEFAULT, builderless=args.DEFAULT, auto_builder_dimension=args.DEFAULT, cores=args.DEFAULT, cpu=args.DEFAULT, mastername=args.DEFAULT, pool=args.DEFAULT, ssd=args.DEFAULT, bucketed_triggers=args.DEFAULT, configure_kitchen=args.DEFAULT, goma_backend=args.DEFAULT, goma_debug=args.DEFAULT, goma_enable_ats=args.DEFAULT, goma_jobs=args.DEFAULT, use_clang_coverage=args.DEFAULT, use_java_coverage=args.DEFAULT, coverage_exclude_sources=args.DEFAULT, coverage_test_types=args.DEFAULT, resultdb_bigquery_exports=args.DEFAULT, **kwargs): """Define a builder. For all of the optional parameters defined by this method, passing None will prevent the emission of any dimensions or property fields associated with that parameter. All parameters defined by this function except for `name` and `kwargs` support module-level defaults. The `defaults` struct defined in this module has an attribute with a `lucicfg.var` for all of the fields defined here as well as all of the parameters of `luci.builder` that support module-level defaults. See https://chromium.googlesource.com/infra/luci/luci-go/+/refs/heads/master/lucicfg/doc/README.md#luci.builder for more information. Arguments: * name - name of the builder, will show up in UIs and logs. Required. * bucket - a bucket the build is in, see luci.bucket(...) rule. Required (may be specified by module-level default). * executable - an executable to run, e.g. a luci.recipe(...). Required (may be specified by module-level default). * os - a member of the `os` enum indicating the OS the builder requires for the machines that run it. Emits a dimension of the form 'os:os'. By default considered None. * builderless - a boolean indicating whether the builder runs on builderless machines. If True, emits a 'builderless:1' dimension. By default, considered True iff `os` refers to a linux OS. * auto_builder_dimension - a boolean indicating whether the builder runs on machines devoted to the builder. If True, a dimension will be emitted of the form 'builder:'. By default, considered True iff `builderless` is considered False. * mastername - a string with the mastername of the builder. Emits a property of the form 'mastername:'. By default, considered None. * cores - an int indicating the number of cores the builder requires for the machines that run it. Emits a dimension of the form 'cores:' will be emitted. By default, considered None. * cpu - a member of the `cpu` enum indicating the cpu the builder requires for the machines that run it. Emits a dimension of the form 'cpu:'. By default, considered None. * pool - a string indicating the pool of the machines that run the builder. Emits a dimension of the form 'pool:'. By default, considered None. When running a builder that has no explicit pool dimension, buildbucket inserts one of the form 'pool:luci..'. * ssd - a boolean indicating whether the builder runs on machines with ssd. If True, emits a 'ssd:1' dimension. If False, emits a 'ssd:0' parameter. By default, considered False if builderless is considered True and otherwise None. * bucketed_triggers - a boolean indicating whether jobs triggered by the builder being defined should have the bucket prepended to the builder name to trigger. If True, the 'bucketed_triggers' field will be set in the '$build/chromium_tests' property. By default, considered False. * configure_kitchen - a boolean indicating whether to configure kitchen. If True, emits a property to set the 'git_auth' and 'devshell' fields of the '$kitchen' property. By default, considered False. * goma_backend - a member of the `goma.backend` enum indicating the goma backend the builder should use. Will be incorporated into the '$build/goma' property. By default, considered None. * goma_debug - a boolean indicating whether goma should be debugged. If True, the 'debug' field will be set in the '$build/goma' property. By default, considered False. * goma_enable_ats - a boolean indicating whether ats should be enabled for goma. If True, the 'enable_ats' field will be set in the '$build/goma' property. By default, considered False. * goma_jobs - a member of the `goma.jobs` enum indicating the number of jobs to be used by the builder. Sets the 'jobs' field of the '$build/goma' property will be set according to the enum member. By default, the 'jobs' considered None. * use_clang_coverage - a boolean indicating whether clang coverage should be used. If True, the 'use_clang_coverage" field will be set in the '$build/code_coverage' property. By default, considered False. * use_java_coverage - a boolean indicating whether java coverage should be used. If True, the 'use_java_coverage" field will be set in the '$build/code_coverage' property. By default, considered False. * coverage_exclude_sources - a string as the key to find the source file exclusion pattern in code_coverage recipe module. Will be copied to '$build/code_coverage' property if set. By default, considered None. * coverage_test_types - a list of string as test types to process data for in code_coverage recipe module. Will be copied to '$build/code_coverage' property. By default, considered None. * resultdb_bigquery_exports - a list of resultdb.export_test_results(...) specifying parameters for exporting test results to BigQuery. By default, do not export. * kwargs - Additional keyword arguments to forward on to `luci.builder`. """ # We don't have any need of an explicit dimensions dict, # instead we have individual arguments for dimensions if 'dimensions' in 'kwargs': fail('Explicit dimensions are not supported: ' + 'use builderless, cores, cpu, os or ssd instead') dimensions = {} properties = kwargs.pop('properties', {}) if '$kitchen' in properties: fail('Setting "$kitchen" property is not supported: ' + 'use configure_kitchen instead') if '$build/goma' in properties: fail('Setting "$build/goma" property is not supported: ' + 'use goma_backend, goma_dbug, goma_enable_ats and goma_jobs instead') if '$build/code_coverage' in properties: fail('Setting "$build/code_coverage" property is not supported: ' + 'use use_clang_coverage, use_java_coverage, coverage_exclude_sources' + ' and/or coverage_test_types instead') properties = dict(properties) os = defaults.get_value('os', os) if os: dimensions['os'] = os.dimension builderless = defaults.get_value('builderless', builderless) if builderless == args.COMPUTE: builderless = os != None and os.category in _DEFAULT_BUILDERLESS_OS_CATEGORIES if builderless: dimensions['builderless'] = '1' auto_builder_dimension = defaults.get_value( 'auto_builder_dimension', auto_builder_dimension) if auto_builder_dimension == args.COMPUTE: auto_builder_dimension = builderless == False if auto_builder_dimension: dimensions['builder'] = name cores = defaults.get_value('cores', cores) if cores != None: dimensions['cores'] = str(cores) cpu = defaults.get_value('cpu', cpu) if cpu != None: dimensions['cpu'] = cpu mastername = defaults.get_value('mastername', mastername) if mastername != None: properties['mastername'] = mastername pool = defaults.get_value('pool', pool) if pool: dimensions['pool'] = pool ssd = defaults.get_value('ssd', ssd) if ssd == args.COMPUTE: ssd = False if builderless else None if ssd != None: dimensions['ssd'] = str(int(ssd)) configure_kitchen = defaults.get_value('configure_kitchen', configure_kitchen) if configure_kitchen: properties['$kitchen'] = { 'devshell': True, 'git_auth': True, } chromium_tests = _chromium_tests_property( bucketed_triggers = bucketed_triggers, ) if chromium_tests != None: properties['$build/chromium_tests'] = chromium_tests goma = _goma_property( goma_backend = goma_backend, goma_debug = goma_debug, goma_enable_ats = goma_enable_ats, goma_jobs = goma_jobs, os = os, ) if goma != None: properties['$build/goma'] = goma code_coverage = _code_coverage_property( use_clang_coverage = use_clang_coverage, use_java_coverage = use_java_coverage, coverage_exclude_sources = coverage_exclude_sources, coverage_test_types = coverage_test_types, ) if code_coverage != None: properties['$build/code_coverage'] = code_coverage kwargs = dict(kwargs) bucket = defaults.get_value('bucket', bucket) if bucket != args.COMPUTE: kwargs['bucket'] = bucket executable = defaults.get_value('executable', executable) if executable != args.COMPUTE: kwargs['executable'] = executable triggered_by = defaults.get_value('triggered_by', triggered_by) if triggered_by != args.COMPUTE: kwargs['triggered_by'] = triggered_by return luci.builder( name = name, dimensions = dimensions, properties = properties, resultdb_settings = resultdb.settings( enable = True, bq_exports = defaults.get_value( 'resultdb_bigquery_exports', resultdb_bigquery_exports), ), **kwargs ) def builder_name(builder, bucket=args.DEFAULT): bucket = defaults.get_value('bucket', bucket) if bucket == args.COMPUTE: fail('Either a default for bucket must be set or bucket must be passed in') return '{}/{}'.format(bucket, builder) builders = struct( builder = builder, builder_name = builder_name, cpu = cpu, defaults = defaults, goma = goma, os = os, xcode_cache = xcode_cache, )