Refactor handling of machine file options

It is much easier to not try to parse options into complicated
structures until we actually collected all options: machine files,
command line, project()'s default_options, environment.

6 files changed, 140 insertions, 221 deletions

diff --git a/mesonbuild/ast/introspection.py b/mesonbuild/ast/introspection.py
index 334ff8325..eca869fa3 100644
--- a/mesonbuild/ast/introspection.py
+++ b/mesonbuild/ast/introspection.py
@@ -125,7 +125,7 @@ class IntrospectionInterpreter(AstInterpreter):
                         self.do_subproject(i)
 
         self.coredata.init_backend_options(self.backend)
-        options = {k: v for k, v in self.environment.meson_options.host[''].items() if k.startswith('backend_')}
+        options = {k: v for k, v in self.environment.raw_options.items() if k.startswith('backend_')}
 
         self.coredata.set_options(options)
         self._add_languages(proj_langs, MachineChoice.HOST)
diff --git a/mesonbuild/compilers/compilers.py b/mesonbuild/compilers/compilers.py
index fd3bdb7ee..0f074bb17 100644
--- a/mesonbuild/compilers/compilers.py
+++ b/mesonbuild/compilers/compilers.py
@@ -28,7 +28,7 @@ from ..mesonlib import (
     Popen_safe, split_args, LibType
 )
 from ..envconfig import (
-    Properties, get_env_var
+    get_env_var
 )
 
 from ..arglist import CompilerArgs
@@ -1246,8 +1246,7 @@ def get_args_from_envvars(lang: str,
 def get_global_options(lang: str,
                        comp: T.Type[Compiler],
                        for_machine: MachineChoice,
-                       is_cross: bool,
-                       properties: Properties) -> 'OptionDictType':
+                       is_cross: bool) -> 'OptionDictType':
     """Retreive options that apply to all compilers for a given language."""
     description = 'Extra arguments passed to the {}'.format(lang)
     opts = {
@@ -1267,11 +1266,7 @@ def get_global_options(lang: str,
         comp.INVOKES_LINKER)
 
     for k, o in opts.items():
-        user_k = lang + '_' + k
-        if user_k in properties:
-            # Get from configuration files.
-            o.set_value(properties[user_k])
-        elif k == 'args':
+        if k == 'args':
             o.set_value(compile_args)
         elif k == 'link_args':
             o.set_value(link_args)
diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py
index e3b3b01ec..9ed939b3c 100644
--- a/mesonbuild/coredata.py
+++ b/mesonbuild/coredata.py
@@ -658,6 +658,23 @@ class CoreData:
             for k1, v1 in v0.items():
                 yield (k0 + k1, v1)
 
+    @classmethod
+    def insert_build_prefix(cls, k):
+        idx = k.find(':')
+        if idx < 0:
+            return 'build.' + k
+        return k[:idx + 1] + 'build.' + k[idx + 1:]
+
+    @classmethod
+    def is_per_machine_option(cls, optname):
+        if optname in BUILTIN_OPTIONS_PER_MACHINE:
+            return True
+        from .compilers import compilers
+        for lang_prefix in [lang + '_' for lang in compilers.all_languages]:
+            if optname.startswith(lang_prefix):
+                return True
+        return False
+
     def _get_all_nonbuiltin_options(self) -> T.Iterable[T.Dict[str, UserOption]]:
         yield self.backend_options
         yield self.user_options
@@ -766,90 +783,75 @@ class CoreData:
             self.copy_build_options_from_regular_ones()
 
     def set_default_options(self, default_options: 'T.OrderedDict[str, str]', subproject: str, env: 'Environment') -> None:
-        def make_key(key: str) -> str:
+        # Preserve order: if env.raw_options has 'buildtype' it must come after
+        # 'optimization' if it is in default_options.
+        raw_options = OrderedDict()
+        for k, v in default_options.items():
             if subproject:
-                return '{}:{}'.format(subproject, key)
-            return key
-
+                k = subproject + ':' + k
+            raw_options[k] = v
+        raw_options.update(env.raw_options)
+        env.raw_options = raw_options
+
+        # Create a subset of raw_options, keeping only project and builtin
+        # options for this subproject.
+        # Language and backend specific options will be set later when adding
+        # languages and setting the backend (builtin options must be set first
+        # to know which backend we'll use).
         options = OrderedDict()
 
-        # TODO: validate these
-        from .compilers import all_languages, base_options
-        lang_prefixes = tuple('{}_'.format(l) for l in all_languages)
-        # split arguments that can be set now, and those that cannot so they
-        # can be set later, when they've been initialized.
-        for k, v in default_options.items():
-            if k.startswith(lang_prefixes):
-                lang, key = k.split('_', 1)
-                for machine in MachineChoice:
-                    if key not in env.compiler_options[machine][lang]:
-                        env.compiler_options[machine][lang][key] = v
-            elif k in base_options:
-                if not subproject and k not in env.base_options:
-                    env.base_options[k] = v
-            else:
-                options[make_key(k)] = v
-
-        for k, v in chain(env.meson_options.host.get('', {}).items(),
-                          env.meson_options.host.get(subproject, {}).items()):
-            options[make_key(k)] = v
-
-        for k, v in chain(env.meson_options.build.get('', {}).items(),
-                          env.meson_options.build.get(subproject, {}).items()):
-            if k in BUILTIN_OPTIONS_PER_MACHINE:
-                options[make_key('build.{}'.format(k))] = v
-
-        options.update({make_key(k): v for k, v in env.user_options.get(subproject, {}).items()})
-
-        # Some options (namely the compiler options) are not preasant in
-        # coredata until the compiler is fully initialized. As such, we need to
-        # put those options into env.meson_options, only if they're not already
-        # in there, as the machine files and command line have precendence.
-        for k, v in default_options.items():
-            if k in BUILTIN_OPTIONS and not BUILTIN_OPTIONS[k].yielding:
-                continue
-            for machine in MachineChoice:
-                if machine is MachineChoice.BUILD and not self.is_cross_build():
+        from . import optinterpreter
+        for k, v in env.raw_options.items():
+            raw_optname = k
+            if subproject:
+                # Subproject: skip options for other subprojects
+                if not k.startswith(subproject + ':'):
                     continue
-                if k not in env.meson_options[machine][subproject]:
-                    env.meson_options[machine][subproject][k] = v
+                raw_optname = k.split(':')[1]
+            elif ':' in k:
+                # Main prject: skip options for subprojects
+                continue
+            # Skip base, compiler, and backend options, they are handled when
+            # adding languages and setting backend.
+            if (k not in self.builtins and
+                k not in self.get_prefixed_options_per_machine(self.builtins_per_machine) and
+                optinterpreter.is_invalid_name(raw_optname, log=False)):
+                continue
+            options[k] = v
 
         self.set_options(options, subproject=subproject)
 
+    def add_compiler_options(self, options, lang, for_machine, env):
+        # prefixed compiler options affect just this machine
+        opt_prefix = for_machine.get_prefix()
+        for k, o in options.items():
+            optname = opt_prefix + lang + '_' + k
+            value = env.raw_options.get(optname)
+            if value is not None:
+                o.set_value(value)
+            self.compiler_options[for_machine][lang].setdefault(k, o)
+
     def add_lang_args(self, lang: str, comp: T.Type['Compiler'],
                       for_machine: MachineChoice, env: 'Environment') -> None:
         """Add global language arguments that are needed before compiler/linker detection."""
         from .compilers import compilers
-
-        for k, o in compilers.get_global_options(
-                lang,
-                comp,
-                for_machine,
-                env.is_cross_build(),
-                env.properties[for_machine]).items():
-            # prefixed compiler options affect just this machine
-            if k in env.compiler_options[for_machine].get(lang, {}):
-                o.set_value(env.compiler_options[for_machine][lang][k])
-            self.compiler_options[for_machine][lang].setdefault(k, o)
+        options = compilers.get_global_options(lang, comp, for_machine,
+                                               env.is_cross_build())
+        self.add_compiler_options(options, lang, for_machine, env)
 
     def process_new_compiler(self, lang: str, comp: 'Compiler', env: 'Environment') -> None:
         from . import compilers
 
         self.compilers[comp.for_machine][lang] = comp
-
-        for k, o in comp.get_options().items():
-            # prefixed compiler options affect just this machine
-            if k in env.compiler_options[comp.for_machine].get(lang, {}):
-                o.set_value(env.compiler_options[comp.for_machine][lang][k])
-            self.compiler_options[comp.for_machine][lang].setdefault(k, o)
+        self.add_compiler_options(comp.get_options(), lang, comp.for_machine, env)
 
         enabled_opts = []
         for optname in comp.base_options:
             if optname in self.base_options:
                 continue
             oobj = compilers.base_options[optname]
-            if optname in env.base_options:
-                oobj.set_value(env.base_options[optname])
+            if optname in env.raw_options:
+                oobj.set_value(env.raw_options[optname])
                 enabled_opts.append(optname)
             self.base_options[optname] = oobj
         self.emit_base_options_warnings(enabled_opts)
diff --git a/mesonbuild/environment.py b/mesonbuild/environment.py
index 394ef6a61..e63e9bf62 100644
--- a/mesonbuild/environment.py
+++ b/mesonbuild/environment.py
@@ -35,8 +35,6 @@ from .envconfig import (
 from . import compilers
 from .compilers import (
     Compiler,
-    all_languages,
-    base_options,
     is_assembly,
     is_header,
     is_library,
@@ -582,12 +580,6 @@ class Environment:
         # CMake toolchain variables
         cmakevars = PerMachineDefaultable()  # type: PerMachineDefaultable[CMakeVariables]
 
-        # We only need one of these as project options are not per machine
-        user_options = collections.defaultdict(dict)  # type: T.DefaultDict[str, T.Dict[str, object]]
-
-        # meson builtin options, as passed through cross or native files
-        meson_options = PerMachineDefaultable()  # type: PerMachineDefaultable[T.DefaultDict[str, T.Dict[str, object]]]
-
         ## Setup build machine defaults
 
         # Will be fully initialized later using compilers later.
@@ -598,80 +590,21 @@ class Environment:
         binaries.build = BinaryTable()
         properties.build = Properties()
 
-        # meson base options
-        _base_options = {}  # type: T.Dict[str, object]
-
-        # Per language compiler arguments
-        compiler_options = PerMachineDefaultable()  # type: PerMachineDefaultable[T.DefaultDict[str, T.Dict[str, object]]]
-        compiler_options.build = collections.defaultdict(dict)
+        # Unparsed options as given by the user in machine files, command line,
+        # and project()'s default_options. Keys are in the command line format:
+        # "[<subproject>:][build.]option_name".
+        # Note that order matters because of 'buildtype', if it is after
+        # 'optimization' and 'debug' keys, it override them.
+        self.raw_options = collections.OrderedDict() # type: collections.OrderedDict[str, str]
 
         ## Read in native file(s) to override build machine configuration
 
-        def load_options(tag: str, store: T.Dict[str, T.Any]) -> None:
-            for section in config.keys():
-                if section.endswith(tag):
-                    if ':' in section:
-                        project = section.split(':')[0]
-                    else:
-                        project = ''
-                    store[project].update(config.get(section, {}))
-
-        def split_base_options(mopts: T.DefaultDict[str, T.Dict[str, object]]) -> None:
-            for k, v in list(mopts.get('', {}).items()):
-                if k in base_options:
-                    _base_options[k] = v
-                    del mopts[k]
-
-        lang_prefixes = tuple('{}_'.format(l) for l in all_languages)
-        def split_compiler_options(mopts: T.DefaultDict[str, T.Dict[str, object]], machine: MachineChoice) -> None:
-            for k, v in list(mopts.get('', {}).items()):
-                if k.startswith(lang_prefixes):
-                    lang, key = k.split('_', 1)
-                    if compiler_options[machine] is None:
-                        compiler_options[machine] = collections.defaultdict(dict)
-                    if lang not in compiler_options[machine]:
-                        compiler_options[machine][lang] = collections.defaultdict(dict)
-                    compiler_options[machine][lang][key] = v
-                    del mopts[''][k]
-
-        def move_compiler_options(properties: Properties, compopts: T.Dict[str, T.DefaultDict[str, object]]) -> None:
-            for k, v in properties.properties.copy().items():
-                for lang in all_languages:
-                    if k == '{}_args'.format(lang):
-                        if 'args' not in compopts[lang]:
-                            compopts[lang]['args'] = v
-                        else:
-                            mlog.warning('Ignoring {}_args in [properties] section for those in the [built-in options]'.format(lang))
-                    elif k == '{}_link_args'.format(lang):
-                        if 'link_args' not in compopts[lang]:
-                            compopts[lang]['link_args'] = v
-                        else:
-                            mlog.warning('Ignoring {}_link_args in [properties] section in favor of the [built-in options] section.')
-                    else:
-                        continue
-                    mlog.deprecation('{} in the [properties] section of the machine file is deprecated, use the [built-in options] section.'.format(k))
-                    del properties.properties[k]
-                    break
-
         if self.coredata.config_files is not None:
             config = coredata.parse_machine_files(self.coredata.config_files)
             binaries.build = BinaryTable(config.get('binaries', {}))
             properties.build = Properties(config.get('properties', {}))
             cmakevars.build = CMakeVariables(config.get('cmake', {}))
-
-            # Don't run this if there are any cross files, we don't want to use
-            # the native values if we're doing a cross build
-            if not self.coredata.cross_files:
-                load_options('project options', user_options)
-            meson_options.build = collections.defaultdict(dict)
-            if config.get('paths') is not None:
-                mlog.deprecation('The [paths] section is deprecated, use the [built-in options] section instead.')
-                load_options('paths', meson_options.build)
-            load_options('built-in options', meson_options.build)
-            if not self.coredata.cross_files:
-                split_base_options(meson_options.build)
-            split_compiler_options(meson_options.build, MachineChoice.BUILD)
-            move_compiler_options(properties.build, compiler_options.build)
+            self.load_machine_file_options(config, properties.build)
 
         ## Read in cross file(s) to override host machine configuration
 
@@ -684,16 +617,10 @@ class Environment:
                 machines.host = MachineInfo.from_literal(config['host_machine'])
             if 'target_machine' in config:
                 machines.target = MachineInfo.from_literal(config['target_machine'])
-            load_options('project options', user_options)
-            meson_options.host = collections.defaultdict(dict)
-            compiler_options.host = collections.defaultdict(dict)
-            if config.get('paths') is not None:
-                mlog.deprecation('The [paths] section is deprecated, use the [built-in options] section instead.')
-                load_options('paths', meson_options.host)
-            load_options('built-in options', meson_options.host)
-            split_base_options(meson_options.host)
-            split_compiler_options(meson_options.host, MachineChoice.HOST)
-            move_compiler_options(properties.host, compiler_options.host)
+            # Keep only per machine options from the native file and prefix them
+            # with "build.". The cross file takes precedence over all other options.
+            self.keep_per_machine_options()
+            self.load_machine_file_options(config, properties.host)
 
         ## "freeze" now initialized configuration, and "save" to the class.
 
@@ -701,66 +628,17 @@ class Environment:
         self.binaries = binaries.default_missing()
         self.properties = properties.default_missing()
         self.cmakevars = cmakevars.default_missing()
-        self.user_options = user_options
-        self.meson_options = meson_options.default_missing()
-        self.base_options = _base_options
-        self.compiler_options = compiler_options.default_missing()
-
-        # Some options default to environment variables if they are
-        # unset, set those now.
-
-        for for_machine in MachineChoice:
-            p_env_pair = get_env_var_pair(for_machine, self.coredata.is_cross_build(), 'PKG_CONFIG_PATH')
-            if p_env_pair is not None:
-                p_env_var, p_env = p_env_pair
-
-                # PKG_CONFIG_PATH may contain duplicates, which must be
-                # removed, else a duplicates-in-array-option warning arises.
-                p_list = list(mesonlib.OrderedSet(p_env.split(':')))
 
-                key = 'pkg_config_path'
+        # Environment options override those from cross/native files
+        self.set_options_from_env()
 
-                if self.first_invocation:
-                    # Environment variables override config
-                    self.meson_options[for_machine][''][key] = p_list
-                elif self.meson_options[for_machine][''].get(key, []) != p_list:
-                    mlog.warning(
-                        p_env_var,
-                        'environment variable does not match configured',
-                        'between configurations, meson ignores this.',
-                        'Use -Dpkg_config_path to change pkg-config search',
-                        'path instead.'
-                    )
-
-        # Read in command line and populate options
-        # TODO: validate all of this
-        all_builtins = set(coredata.BUILTIN_OPTIONS) | set(coredata.BUILTIN_OPTIONS_PER_MACHINE) | set(coredata.builtin_dir_noprefix_options)
-        for k, v in options.cmd_line_options.items():
-            try:
-                subproject, k = k.split(':')
-            except ValueError:
-                subproject = ''
-            if k in base_options:
-                self.base_options[k] = v
-            elif k.startswith(lang_prefixes):
-                lang, key = k.split('_', 1)
-                self.compiler_options.host[lang][key] = v
-            elif k in all_builtins or k.startswith('backend_'):
-                self.meson_options.host[subproject][k] = v
-            elif k.startswith('build.'):
-                k = k.lstrip('build.')
-                if k in coredata.BUILTIN_OPTIONS_PER_MACHINE:
-                    if self.meson_options.build is None:
-                        self.meson_options.build = collections.defaultdict(dict)
-                    self.meson_options.build[subproject][k] = v
-            else:
-                assert not k.startswith('build.')
-                self.user_options[subproject][k] = v
+        # Command line options override those from cross/native files
+        self.raw_options.update(options.cmd_line_options)
 
         # Warn if the user is using two different ways of setting build-type
         # options that override each other
-        if meson_options.build and 'buildtype' in meson_options.build[''] and \
-           ('optimization' in meson_options.build[''] or 'debug' in meson_options.build['']):
+        if 'buildtype' in self.raw_options and \
+           ('optimization' in self.raw_options or 'debug' in self.raw_options):
             mlog.warning('Recommend using either -Dbuildtype or -Doptimization + -Ddebug. '
                          'Using both is redundant since they override each other. '
                          'See: https://mesonbuild.com/Builtin-options.html#build-type-options')
@@ -819,6 +697,57 @@ class Environment:
         self.default_pkgconfig = ['pkg-config']
         self.wrap_resolver = None
 
+    def load_machine_file_options(self, config, properties):
+        paths = config.get('paths')
+        if paths:
+            mlog.deprecation('The [paths] section is deprecated, use the [built-in options] section instead.')
+            self.raw_options.update(paths)
+        deprecated_properties = set()
+        for lang in compilers.all_languages:
+            deprecated_properties.add(lang + '_args')
+            deprecated_properties.add(lang + '_link_args')
+        for k, v in properties.properties.copy().items():
+            if k in deprecated_properties:
+                mlog.deprecation('{} in the [properties] section of the machine file is deprecated, use the [built-in options] section.'.format(k))
+                self.raw_options[k] = v
+                del properties.properties[k]
+        for section, values in config.items():
+            prefix = ''
+            if ':' in section:
+                subproject, section = section.split(':')
+                prefix = subproject + ':'
+            if section in ['project options', 'built-in options']:
+                self.raw_options.update({prefix + k: v for k, v in values.items()})
+
+    def keep_per_machine_options(self):
+        per_machine_options = {}
+        for optname, value in self.raw_options.items():
+            if self.coredata.is_per_machine_option(optname):
+                build_optname = self.coredata.insert_build_prefix(optname)
+                per_machine_options[build_optname] = value
+        self.raw_options = per_machine_options
+
+    def set_options_from_env(self):
+        for for_machine in MachineChoice:
+            p_env_pair = get_env_var_pair(for_machine, self.is_cross_build(), 'PKG_CONFIG_PATH')
+            if p_env_pair is not None:
+                p_env_var, p_env = p_env_pair
+
+                # PKG_CONFIG_PATH may contain duplicates, which must be
+                # removed, else a duplicates-in-array-option warning arises.
+                p_list = list(mesonlib.OrderedSet(p_env.split(':')))
+
+                key = 'pkg_config_path'
+                if for_machine == MachineChoice.BUILD:
+                    key = 'build.' + key
+
+                # Take env vars only on first invocation, if the env changes when
+                # reconfiguring it gets ignored.
+                # FIXME: We should remember if we took the value from env to warn
+                # if it changes on future invocations.
+                if self.first_invocation:
+                    self.raw_options[key] = p_list
+
     def create_new_coredata(self, options: 'argparse.Namespace') -> None:
         # WARNING: Don't use any values from coredata in __init__. It gets
         # re-initialized with project options by the interpreter during
diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py
index cd201d097..0dc38f393 100644
--- a/mesonbuild/interpreter.py
+++ b/mesonbuild/interpreter.py
@@ -3113,9 +3113,8 @@ external dependencies (including libraries) must go to "dependencies".''')
         if self.environment.first_invocation:
             self.coredata.init_backend_options(backend)
 
-        if '' in self.environment.meson_options.host:
-            options = {k: v for k, v in self.environment.meson_options.host[''].items() if k.startswith('backend_')}
-            self.coredata.set_options(options)
+        options = {k: v for k, v in self.environment.raw_options.items() if k.startswith('backend_')}
+        self.coredata.set_options(options)
 
     @stringArgs
     @permittedKwargs(permitted_kwargs['project'])
diff --git a/mesonbuild/mconf.py b/mesonbuild/mconf.py
index 02451558d..774dc5a07 100644
--- a/mesonbuild/mconf.py
+++ b/mesonbuild/mconf.py
@@ -197,19 +197,13 @@ class Conf:
         test_options = {k: o for k, o in self.coredata.builtins.items() if k in test_option_names}
         core_options = {k: o for k, o in self.coredata.builtins.items() if k in core_option_names}
 
-        def insert_build_prefix(k):
-            idx = k.find(':')
-            if idx < 0:
-                return 'build.' + k
-            return k[:idx + 1] + 'build.' + k[idx + 1:]
-
         core_options = self.split_options_per_subproject(core_options)
         host_compiler_options = self.split_options_per_subproject(
             dict(self.coredata.flatten_lang_iterator(
                 self.coredata.compiler_options.host.items())))
         build_compiler_options = self.split_options_per_subproject(
             dict(self.coredata.flatten_lang_iterator(
-                (insert_build_prefix(k), o)
+                (self.coredata.insert_build_prefix(k), o)
                 for k, o in self.coredata.compiler_options.build.items())))
         project_options = self.split_options_per_subproject(self.coredata.user_options)
         show_build_options = self.default_values_only or self.build.environment.is_cross_build()
@@ -218,7 +212,7 @@ class Conf:
         self.print_options('Core options', core_options[''])
         self.print_options('', self.coredata.builtins_per_machine.host)
         if show_build_options:
-            self.print_options('', {insert_build_prefix(k): o for k, o in self.coredata.builtins_per_machine.build.items()})
+            self.print_options('', {self.coredata.insert_build_prefix(k): o for k, o in self.coredata.builtins_per_machine.build.items()})
         self.print_options('Backend options', self.coredata.backend_options)
         self.print_options('Base options', self.coredata.base_options)
         self.print_options('Compiler options', host_compiler_options.get('', {}))