python module: add an automatic byte-compilation step

For all source `*.py` files installed via either py.install_sources() or
an `install_dir: py.get_install_dir()`, produce `*.pyc` files at install
time. Controllable via a module option.

8 files changed, 149 insertions, 7 deletions

diff --git a/docs/markdown/Builtin-options.md b/docs/markdown/Builtin-options.md
index c0a8a0de2..276479845 100644
--- a/docs/markdown/Builtin-options.md
+++ b/docs/markdown/Builtin-options.md
@@ -351,6 +351,7 @@ install prefix. For example: if the install prefix is `/usr` and the
 
 | Option           | Default value | Possible values             | Description |
 | ------           | ------------- | -----------------           | ----------- |
+| bytecompile      | 0             | integer from -1 to 2        | What bytecode optimization level to use (Since 1.2.0) |
 | install_env      | prefix        | {auto,prefix,system,venv}   | Which python environment to install to (Since 0.62.0) |
 | platlibdir       |               | Directory path              | Directory for site-specific, platform-specific files (Since 0.60.0) |
 | purelibdir       |               | Directory path              | Directory for site-specific, non-platform-specific files  (Since 0.60.0) |
@@ -373,3 +374,13 @@ installation is a virtualenv, and use `venv` or `system` as appropriate (but
 never `prefix`). This option is mutually exclusive with the `platlibdir`/`purelibdir`.
 
 For backwards compatibility purposes, the default `install_env` is `prefix`.
+
+*Since 1.2.0* The `python.bytecompile` option can be used to enable compiling
+python bytecode. Bytecode has 3 optimization levels:
+
+- 0, bytecode without optimizations
+- 1, bytecode with some optimizations
+- 2, bytecode with some more optimizations
+
+To this, Meson adds level `-1`, which is to not attempt to compile bytecode at
+all.
diff --git a/docs/markdown/snippets/python_bytecompile.md b/docs/markdown/snippets/python_bytecompile.md
new file mode 100644
index 000000000..0240c9d08
--- /dev/null
+++ b/docs/markdown/snippets/python_bytecompile.md
@@ -0,0 +1,4 @@
+## Python module can now compile bytecode
+
+A new builtin option is available: `-Dpython.bytecompile=2`. It can be used to
+compile bytecode for all pure python files installed via the python module.
diff --git a/mesonbuild/coredata.py b/mesonbuild/coredata.py
index c77942286..6ce30c9c8 100644
--- a/mesonbuild/coredata.py
+++ b/mesonbuild/coredata.py
@@ -1266,6 +1266,8 @@ BUILTIN_CORE_OPTIONS: 'MutableKeyedOptionDictType' = OrderedDict([
      BuiltinOption(UserBooleanOption, 'Generate pkgconfig files as relocatable', False)),
 
     # Python module
+    (OptionKey('bytecompile', module='python'),
+     BuiltinOption(UserIntegerOption, 'Whether to compile bytecode', (-1, 2, 0))),
     (OptionKey('install_env', module='python'),
      BuiltinOption(UserComboOption, 'Which python environment to install to', 'prefix', choices=['auto', 'prefix', 'system', 'venv'])),
     (OptionKey('platlibdir', module='python'),
diff --git a/mesonbuild/modules/python.py b/mesonbuild/modules/python.py
index 162f8c515..a3868a065 100644
--- a/mesonbuild/modules/python.py
+++ b/mesonbuild/modules/python.py
@@ -13,9 +13,7 @@
 # limitations under the License.
 from __future__ import annotations
 
-import copy
-import os
-import shutil
+import copy, json, os, shutil
 import typing as T
 
 from . import ExtensionModule, ModuleInfo
@@ -41,7 +39,7 @@ if T.TYPE_CHECKING:
     from typing_extensions import TypedDict
 
     from . import ModuleState
-    from ..build import SharedModule, Data
+    from ..build import Build, SharedModule, Data
     from ..dependencies import Dependency
     from ..interpreter import Interpreter
     from ..interpreter.kwargs import ExtractRequired
@@ -66,6 +64,12 @@ mod_kwargs -= {'name_prefix', 'name_suffix'}
 
 
 class PythonExternalProgram(BasicPythonExternalProgram):
+
+    # This is a ClassVar instead of an instance bool, because although an
+    # installation is cached, we actually copy it, modify attributes such as pure,
+    # and return a temporary one rather than the cached object.
+    run_bytecompile: T.ClassVar[T.Dict[str, bool]] = {}
+
     def sanity(self, state: T.Optional['ModuleState'] = None) -> bool:
         ret = super().sanity()
         if ret:
@@ -216,6 +220,7 @@ class PythonInstallation(ExternalProgramHolder):
     )
     def install_sources_method(self, args: T.Tuple[T.List[T.Union[str, mesonlib.File]]],
                                kwargs: 'PyInstallKw') -> 'Data':
+        self.held_object.run_bytecompile[self.version] = True
         tag = kwargs['install_tag'] or 'python-runtime'
         pure = kwargs['pure'] if kwargs['pure'] is not None else self.pure
         install_dir = self._get_install_dir_impl(pure, kwargs['subdir'])
@@ -229,6 +234,7 @@ class PythonInstallation(ExternalProgramHolder):
     @noPosargs
     @typed_kwargs('python_installation.install_dir', _PURE_KW, _SUBDIR_KW)
     def get_install_dir_method(self, args: T.List['TYPE_var'], kwargs: 'PyInstallKw') -> str:
+        self.held_object.run_bytecompile[self.version] = True
         pure = kwargs['pure'] if kwargs['pure'] is not None else self.pure
         return self._get_install_dir_impl(pure, kwargs['subdir'])
 
@@ -297,6 +303,56 @@ class PythonModule(ExtensionModule):
             'find_installation': self.find_installation,
         })
 
+    def _get_install_scripts(self) -> T.List[mesonlib.ExecutableSerialisation]:
+        backend = self.interpreter.backend
+        ret = []
+        optlevel = self.interpreter.environment.coredata.get_option(mesonlib.OptionKey('bytecompile', module='python'))
+        if optlevel == -1:
+            return ret
+        if not any(PythonExternalProgram.run_bytecompile.values()):
+            return ret
+
+        installdata = backend.create_install_data()
+        py_files = []
+
+        def should_append(f, isdir: bool = False):
+            # This uses the install_plan decorated names to see if the original source was propagated via
+            # install_sources() or get_install_dir().
+            return f.startswith(('{py_platlib}', '{py_purelib}')) and (f.endswith('.py') or isdir)
+
+        for t in installdata.targets:
+            if should_append(t.out_name):
+                py_files.append(os.path.join(installdata.prefix, t.outdir, os.path.basename(t.fname)))
+        for d in installdata.data:
+            if should_append(d.install_path_name):
+                py_files.append(os.path.join(installdata.prefix, d.install_path))
+        for d in installdata.install_subdirs:
+            if should_append(d.install_path_name, True):
+                py_files.append(os.path.join(installdata.prefix, d.install_path))
+
+        import importlib.resources
+        pycompile = os.path.join(self.interpreter.environment.get_scratch_dir(), 'pycompile.py')
+        with open(pycompile, 'wb') as f:
+            f.write(importlib.resources.read_binary('mesonbuild.scripts', 'pycompile.py'))
+
+        for i in self.installations.values():
+            if isinstance(i, PythonExternalProgram) and i.run_bytecompile[i.info['version']]:
+                i = T.cast(PythonExternalProgram, i)
+                manifest = f'python-{i.info["version"]}-installed.json'
+                manifest_json = []
+                for f in py_files:
+                    if f.startswith((os.path.join(installdata.prefix, i.platlib), os.path.join(installdata.prefix, i.purelib))):
+                        manifest_json.append(f)
+                with open(os.path.join(self.interpreter.environment.get_scratch_dir(), manifest), 'w', encoding='utf-8') as f:
+                    json.dump(manifest_json, f)
+                cmd = i.command + [pycompile, manifest, str(optlevel)]
+                script = backend.get_executable_serialisation(cmd, verbose=True)
+                ret.append(script)
+        return ret
+
+    def postconf_hook(self, b: Build) -> None:
+        b.install_scripts.extend(self._get_install_scripts())
+
     # https://www.python.org/dev/peps/pep-0397/
     @staticmethod
     def _get_win_pythonpath(name_or_path: str) -> T.Optional[str]:
@@ -421,6 +477,7 @@ class PythonModule(ExtensionModule):
         else:
             python = copy.copy(python)
             python.pure = kwargs['pure']
+            python.run_bytecompile.setdefault(python.info['version'], False)
             return python
 
         raise mesonlib.MesonBugException('Unreachable code was reached (PythonModule.find_installation).')
diff --git a/mesonbuild/scripts/pycompile.py b/mesonbuild/scripts/pycompile.py
new file mode 100644
index 000000000..da92655b0
--- /dev/null
+++ b/mesonbuild/scripts/pycompile.py
@@ -0,0 +1,67 @@
+# Copyright 2016 The Meson development team
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# ignore all lints for this file, since it is run by python2 as well
+
+# type: ignore
+# pylint: disable=deprecated-module
+
+import json, os, subprocess, sys
+from compileall import compile_file
+
+destdir = os.environ.get('DESTDIR')
+quiet = int(os.environ.get('MESON_INSTALL_QUIET', 0))
+
+def destdir_join(d1, d2):
+    if not d1:
+        return d2
+    # c:\destdir + c:\prefix must produce c:\destdir\prefix
+    parts = os.path.splitdrive(d2)
+    return d1 + parts[1]
+
+def compileall(files):
+    for f in files:
+        if destdir is not None:
+            ddir = os.path.dirname(f)
+            fullpath = destdir_join(destdir, f)
+        else:
+            ddir = None
+            fullpath = f
+
+        if os.path.isdir(fullpath):
+            for root, _, files in os.walk(fullpath):
+                ddir = os.path.dirname(os.path.splitdrive(f)[0] + root[len(destdir):])
+                for dirf in files:
+                    if dirf.endswith('.py'):
+                        fullpath = os.path.join(root, dirf)
+                        compile_file(fullpath, ddir, force=True, quiet=quiet)
+        else:
+            compile_file(fullpath, ddir, force=True, quiet=quiet)
+
+def run(manifest):
+    data_file = os.path.join(os.path.dirname(__file__), manifest)
+    with open(data_file, 'rb') as f:
+        dat = json.load(f)
+    compileall(dat)
+
+if __name__ == '__main__':
+    manifest = sys.argv[1]
+    run(manifest)
+    if len(sys.argv) > 2:
+        optlevel = int(sys.argv[2])
+        # python2 only needs one or the other
+        if optlevel == 1 or (sys.version_info >= (3,) and optlevel > 0):
+            subprocess.check_call([sys.executable, '-O'] + sys.argv[:2])
+        if optlevel == 2:
+            subprocess.check_call([sys.executable, '-OO'] + sys.argv[:2])
diff --git a/test cases/common/252 install data structured/meson.build b/test cases/common/252 install data structured/meson.build
index 9d297943f..483474712 100644
--- a/test cases/common/252 install data structured/meson.build
+++ b/test cases/common/252 install data structured/meson.build
@@ -1,4 +1,4 @@
-project('install structured data')
+project('install structured data', default_options: ['python.bytecompile=-1'])
 
 install_data(
     'dir1/file1',
diff --git a/test cases/python/2 extmodule/meson.build b/test cases/python/2 extmodule/meson.build
index 8332afdd1..079463160 100644
--- a/test cases/python/2 extmodule/meson.build
+++ b/test cases/python/2 extmodule/meson.build
@@ -1,5 +1,5 @@
 project('Python extension module', 'c',
-  default_options : ['buildtype=release', 'werror=true'])
+  default_options : ['buildtype=release', 'werror=true', 'python.bytecompile=-1'])
 # Because Windows Python ships only with optimized libs,
 # we must build this project the same way.
 
diff --git a/test cases/python/7 install path/meson.build b/test cases/python/7 install path/meson.build
index 22e33c653..2cac6523c 100644
--- a/test cases/python/7 install path/meson.build
+++ b/test cases/python/7 install path/meson.build
@@ -1,7 +1,8 @@
 project('install path',
   default_options: [
+    'python.bytecompile=-1',
     'python.purelibdir=/pure',
-    'python.platlibdir=/plat'
+    'python.platlibdir=/plat',
   ]
 )