diff options
Diffstat (limited to 'zephyr/zmake/zmake/jobserver.py')
| -rw-r--r-- | zephyr/zmake/zmake/jobserver.py | 149 |
1 files changed, 18 insertions, 131 deletions
diff --git a/zephyr/zmake/zmake/jobserver.py b/zephyr/zmake/zmake/jobserver.py index c1c9538b35..a3d6287da2 100644 --- a/zephyr/zmake/zmake/jobserver.py +++ b/zephyr/zmake/zmake/jobserver.py @@ -3,17 +3,13 @@ # found in the LICENSE file. """Module for job counters, limiting the amount of concurrent executions.""" -import fcntl -import functools import logging import multiprocessing import os import re import select -import selectors import shlex import subprocess -import sys import zmake @@ -30,8 +26,7 @@ class JobHandle: pass def __exit__(self, exc_type, exc_value, traceback): - if self.release_func: - self.release_func(*self.args, **self.kwargs) + self.release_func(*self.args, **self.kwargs) class JobClient: @@ -46,16 +41,6 @@ class JobClient: """Get the environment variables necessary to share the job server.""" return {} - @staticmethod - def is_sequential(): - """Returns True if the jobserver is using -j1.""" - return False - - @staticmethod - def pass_fds(): - """Returns the file descriptors that should be passed to subprocesses.""" - return [] - def popen(self, argv, **kwargs): """Start a process using subprocess.Popen @@ -68,9 +53,7 @@ class JobClient: # the bare minimum (PATH only). This prevents us from building obscure # dependencies on the environment variables. kwargs.setdefault("env", {"PATH": "/bin:/usr/bin"}) - kwargs.setdefault("pass_fds", []) kwargs["env"].update(self.env()) - kwargs["pass_fds"] += self.pass_fds() logger = logging.getLogger(self.__class__.__name__) logger.debug( @@ -79,53 +62,17 @@ class JobClient: " " if kwargs["env"] else "", zmake.util.repr_command(argv), ) - return subprocess.Popen( # pylint:disable=consider-using-with - argv, **kwargs - ) + return subprocess.Popen(argv, **kwargs) class GNUMakeJobClient(JobClient): - """A job client for GNU make. + """A job client for GNU make.""" - A client of jobserver is allowed to run 1 job without contacting the - jobserver, so maintain an optional self._internal_pipe to hold that - job. - """ - - def __init__(self, inheritable_pipe, jobs, internal_jobs=0, makeflags=None): - self._makeflags = makeflags - self._inheritable_pipe = inheritable_pipe - self.jobs = jobs - self._selector = selectors.DefaultSelector() - if internal_jobs: - self._internal_pipe = os.pipe() - os.write(self._internal_pipe[1], b"+" * internal_jobs) - os.set_blocking(self._internal_pipe[0], False) - self._selector.register( - self._internal_pipe[0], - selectors.EVENT_READ, - self._internal_pipe[1], - ) - else: - self._internal_pipe = None - if self._inheritable_pipe is not None: - os.set_blocking(self._inheritable_pipe[0], False) - self._selector.register( - self._inheritable_pipe[0], - selectors.EVENT_READ, - self._inheritable_pipe[1], - ) - - def __del__(self): - if self._inheritable_pipe: - os.close(self._inheritable_pipe[0]) - os.close(self._inheritable_pipe[1]) - if self._internal_pipe: - os.close(self._internal_pipe[0]) - os.close(self._internal_pipe[1]) + def __init__(self, read_fd, write_fd): + self._pipe = [read_fd, write_fd] @classmethod - def from_environ(cls, env=None, jobs=0): + def from_environ(cls, env=None): """Create a job client from an environment with the MAKEFLAGS variable. If we are started under a GNU Make Job Server, we can search @@ -134,57 +81,22 @@ class GNUMakeJobClient(JobClient): respectively. If we don't find this environment variable (or the string inside of it), this will raise an OSError. - The specification for MAKEFLAGS is: - * If the first char is "n", this is a dry run, just exit. - * If the flags contains -j1, go to sequential mode. - * If the flags contains --jobserver-auth=R,W AND those file - descriptors are valid, use the jobserver. Otherwise output a - warning. - Args: env: Optionally, the environment to search. - jobs: The number of jobs set by the user on the command line. Returns: - A GNUMakeJobClient configured appropriately or None if there is - no MAKEFLAGS environment variable. + A GNUMakeJobClient configured appropriately. """ if env is None: env = os.environ makeflags = env.get("MAKEFLAGS") if not makeflags: - return None + raise OSError("MAKEFLAGS is not set in the environment") match = re.search(r"--jobserver-auth=(\d+),(\d+)", makeflags) - if match: - pipe = [int(x) for x in match.groups()] - if jobs: - pipe = None - logging.warning( - "-jN forced on command line; ignoring GNU make jobserver" - ) - else: - try: - # Use F_GETFD to see if file descriptors are valid - fcntl.fcntl(pipe[0], fcntl.F_GETFD) - fcntl.fcntl(pipe[1], fcntl.F_GETFD) - logging.info("using GNU make jobserver") - except OSError: - pipe = None - logging.warning( - "No file descriptors; ignoring GNU make jobserver" - ) - else: - pipe = None - if not jobs: - match = re.search(r"-j(\d+)", makeflags) - if match: - jobs = int(match.group(1)) - if jobs == 1: - logging.info("Running in sequential mode (-j1)") - if makeflags[0] == "n": - logging.info("MAKEFLAGS contained dry-run flag") - sys.exit(0) - return cls(pipe, jobs, internal_jobs=1, makeflags=makeflags) + if not match: + raise OSError("MAKEFLAGS did not contain jobserver flags") + read_fd, write_fd = map(int, match.groups()) + return cls(read_fd, write_fd) def get_job(self): """Claim a job. @@ -192,38 +104,12 @@ class GNUMakeJobClient(JobClient): Returns: A JobHandle object. """ - while True: - ready_items = self._selector.select() - if len(ready_items) > 0: - read_fd = ready_items[0][0].fd - write_fd = ready_items[0][0].data - try: - byte = os.read(read_fd, 1) - return JobHandle( - functools.partial(os.write, write_fd, byte) - ) - except BlockingIOError: - pass + byte = os.read(self._pipe[0], 1) + return JobHandle(lambda: os.write(self._pipe[1], byte)) def env(self): """Get the environment variables necessary to share the job server.""" - if self._makeflags: - return {"MAKEFLAGS": self._makeflags} - flag = "" - if self.jobs: - flag += f" -j{self.jobs}" - if self.jobs != 1 and self._inheritable_pipe is not None: - flag += " --jobserver-auth={},{}".format(*self._inheritable_pipe) - return {"MAKEFLAGS": flag} - - def is_sequential(self): - return self.jobs == 1 - - def pass_fds(self): - """Returns the file descriptors that should be passed to subprocesses.""" - if self.jobs != 1 and self._inheritable_pipe is not None: - return self._inheritable_pipe - return [] + return {"MAKEFLAGS": "--jobserver-auth={},{}".format(*self._pipe)} class GNUMakeJobServer(GNUMakeJobClient): @@ -234,10 +120,11 @@ class GNUMakeJobServer(GNUMakeJobClient): """ def __init__(self, jobs=0): + [read_fd, write_fd] = os.pipe() + super().__init__(read_fd, write_fd) if not jobs: jobs = multiprocessing.cpu_count() elif jobs > select.PIPE_BUF: jobs = select.PIPE_BUF - super().__init__(os.pipe(), jobs) - os.write(self._inheritable_pipe[1], b"+" * jobs) + os.write(self._pipe[1], b"+" * jobs) |