diff options
Diffstat (limited to 'sphinx/util/parallel.py')
| -rw-r--r-- | sphinx/util/parallel.py | 14 |
1 files changed, 9 insertions, 5 deletions
diff --git a/sphinx/util/parallel.py b/sphinx/util/parallel.py index ed73ee4d6..9fa1e0c3e 100644 --- a/sphinx/util/parallel.py +++ b/sphinx/util/parallel.py @@ -100,12 +100,13 @@ class ParallelTasks: args=(psend, task_func, arg)) self._procs[tid] = proc self._precvsWaiting[tid] = precv - self._join_one() + _ = self._join_one() def join(self) -> None: try: while self._pworking: - self._join_one() + if not self._join_one(): + time.sleep(0.02) except Exception: # shutdown other child processes on failure self.terminate() @@ -119,7 +120,8 @@ class ParallelTasks: self._precvs.pop(tid) self._pworking -= 1 - def _join_one(self) -> None: + def _join_one(self) -> bool: + joined_any = False for tid, pipe in self._precvs.items(): if pipe.poll(): exc, logs, result = pipe.recv() @@ -131,15 +133,17 @@ class ParallelTasks: self._procs[tid].join() self._precvs.pop(tid) self._pworking -= 1 + joined_any = True break - else: - time.sleep(0.02) + while self._precvsWaiting and self._pworking < self.nproc: newtid, newprecv = self._precvsWaiting.popitem() self._precvs[newtid] = newprecv self._procs[newtid].start() self._pworking += 1 + return joined_any + def make_chunks(arguments: Sequence[str], nproc: int, maxbatch: int = 10) -> List[Any]: # determine how many documents to read in one go |