diff options
author | Georg Brandl <georg@python.org> | 2014-09-22 18:16:53 +0200 |
---|---|---|
committer | Georg Brandl <georg@python.org> | 2014-09-22 18:16:53 +0200 |
commit | 19e084ca66cf5e37ea57e9e18c8eefdff36a02bd (patch) | |
tree | 5de1998ab591e6a0d24c159eebeb52aa169a8a1b | |
parent | 0488306cad87d948c89f00f2968eabc3dc5caec0 (diff) | |
download | sphinx-19e084ca66cf5e37ea57e9e18c8eefdff36a02bd.tar.gz |
Add a possibility to later execute finishing-up tasks in parallel.
-rw-r--r-- | sphinx/application.py | 9 | ||||
-rw-r--r-- | sphinx/builders/__init__.py | 66 | ||||
-rw-r--r-- | sphinx/builders/html.py | 49 |
3 files changed, 74 insertions, 50 deletions
diff --git a/sphinx/application.py b/sphinx/application.py index ff4d45b0..8fc378c0 100644 --- a/sphinx/application.py +++ b/sphinx/application.py @@ -248,6 +248,15 @@ class Sphinx(object): else: self.builder.compile_update_catalogs() self.builder.build_update() + + status = (self.statuscode == 0 + and 'succeeded' or 'finished with problems') + if self._warncount: + self.info(bold('build %s, %s warning%s.' % + (status, self._warncount, + self._warncount != 1 and 's' or ''))) + else: + self.info(bold('build %s.' % status)) except Exception as err: # delete the saved env to force a fresh build next time envfile = path.join(self.doctreedir, ENV_PICKLE_FILENAME) diff --git a/sphinx/builders/__init__.py b/sphinx/builders/__init__.py index 33120a2b..9edc3377 100644 --- a/sphinx/builders/__init__.py +++ b/sphinx/builders/__init__.py @@ -23,7 +23,8 @@ from docutils import nodes from sphinx.util import i18n, path_stabilize from sphinx.util.osutil import SEP, relative_uri, find_catalog from sphinx.util.console import bold, darkgreen -from sphinx.util.parallel import ParallelChunked, parallel_available +from sphinx.util.parallel import ParallelChunked, ParallelTasks, SerialTasks, \ + parallel_available # side effect: registers roles and directives from sphinx import roles @@ -70,6 +71,10 @@ class Builder(object): # images that need to be copied over (source -> dest) self.images = {} + # these get set later + self.parallel_ok = False + self.finish_tasks = None + # load default translator class self.translator_class = app._translators.get(self.name) @@ -277,20 +282,33 @@ class Builder(object): if docnames and docnames != ['__all__']: docnames = set(docnames) & self.env.found_docs - # another indirection to support builders that don't build - # files individually + # determine if we can write in parallel + self.parallel_ok = False + if parallel_available and self.app.parallel > 1 and self.allow_parallel: + self.parallel_ok = True + for extname, md in self.app._extension_metadata.items(): + par_ok = md.get('parallel_write_safe', True) + if not par_ok: + self.app.warn('the %s extension is not safe for parallel ' + 'writing, doing serial read' % extname) + self.parallel_ok = False + break + + # create a task executor to use for misc. "finish-up" tasks + # if self.parallel_ok: + # self.finish_tasks = ParallelTasks(self.app.parallel) + # else: + # for now, just execute them serially + self.finish_tasks = SerialTasks() + + # write all "normal" documents (or everything for some builders) self.write(docnames, list(updated_docnames), method) # finish (write static files etc.) self.finish() - status = (self.app.statuscode == 0 - and 'succeeded' or 'finished with problems') - if self.app._warncount: - self.info(bold('build %s, %s warning%s.' % - (status, self.app._warncount, - self.app._warncount != 1 and 's' or ''))) - else: - self.info(bold('build %s.' % status)) + + # wait for all tasks + self.finish_tasks.join() def write(self, build_docnames, updated_docnames, method='update'): if build_docnames is None or build_docnames == ['__all__']: @@ -316,25 +334,13 @@ class Builder(object): warnings = [] self.env.set_warnfunc(lambda *args: warnings.append(args)) - # check for prerequisites to parallel build - # (parallel only works on POSIX, because the forking impl of - # multiprocessing is required) - if parallel_available and len(docnames) > 5 and self.app.parallel > 1 \ - and self.allow_parallel: - for extname, md in self.app._extension_metadata.items(): - par_ok = md.get('parallel_write_safe', True) - if not par_ok: - self.app.warn('the %s extension is not safe for parallel ' - 'writing, doing serial read' % extname) - break - else: # means no break, means everything is safe - # number of subprocesses is parallel-1 because the main process - # is busy loading doctrees and doing write_doc_serialized() - self._write_parallel(sorted(docnames), warnings, - nproc=self.app.parallel - 1) - self.env.set_warnfunc(self.warn) - return - self._write_serial(sorted(docnames), warnings) + if self.parallel_ok: + # number of subprocesses is parallel-1 because the main process + # is busy loading doctrees and doing write_doc_serialized() + self._write_parallel(sorted(docnames), warnings, + nproc=self.app.parallel - 1) + else: + self._write_serial(sorted(docnames), warnings) self.env.set_warnfunc(self.warn) def _write_serial(self, docnames, warnings): diff --git a/sphinx/builders/html.py b/sphinx/builders/html.py index 8f853310..164d81b2 100644 --- a/sphinx/builders/html.py +++ b/sphinx/builders/html.py @@ -443,12 +443,19 @@ class StandaloneHTMLBuilder(Builder): self.index_page(docname, doctree, title) def finish(self): - self.info(bold('writing additional files...'), nonl=1) + self.finish_tasks.add_task(self.gen_indices) + self.finish_tasks.add_task(self.gen_additional_pages) + self.finish_tasks.add_task(self.copy_image_files) + self.finish_tasks.add_task(self.copy_download_files) + self.finish_tasks.add_task(self.copy_static_files) + self.finish_tasks.add_task(self.copy_extra_files) + self.finish_tasks.add_task(self.write_buildinfo) - # pages from extensions - for pagelist in self.app.emit('html-collect-pages'): - for pagename, context, template in pagelist: - self.handle_page(pagename, context, template) + # dump the search index + self.handle_finish() + + def gen_indices(self): + self.info(bold('generating indices...'), nonl=1) # the global general index if self.get_builder_config('use_index', 'html'): @@ -457,16 +464,27 @@ class StandaloneHTMLBuilder(Builder): # the global domain-specific indices self.write_domain_indices() - # the search page - if self.name != 'htmlhelp': - self.info(' search', nonl=1) - self.handle_page('search', {}, 'search.html') + self.info() + + def gen_additional_pages(self): + self.info(bold('writing additional pages...'), nonl=1) + + # pages from extensions + for pagelist in self.app.emit('html-collect-pages'): + for pagename, context, template in pagelist: + self.handle_page(pagename, context, template) # additional pages from conf.py for pagename, template in self.config.html_additional_pages.items(): self.info(' '+pagename, nonl=1) self.handle_page(pagename, {}, template) + # the search page + if self.name != 'htmlhelp': + self.info(' search', nonl=1) + self.handle_page('search', {}, 'search.html') + + # the opensearch xml file if self.config.html_use_opensearch and self.name != 'htmlhelp': self.info(' opensearch', nonl=1) fn = path.join(self.outdir, '_static', 'opensearch.xml') @@ -474,15 +492,6 @@ class StandaloneHTMLBuilder(Builder): self.info() - self.copy_image_files() - self.copy_download_files() - self.copy_static_files() - self.copy_extra_files() - self.write_buildinfo() - - # dump the search index - self.handle_finish() - def write_genindex(self): # the total count of lines for each index letter, used to distribute # the entries into two columns @@ -786,8 +795,8 @@ class StandaloneHTMLBuilder(Builder): copyfile(self.env.doc2path(pagename), source_name) def handle_finish(self): - self.dump_search_index() - self.dump_inventory() + self.finish_tasks.add_task(self.dump_search_index) + self.finish_tasks.add_task(self.dump_inventory) def dump_inventory(self): self.info(bold('dumping object inventory... '), nonl=True) |