From b8f1045fe711f68fb65441e13d8ac9885500d4d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Fri, 14 May 2021 17:53:31 +0200 Subject: Revert "tools/make-directive-index: parallelize" This reverts commit a2031de849da52aa85b7e4326c0112ed7e5b5672. The patch itself seems OK, but it exposes a bug in lxml or libxml2-2.9.12 which was just released. This is being resolved in https://gitlab.gnome.org/GNOME/libxml2/-/issues/255, but it might be while. So let's revert this for now to unbreak our CI. Fixes #19601. --- tools/make-directive-index.py | 63 +++++++++++++++---------------------------- 1 file changed, 21 insertions(+), 42 deletions(-) (limited to 'tools/make-directive-index.py') diff --git a/tools/make-directive-index.py b/tools/make-directive-index.py index 6aaad6700c..bbdc5572f5 100755 --- a/tools/make-directive-index.py +++ b/tools/make-directive-index.py @@ -4,21 +4,18 @@ import sys import collections import re -import concurrent.futures from xml_helper import xml_parse, xml_print, tree +from copy import deepcopy COLOPHON = '''\ This index contains {count} entries in {sections} sections, referring to {pages} individual manual pages. ''' -def _extract_directives(page, names): - directive_groups = {name:collections.defaultdict(set) for name in names} - +def _extract_directives(directive_groups, formatting, page): t = xml_parse(page) section = t.find('./refmeta/manvolnum').text pagename = t.find('./refmeta/refentrytitle').text - formatting = {} storopt = directive_groups['options'] for variablelist in t.iterfind('.//variablelist'): @@ -34,7 +31,7 @@ def _extract_directives(page, names): if text.startswith('-'): # for options, merge options with and without mandatory arg text = text.partition('=')[0] - stor[text].add((pagename, section)) + stor[text].append((pagename, section)) if text not in formatting: # use element as formatted display if name.text[-1] in "= '": @@ -45,7 +42,7 @@ def _extract_directives(page, names): formatting[text] = name extra = variablelist.attrib.get('extra-ref') if extra: - stor[extra].add((pagename, section)) + stor[extra].append((pagename, section)) if extra not in formatting: elt = tree.Element("varname") elt.text= extra @@ -71,13 +68,13 @@ def _extract_directives(page, names): name.text = text if text.endswith('/'): text = text[:-1] - storfile[text].add((pagename, section)) + storfile[text].append((pagename, section)) if text not in formatting: # use element as formatted display formatting[text] = name else: text = ' '.join(name.itertext()) - storfile[text].add((pagename, section)) + storfile[text].append((pagename, section)) formatting[text] = name storfile = directive_groups['constants'] @@ -87,7 +84,7 @@ def _extract_directives(page, names): name.tail = '' if name.text.startswith('('): # a cast, strip it name.text = name.text.partition(' ')[2] - storfile[name.text].add((pagename, section)) + storfile[name.text].append((pagename, section)) formatting[name.text] = name storfile = directive_groups['specifiers'] @@ -96,30 +93,18 @@ def _extract_directives(page, names): continue if name.attrib.get('index') == 'false': continue - storfile[name.text].add((pagename, section)) + storfile[name.text].append((pagename, section)) formatting[name.text] = name for name in t.iterfind(".//literal[@class='specifiers']"): - storfile[name.text].add((pagename, section)) + storfile[name.text].append((pagename, section)) formatting[name.text] = name - # Serialize to allow pickling - formatting = {name:xml_print(value) for name, value in formatting.items()} - - return directive_groups, formatting - -def extract_directives(arg): - page, names = arg - try: - return _extract_directives(page, names) - except Exception: - raise ValueError("Failed to process {}".format(page)) - def _make_section(template, name, directives, formatting): varlist = template.find(".//*[@id='{}']".format(name)) for varname, manpages in sorted(directives.items()): entry = tree.SubElement(varlist, 'varlistentry') term = tree.SubElement(entry, 'term') - display = tree.fromstring(formatting[varname]) + display = deepcopy(formatting[varname]) term.append(display) para = tree.SubElement(tree.SubElement(entry, 'listitem'), 'para') @@ -169,26 +154,20 @@ def make_page(template_path, xml_files): "Extract directives from xml_files and return XML index tree." template = xml_parse(template_path) names = [vl.get('id') for vl in template.iterfind('.//variablelist')] - - with concurrent.futures.ProcessPoolExecutor() as pool: - args = ((xml_file, names) for xml_file in xml_files) - results = list(pool.map(extract_directives, args)) - - directive_groups = {name:collections.defaultdict(set) for name in names} + directive_groups = {name:collections.defaultdict(list) + for name in names} formatting = {} - for d_g, f in reversed(results): - for group, mapping in d_g.items(): - for name, value in mapping.items(): - directive_groups[group][name].update(value) - - formatting.update(f) + for page in xml_files: + try: + _extract_directives(directive_groups, formatting, page) + except Exception: + raise ValueError("failed to process " + page) return _make_page(template, directive_groups, formatting) -def main(output, template_path, *xml_files): - with open(output, 'wb') as f: +if __name__ == '__main__': + with open(sys.argv[1], 'wb') as f: + template_path = sys.argv[2] + xml_files = sys.argv[3:] xml = make_page(template_path, xml_files) f.write(xml_print(xml)) - -if __name__ == '__main__': - main(*sys.argv[1:]) -- cgit v1.2.1