- started to implement new XML tools

- prepared introduction of SCons XSD namespace

4 files changed, 404 insertions, 439 deletions

diff --git a/bin/SConsDoc.py b/bin/SConsDoc.py
index 4927dc04..8889923f 100644
--- a/bin/SConsDoc.py
+++ b/bin/SConsDoc.py
@@ -112,7 +112,173 @@ import imp
 import os.path
 import re
 import sys
-import xml.sax.handler
+
+# Do we have libxml2/libxslt/lxml?
+has_libxml2 = True
+has_lxml = True
+try:
+    import libxml2
+    import libxslt
+except:
+    has_libxml2 = False
+try:
+    import lxml
+except:
+    has_lxml = False
+
+
+re_entity = re.compile("\&([^;]+);")
+
+entity_header = """<!DOCTYPE sconsdoc [
+<!ENTITY % scons SYSTEM 'scons.mod'>
+%scons;
+<!ENTITY % builders-mod SYSTEM 'builders.mod'>
+%builders-mod;
+<!ENTITY % functions-mod SYSTEM 'functions.mod'>
+%functions-mod;
+<!ENTITY % tools-mod SYSTEM 'tools.mod'>
+%tools-mod;
+<!ENTITY % variables-mod SYSTEM 'variables.mod'>
+%variables-mod;
+]>"""
+
+# Namespace for the SCons Docbook XSD
+dbxsd="http://www.scons.org/dbxsd/v1.0"
+
+xml_header = """<?xml version="1.0" encoding="UTF-8"?>
+<!--
+__COPYRIGHT__
+
+This file is processed by the bin/SConsDoc.py module.
+See its __doc__ string for a discussion of the format.
+-->
+
+%s
+
+<sconsdoc xmlns="http://www.scons.org/dbxsd/v1.0"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="%s scons.xsd">
+""" % (entity_header, dbxsd)
+
+def remove_entities(content):
+    # Cut out entity inclusions
+    content = content.replace(entity_header, "")
+    # Cut out entities themselves
+    content = re_entity.sub(lambda match: match.group(1), content)
+    
+    return content
+
+default_xsd = os.path.join('doc','xsd','scons.xsd')
+
+def validate_xml(fpath, xmlschema_context):
+    if not has_libxml2:
+        # At the moment we prefer libxml2 over lxml, the latter can lead
+        # to conflicts when installed together with libxml2.
+        if has_lxml:
+            # Use lxml
+            from lxml import etree
+            xmlschema = etree.XMLSchema(xmlschema_context)
+            doc = etree.parse(fpath)
+            try:
+                xmlschema.assertValid(doc)
+            except:
+                return False
+            return True
+        else:
+            # Try xmllint as a last fallback
+            try:
+                import subprocess
+                p = subprocess.Popen(['xmllint','--noout','--noent','--schema',default_xsd,fpath],
+                                     stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                sout, serr = p.communicate()
+                if serr and not 'validates' in serr:
+                    print serr
+                    return False
+                
+                return True
+                
+            except:
+                print "Can't validate %s! Neither lxml/libxml2, nor xmllint found." % fpath
+                return False
+
+    # Read file and resolve entities
+    doc = libxml2.readFile(fpath, None, libxml2.XML_PARSE_NOENT)
+    err = xmlschema_context.schemaValidateDoc(doc)
+    # Cleanup
+    doc.freeDoc()
+    
+    if err:
+        # TODO: print error message "Haha",err
+        return False
+        
+    return True
+
+perc="%"
+
+def validate_all_xml(dpath='src', xsdfile=default_xsd):
+    xmlschema_context = None
+    if not has_libxml2:
+        # At the moment we prefer libxml2 over lxml, the latter can lead
+        # to conflicts when installed together with libxml2.
+        if has_lxml:
+            # Use lxml
+            from lxml import etree
+            xmlschema_context = etree.parse(xsdfile)
+    else:
+        # Use libxml2 and prepare the schema validation context
+        ctxt = libxml2.schemaNewParserCtxt(xsdfile)
+        schema = ctxt.schemaParse()
+        del ctxt
+        xmlschema_context = schema.schemaNewValidCtxt()
+    
+    fpaths = []
+    for path, dirs, files in os.walk(dpath):
+        for f in files:
+            if f.endswith('.xml'):
+                fp = os.path.join(path, f)
+                fpaths.append(fp)
+                
+    fails = []
+    for idx, fp in enumerate(fpaths):
+        fpath = os.path.join(path, f)
+        print "%.2f%s (%d/%d) %s" % (float(idx+1)*100.0/float(len(fpaths)),
+                                     perc, idx+1, len(fpaths),fp)
+                                              
+        if not validate_xml(fp, xmlschema_context):
+            fails.append(fp)
+            continue
+
+    if has_libxml2:
+        # Cleanup
+        del xmlschema_context
+        del schema
+
+    if fails:
+        return False
+    
+    return True
+
+try:
+  from lxml import etree
+except ImportError:
+  try:
+    # Python 2.5
+    import xml.etree.cElementTree as etree
+  except ImportError:
+    try:
+      # Python 2.5
+      import xml.etree.ElementTree as etree
+    except ImportError:
+      try:
+        # normal cElementTree install
+        import cElementTree as etree
+      except ImportError:
+        try:
+          # normal ElementTree install
+          import elementtree.ElementTree as etree
+        except ImportError:
+          print("Failed to import ElementTree from any known place")
+          sys.exit(1)
 
 class Item(object):
     def __init__(self, name):
@@ -121,8 +287,8 @@ class Item(object):
         if self.sort_name[0] == '_':
             self.sort_name = self.sort_name[1:]
         self.summary = []
-        self.sets = None
-        self.uses = None
+        self.sets = []
+        self.uses = []
     def cmp_name(self, name):
         if name[0] == '_':
             name = name[1:]
@@ -175,201 +341,78 @@ class Arguments(object):
     def append(self, data):
         self.body.append(data)
 
-class Summary(object):
+class SConsDocHandler(object):
     def __init__(self):
-        self.body = []
-        self.collect = []
-    def append(self, data):
-        self.collect.append(data)
-    def end_para(self):
-        text = ''.join(self.collect)
-        paras = text.split('\n\n')
-        if paras == ['\n']:
-            return
-        if paras[0] == '':
-            self.body.append('\n')
-            paras = paras[1:]
-            paras[0] = '\n' + paras[0]
-        if paras[-1] == '':
-            paras = paras[:-1]
-            paras[-1] = paras[-1] + '\n'
-            last = '\n'
-        else:
-            last = None
-        sep = None
-        for p in paras:
-            c = Chunk("para", p)
-            if sep:
-                self.body.append(sep)
-            self.body.append(c)
-            sep = '\n'
-        if last:
-            self.body.append(last)
-    def begin_chunk(self, chunk):
-        self.end_para()
-        self.collect = chunk
-    def end_chunk(self):
-        self.body.append(self.collect)
-        self.collect = []
-
-class SConsDocHandler(xml.sax.handler.ContentHandler,
-                      xml.sax.handler.ErrorHandler):
-    def __init__(self):
-        self._start_dispatch = {}
-        self._end_dispatch = {}
-        keys = list(self.__class__.__dict__.keys())
-        start_tag_method_names = [k for k in keys if k[:6] == 'start_']
-        end_tag_method_names = [k for k in keys if k[:4] == 'end_']
-        for method_name in start_tag_method_names:
-            tag = method_name[6:]
-            self._start_dispatch[tag] = getattr(self, method_name)
-        for method_name in end_tag_method_names:
-            tag = method_name[4:]
-            self._end_dispatch[tag] = getattr(self, method_name)
-        self.stack = []
-        self.collect = []
-        self.current_object = []
         self.builders = {}
         self.functions = {}
         self.tools = {}
         self.cvars = {}
 
-    def startElement(self, name, attrs):
-        try:
-            start_element_method = self._start_dispatch[name]
-        except KeyError:
-            self.characters('<%s>' % name)
-        else:
-            start_element_method(attrs)
+    def parseText(self, root):
+        txt = ""
+        for e in root.childNodes:
+            if (e.nodeType == e.TEXT_NODE):
+                txt += e.data
+        return txt
 
-    def endElement(self, name):
-        try:
-            end_element_method = self._end_dispatch[name]
-        except KeyError:
-            self.characters('</%s>' % name)
-        else:
-            end_element_method()
-
-    #
-    #
-    def characters(self, chars):
-        self.collect.append(chars)
-
-    def begin_collecting(self, chunk):
-        self.collect = chunk
-    def end_collecting(self):
-        self.collect = []
-
-    def begin_chunk(self):
-        pass
-    def end_chunk(self):
-        pass
-
-    #
-    #
-    #
-
-    def begin_xxx(self, obj):
-        self.stack.append(self.current_object)
-        self.current_object = obj
-    def end_xxx(self):
-        self.current_object = self.stack.pop()
-
-    #
-    #
-    #
-    def start_scons_doc(self, attrs):
-        pass
-    def end_scons_doc(self):
-        pass
-
-    def start_builder(self, attrs):
-        name = attrs.get('name')
-        try:
-            builder = self.builders[name]
-        except KeyError:
-            builder = Builder(name)
-            self.builders[name] = builder
-        self.begin_xxx(builder)
-    def end_builder(self):
-        self.end_xxx()
-
-    def start_scons_function(self, attrs):
-        name = attrs.get('name')
-        try:
-            function = self.functions[name]
-        except KeyError:
-            function = Function(name)
-            self.functions[name] = function
-        self.begin_xxx(function)
-    def end_scons_function(self):
-        self.end_xxx()
-
-    def start_tool(self, attrs):
-        name = attrs.get('name')
-        try:
-            tool = self.tools[name]
-        except KeyError:
-            tool = Tool(name)
-            self.tools[name] = tool
-        self.begin_xxx(tool)
-    def end_tool(self):
-        self.end_xxx()
-
-    def start_cvar(self, attrs):
-        name = attrs.get('name')
+    def parseItems(self, domelem):
+        items = []
+
+        for i in domelem.iterchildren(tag="item"):
+            items.append(self.parseText(i))
+
+        return items
+
+    def parseUsesSets(self, domelem):
+        uses = []
+        sets = []
+
+        for u in domelem.iterchildren(tag="uses"):
+            uses.extend(self.parseItems(u))
+        for s in domelem.iterchildren(tag="sets"):
+            sets.extend(self.parseItems(s))
+        
+        return sorted(uses), sorted(sets)
+
+    def parseInstance(self, domelem, map, Class):
+        name = domelem.attrib.get('name','unknown')
         try:
-            cvar = self.cvars[name]
+            instance = map[name]
         except KeyError:
-            cvar = ConstructionVariable(name)
-            self.cvars[name] = cvar
-        self.begin_xxx(cvar)
-    def end_cvar(self):
-        self.end_xxx()
-
-    def start_arguments(self, attrs):
-        arguments = Arguments(attrs.get('signature', "both"))
-        self.current_object.arguments.append(arguments)
-        self.begin_xxx(arguments)
-        self.begin_collecting(arguments)
-    def end_arguments(self):
-        self.end_xxx()
-
-    def start_summary(self, attrs):
-        summary = Summary()
-        self.current_object.summary = summary
-        self.begin_xxx(summary)
-        self.begin_collecting(summary)
-    def end_summary(self):
-        self.current_object.end_para()
-        self.end_xxx()
-
-    def start_example(self, attrs):
-        example = Chunk("programlisting")
-        self.current_object.begin_chunk(example)
-    def end_example(self):
-        self.current_object.end_chunk()
-
-    def start_uses(self, attrs):
-        self.begin_collecting([])
-    def end_uses(self):
-        self.current_object.uses = sorted(''.join(self.collect).split())
-        self.end_collecting()
-
-    def start_sets(self, attrs):
-        self.begin_collecting([])
-    def end_sets(self):
-        self.current_object.sets = sorted(''.join(self.collect).split())
-        self.end_collecting()
-
-    # Stuff for the ErrorHandler portion.
-    def error(self, exception):
-        linenum = exception._linenum - self.preamble_lines
-        sys.stderr.write('%s:%d:%d: %s (error)\n' % (self.filename, linenum, exception._colnum, ''.join(exception.args)))
-
-    def fatalError(self, exception):
-        linenum = exception._linenum - self.preamble_lines
-        sys.stderr.write('%s:%d:%d: %s (fatalError)\n' % (self.filename, linenum, exception._colnum, ''.join(exception.args)))
+            instance = Class(name)
+            map[name] = instance
+        uses, sets = self.parseUsesSets(domelem)
+        instance.uses.extend(uses)
+        instance.sets.extend(sets)
+        # Parse summary and function blobs
+        
+    def parseDomtree(self, root):    
+        # Process Builders
+        for b in root.iterchildren(tag="{%s}builder" % dbxsd):
+            self.parseInstance(b, self.builders, Builder)
+        # Process Functions
+        for f in root.iterchildren(tag="{%s}scons_function" % dbxsd):
+            self.parseInstance(f, self.functions, Function)
+        # Process Tools
+        for t in root.iterchildren(tag="{%s}tool" % dbxsd):
+            self.parseInstance(t, self.tools, Tool)
+        # Process CVars
+        for c in root.iterchildren(tag="{%s}cvar" % dbxsd):
+            self.parseInstance(c, self.cvars, ConstructionVariable)
+        
+    def parseContent(self, content, include_entities=True):
+        if not include_entities:
+            content = remove_entities(content)
+        # Create domtree from given content string
+        root = etree.fromstring(content)
+        # Parse it
+        self.parseDomtree(root)
+
+    def parseXmlFile(self, fpath):
+        # Create domtree from file
+        domtree = etree.parse(fpath)
+        # Parse it
+        self.parseDomtree(domtree.getroot())
 
     def set_file_info(self, filename, preamble_lines):
         self.filename = filename
diff --git a/bin/docs-update-generated.py b/bin/docs-update-generated.py
new file mode 100644
index 00000000..2a78b9ff
--- /dev/null
+++ b/bin/docs-update-generated.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+#
+# Searches through the whole source tree and updates
+# the generated *.gen/*.mod files in the docs folder, keeping all
+# documentation for the tools, builders and functions...
+# as well as the entity declarations for them.
+# Uses scons-proc.py under the hood...
+#
+
+import os
+
+# Directory where all generated files are stored
+gen_folder = 'doc/generated'
+
+def argpair(key):
+    """ Return the argument pair *.gen,*.mod for the given key. """
+    arg = '%s,%s' % (os.path.join(gen_folder,'%s.gen' % key),
+                     os.path.join(gen_folder,'%s.mod' % key))
+    
+    return arg
+
+def generate_all():
+    """ Scan for XML files in the src directory and call scons-proc.py
+        to generate the *.gen/*.mod files from it.
+    """
+    flist = []
+    for path, dirs, files in os.walk('src'):
+        for f in files:
+            if f.endswith('.xml'):
+                fpath = os.path.join(path, f)
+                flist.append(fpath)
+
+    if flist:
+        # Does the destination folder exist
+        if not os.path.isdir(gen_folder):
+            try:
+                os.makedirs(gen_folder)
+            except:
+                print "Couldn't create destination folder %s! Exiting..." % gen_folder
+                return
+        # Call scons-proc.py
+        os.system('python bin/scons-proc.py -b %s -f %s -t %s -v %s %s' %
+                  (argpair('builders'), argpair('functions'),
+                   argpair('tools'), argpair('variables'), ' '.join(flist)))
+    
+    
+if __name__ == "__main__":
+    generate_all()
diff --git a/bin/docs-validate.py b/bin/docs-validate.py
new file mode 100644
index 00000000..2bc70a98
--- /dev/null
+++ b/bin/docs-validate.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+#
+# Searches through the whole source tree and validates all
+# documentation files against our own XSD in docs/xsd.
+# Additionally, it rewrites all files such that the XML gets
+# pretty-printed in a consistent way. This is done to ensure that
+# merging and diffing doesn't get too hard when people start to
+# use different XML editors...
+#
+
+import SConsDoc
+
+if __name__ == "__main__":
+    if SConsDoc.validate_all_xml():
+        print "OK"
+    else:
+        print "Validation failed! Please correct the errors above and try again."
diff --git a/bin/scons-proc.py b/bin/scons-proc.py
index 1f537c71..7a56a798 100644
--- a/bin/scons-proc.py
+++ b/bin/scons-proc.py
@@ -5,17 +5,15 @@
 # This script creates formatted lists of the Builders, functions, Tools
 # or construction variables documented in the specified XML files.
 #
-# Dependening on the options, the lists are output in either
+# Depending on the options, the lists are output in either
 # DocBook-formatted generated XML files containing the summary text
-# and/or .mod files contining the ENTITY definitions for each item,
-# or in man-page-formatted output.
+# and/or .mod files containing the ENTITY definitions for each item.
 #
 import getopt
 import os
 import re
 import string
 import sys
-import xml.sax
 try:
     from io import StringIO     # usable as of 2.6; takes unicode only
 except ImportError:
@@ -27,28 +25,26 @@ import SConsDoc
 base_sys_path = [os.getcwd() + '/build/test-tar-gz/lib/scons'] + sys.path
 
 helpstr = """\
-Usage: scons-proc.py [--man|--xml]
-                     [-b file(s)] [-f file(s)] [-t file(s)] [-v file(s)]
+Usage: scons-proc.py [-b file(s)] [-f file(s)] [-t file(s)] [-v file(s)]
                      [infile ...]
 Options:
   -b file(s)        dump builder information to the specified file(s)
   -f file(s)        dump function information to the specified file(s)
   -t file(s)        dump tool information to the specified file(s)
   -v file(s)        dump variable information to the specified file(s)
-  --man             print info in man page format, each -[btv] argument
-                    is a single file name
-  --xml             (default) print info in SML format, each -[btv] argument
-                    is a pair of comma-separated .gen,.mod file names
+  
+  Regard that each -[btv] argument is a pair of
+  comma-separated .gen,.mod file names.
+  
 """
 
 opts, args = getopt.getopt(sys.argv[1:],
                            "b:f:ht:v:",
                            ['builders=', 'help',
-                            'man', 'xml', 'tools=', 'variables='])
+                            'tools=', 'variables='])
 
 buildersfiles = None
 functionsfiles = None
-output_type = '--xml'
 toolsfiles = None
 variablesfiles = None
 
@@ -60,55 +56,29 @@ for o, a in opts:
     elif o in ['-h', '--help']:
         sys.stdout.write(helpstr)
         sys.exit(0)
-    elif o in ['--man', '--xml']:
-        output_type = o
     elif o in ['-t', '--tools']:
         toolsfiles = a
     elif o in ['-v', '--variables']:
         variablesfiles = a
 
-h = SConsDoc.SConsDocHandler()
-saxparser = xml.sax.make_parser()
-saxparser.setContentHandler(h)
-saxparser.setErrorHandler(h)
-
-xml_preamble = """\
-<?xml version="1.0"?>
-<scons_doc>
-"""
-
-xml_postamble = """\
-</scons_doc>
-"""
-
-for f in args:
-    _, ext = os.path.splitext(f)
-    if ext == '.py':
-        dir, _ = os.path.split(f)
-        if dir:
-            sys.path = [dir] + base_sys_path
-        module = SConsDoc.importfile(f)
-        h.set_file_info(f, len(xml_preamble.split('\n')))
-        try:
-            content = module.__scons_doc__
-        except AttributeError:
-            content = None
+def parse_docs(args, include_entities=True):
+    h = SConsDoc.SConsDocHandler()
+    for f in args:
+        if include_entities:
+            try:
+                h.parseXmlFile(f)
+            except:
+                sys.stderr.write("error in %s\n" % f)
+                raise
         else:
-            del module.__scons_doc__
-    else:
-        h.set_file_info(f, len(xml_preamble.split('\n')))
-        content = open(f).read()
-    if content:
-        content = content.replace('&', '&amp;')
-        # Strip newlines after comments so they don't turn into
-        # spurious paragraph separators.
-        content = content.replace('-->\n', '-->')
-        input = xml_preamble + content + xml_postamble
-        try:
-            saxparser.parse(StringIO(unicode(input)))
-        except:
-            sys.stderr.write("error in %s\n" % f)
-            raise
+            content = open(f).read()
+            if content:
+                try:
+                    h.parseContent(content, include_entities)
+                except:
+                    sys.stderr.write("error in %s\n" % f)
+                    raise
+    return h
 
 Warning = """\
 <!--
@@ -137,27 +107,35 @@ class SCons_XML(object):
         self.values = entries
         for k, v in kw.items():
             setattr(self, k, v)
+            
     def fopen(self, name):
         if name == '-':
             return sys.stdout
         return open(name, 'w')
-
-class SCons_XML_to_XML(SCons_XML):
+    
     def write(self, files):
         gen, mod = files.split(',')
-        g.write_gen(gen)
-        g.write_mod(mod)
+        self.write_gen(gen)
+        self.write_mod(mod)
+        
     def write_gen(self, filename):
         if not filename:
             return
+        # Try to split off .gen filename
+        if filename.count(','):
+            fl = filename.split(',')
+            filename = fl[0]
         f = self.fopen(filename)
         for v in self.values:
             f.write('\n<varlistentry id="%s%s">\n' %
                         (v.prefix, v.idfunc()))
             f.write('%s\n' % v.xml_term())
             f.write('<listitem>\n')
-            for chunk in v.summary.body:
-                f.write(str(chunk))
+            # TODO: write summary
+            f.write('<para>\n')
+            if v.summary:
+                pass
+            f.write('</para>\n')
             if v.sets:
                 s = ['&cv-link-%s;' % x for x in v.sets]
                 f.write('<para>\n')
@@ -170,143 +148,53 @@ class SCons_XML_to_XML(SCons_XML):
                 f.write('</para>\n')
             f.write('</listitem>\n')
             f.write('</varlistentry>\n')
+            
     def write_mod(self, filename):
-        description = self.values[0].description
+        try:
+            description = self.values[0].description
+        except:
+            description = ""
         if not filename:
             return
+        # Try to split off .mod filename
+        if filename.count(','):
+            fl = filename.split(',')
+            filename = fl[1]
         f = self.fopen(filename)
         f.write(Warning)
         f.write('\n')
         f.write(Regular_Entities_Header % description)
         f.write('\n')
         for v in self.values:
-            f.write('<!ENTITY %s%s "<%s>%s</%s>">\n' %
+            f.write('<!ENTITY %s%s "<%s xmlns=\'%s\'>%s</%s>">\n' %
                         (v.prefix, v.idfunc(),
-                         v.tag, v.entityfunc(), v.tag))
+                         v.tag, SConsDoc.dbxsd, v.entityfunc(), v.tag))
         if self.env_signatures:
             f.write('\n')
             for v in self.values:
-                f.write('<!ENTITY %senv-%s "<%s>env.%s</%s>">\n' %
+                f.write('<!ENTITY %senv-%s "<%s xmlns=\'%s\'>env.%s</%s>">\n' %
                             (v.prefix, v.idfunc(),
-                             v.tag, v.entityfunc(), v.tag))
+                             v.tag, SConsDoc.dbxsd, v.entityfunc(), v.tag))
         f.write('\n')
         f.write(Warning)
         f.write('\n')
         f.write(Link_Entities_Header % description)
         f.write('\n')
         for v in self.values:
-            f.write('<!ENTITY %slink-%s \'<link linkend="%s%s"><%s>%s</%s></link>\'>\n' %
+            f.write('<!ENTITY %slink-%s "<link linkend=\'%s%s\' xmlns=\'%s\'><%s>%s</%s></link>">\n' %
                         (v.prefix, v.idfunc(),
-                         v.prefix, v.idfunc(),
+                         v.prefix, v.idfunc(), SConsDoc.dbxsd,
                          v.tag, v.entityfunc(), v.tag))
         if self.env_signatures:
             f.write('\n')
             for v in self.values:
-                f.write('<!ENTITY %slink-env-%s \'<link linkend="%s%s"><%s>env.%s</%s></link>\'>\n' %
+                f.write('<!ENTITY %slink-env-%s "<link linkend=\'%s%s\' xmlns=\'%s\'><%s>env.%s</%s></link>">\n' %
                             (v.prefix, v.idfunc(),
-                             v.prefix, v.idfunc(),
+                             v.prefix, v.idfunc(), SConsDoc.dbxsd,
                              v.tag, v.entityfunc(), v.tag))
         f.write('\n')
         f.write(Warning)
 
-class SCons_XML_to_man(SCons_XML):
-    def write(self, filename):
-        """
-        Converts the contents of the specified filename from DocBook XML
-        to man page macros.
-
-        This does not do an intelligent job.  In particular, it doesn't
-        actually use the structured nature of XML to handle arbitrary
-        input.  Instead, we're using text replacement and regular
-        expression substitutions to convert observed patterns into the
-        macros we want.  To the extent that we're relatively consistent
-        with our input .xml, this works, but could easily break if handed
-        input that doesn't match these specific expectations.
-        """
-        if not filename:
-            return
-        f = self.fopen(filename)
-        chunks = []
-        for v in self.values:
-            chunks.extend(v.man_separator())
-            chunks.extend(v.initial_man_chunks())
-            chunks.extend(list(map(str, v.summary.body)))
-
-        body = ''.join(chunks)
-
-        # Simple transformation of examples into our defined macros for those.
-        body = body.replace('<programlisting>', '.ES')
-        body = body.replace('</programlisting>', '.EE')
-
-        # Replace groupings of <para> tags and surrounding newlines
-        # with single blank lines.
-        body = body.replace('\n</para>\n<para>\n', '\n\n')
-        body = body.replace('<para>\n', '')
-        body = body.replace('<para>', '\n')
-        body = body.replace('</para>\n', '')
-
-        # Convert <variablelist> and its child tags.
-        body = body.replace('<variablelist>\n', '.RS 10\n')
-        # Handling <varlistentry> needs to be rationalized and made
-        # consistent.  Right now, the <term> values map to arbitrary,
-        # ad-hoc idioms in the current man page.
-        body = re.compile(r'<varlistentry>\n<term><literal>([^<]*)</literal></term>\n<listitem>\n').sub(r'.TP 6\n.B \1\n', body)
-        body = re.compile(r'<varlistentry>\n<term><parameter>([^<]*)</parameter></term>\n<listitem>\n').sub(r'.IP \1\n', body)
-        body = re.compile(r'<varlistentry>\n<term>([^<]*)</term>\n<listitem>\n').sub(r'.HP 6\n.B \1\n', body)
-        body = body.replace('</listitem>\n', '')
-        body = body.replace('</varlistentry>\n', '')
-        body = body.replace('</variablelist>\n', '.RE\n')
-
-        # Get rid of unnecessary .IP macros, and unnecessary blank lines
-        # in front of .IP macros.
-        body = re.sub(r'\.EE\n\n+(?!\.IP)', '.EE\n.IP\n', body)
-        body = body.replace('\n.EE\n.IP\n.ES\n', '\n.EE\n\n.ES\n')
-        body = body.replace('\n.IP\n\'\\"', '\n\n\'\\"')
-
-        # Convert various named entities and tagged names to nroff
-        # in-line font conversions (\fB, \fI, \fP).
-        body = re.sub('&(scons|SConstruct|SConscript|Dir|jar|Make|lambda);',
-                      r'\\fB\1\\fP', body)
-        body = re.sub('&(TARGET|TARGETS|SOURCE|SOURCES);', r'\\fB$\1\\fP', body)
-        body = re.sub('&(target|source);', r'\\fI\1\\fP', body)
-        body = re.sub('&b(-link)?-([^;]*);', r'\\fB\2\\fP()', body)
-        body = re.sub('&cv(-link)?-([^;]*);', r'\\fB$\2\\fP', body)
-        body = re.sub('&f(-link)?-env-([^;]*);', r'\\fBenv.\2\\fP()', body)
-        body = re.sub('&f(-link)?-([^;]*);', r'\\fB\2\\fP()', body)
-        body = re.sub(r'<(application|command|envar|filename|function|literal|option)>([^<]*)</\1>',
-                      r'\\fB\2\\fP', body)
-        body = re.sub(r'<(classname|emphasis|varname)>([^<]*)</\1>',
-                      r'\\fI\2\\fP', body)
-
-        # Convert groupings of font conversions (\fB, \fI, \fP) to
-        # man page .B, .BR, .I, .IR, .R, .RB and .RI macros.
-        body = re.compile(r'^\\f([BI])([^\\]* [^\\]*)\\fP\s*$', re.M).sub(r'.\1 "\2"', body)
-        body = re.compile(r'^\\f([BI])(.*)\\fP\s*$', re.M).sub(r'.\1 \2', body)
-        body = re.compile(r'^\\f([BI])(.*)\\fP(\S+)$', re.M).sub(r'.\1R \2 \3', body)
-        body = re.compile(r'^(\.B)( .*)\\fP(.*)\\fB(.*)$', re.M).sub(r'\1R\2 \3 \4', body)
-        body = re.compile(r'^(\.B)R?( .*)\\fP(.*)\\fI(.*)$', re.M).sub(r'\1I\2\3 \4', body)
-        body = re.compile(r'^(\.I)( .*)\\fP\\fB(.*)\\fP\\fI(.*)$', re.M).sub(r'\1R\2 \3 \4', body)
-        body = re.compile(r'^(\S+)\\f([BI])(.*)\\fP$', re.M).sub(r'.R\2 \1 \3', body)
-        body = re.compile(r'^(\S+)\\f([BI])(.*)\\fP([^\s\\]+)$', re.M).sub(r'.R\2 \1 \3 \4', body)
-        body = re.compile(r'^(\.R[BI].*[\S])\s+$;', re.M).sub(r'\1', body)
-
-        # Convert &lt; and &gt; entities to literal < and > characters.
-        body = body.replace('&lt;', '<')
-        body = body.replace('&gt;', '>')
-
-        # Backslashes.  Oh joy.
-        body = re.sub(r'\\(?=[^f])', r'\\\\', body)
-        body = re.compile("^'\\\\\\\\", re.M).sub("'\\\\", body)
-        body = re.compile(r'^\.([BI]R?) ([^"]\S*\\\\\S+[^"])$', re.M).sub(r'.\1 "\2"', body)
-
-        # Put backslashes in front of various hyphens that need
-        # to be long em-dashes.
-        body = re.compile(r'^\.([BI]R?) --', re.M).sub(r'.\1 \-\-', body)
-        body = re.compile(r'^\.([BI]R?) -', re.M).sub(r'.\1 \-', body)
-        body = re.compile(r'\\f([BI])-', re.M).sub(r'\\f\1\-', body)
-
-        f.write(body)
-
 class Proxy(object):
     def __init__(self, subject):
         """Wrap an object as a Proxy object"""
@@ -329,6 +217,7 @@ class Proxy(object):
 class SConsThing(Proxy):
     def idfunc(self):
         return self.name
+    
     def xml_term(self):
         return '<term>%s</term>' % self.name
 
@@ -336,20 +225,19 @@ class Builder(SConsThing):
     description = 'builder'
     prefix = 'b-'
     tag = 'function'
+    
     def xml_term(self):
         return ('<term><synopsis><%s>%s()</%s></synopsis>\n<synopsis><%s>env.%s()</%s></synopsis></term>' %
                 (self.tag, self.name, self.tag, self.tag, self.name, self.tag))
+        
     def entityfunc(self):
         return self.name
-    def man_separator(self):
-        return ['\n', "'\\" + '"'*69 + '\n']
-    def initial_man_chunks(self):
-        return [ '.IP %s()\n.IP env.%s()\n' % (self.name, self.name) ]
 
 class Function(SConsThing):
     description = 'function'
     prefix = 'f-'
     tag = 'function'
+    
     def args_to_xml(self, arg):
         s = ''.join(arg.body).strip()
         result = []
@@ -362,6 +250,7 @@ class Function(SConsThing):
             else:
                 result.append(m)
         return ''.join(result)
+    
     def xml_term(self):
         try:
             arguments = self.arguments
@@ -380,105 +269,73 @@ class Function(SConsThing):
                 result.append('<synopsis><varname>env</varname>.%s%s</synopsis>' % (self.name, s))
         result.append('</term>')
         return ''.join(result)
+    
     def entityfunc(self):
         return self.name
-    def man_separator(self):
-        return ['\n', "'\\" + '"'*69 + '\n']
-    def args_to_man(self, arg):
-        """Converts the contents of an <arguments> tag, which
-        specifies a function's calling signature, into a series
-        of tokens that alternate between literal tokens
-        (to be displayed in roman or bold face) and variable
-        names (to be displayed in italics).
-
-        This is complicated by the presence of Python "keyword=var"
-        arguments, where "keyword=" should be displayed literally,
-        and "var" should be displayed in italics.  We do this by
-        detecting the keyword= var portion and appending it to the
-        previous string, if any.
-        """
-        s = ''.join(arg.body).strip()
-        result = []
-        for m in re.findall('([a-zA-Z/_]+=?|[^a-zA-Z/_]+)', s):
-            if m[-1] == '=' and result:
-                if result[-1][-1] == '"':
-                    result[-1] = result[-1][:-1] + m + '"'
-                else:
-                    result[-1] += m
-            else:
-                if ' ' in m:
-                    m = '"%s"' % m
-                result.append(m)
-        return ' '.join(result)
-    def initial_man_chunks(self):
-        try:
-            arguments = self.arguments
-        except AttributeError:
-            arguments = ['()']
-        result = []
-        for arg in arguments:
-            try:
-                signature = arg.signature
-            except AttributeError:
-                signature = "both"
-            s = self.args_to_man(arg)
-            if signature in ('both', 'global'):
-                result.append('.TP\n.RI %s%s\n' % (self.name, s))
-            if signature in ('both', 'env'):
-                result.append('.TP\n.IR env .%s%s\n' % (self.name, s))
-        return result
 
 class Tool(SConsThing):
     description = 'tool'
     prefix = 't-'
     tag = 'literal'
+    
     def idfunc(self):
         return self.name.replace('+', 'X')
+    
     def entityfunc(self):
         return self.name
-    def man_separator(self):
-        return ['\n']
-    def initial_man_chunks(self):
-        return ['.IP %s\n' % self.name]
 
 class Variable(SConsThing):
     description = 'construction variable'
     prefix = 'cv-'
     tag = 'envar'
+    
     def entityfunc(self):
         return '$' + self.name
-    def man_separator(self):
-        return ['\n']
-    def initial_man_chunks(self):
-        return ['.IP %s\n' % self.name]
 
-if output_type == '--man':
-    processor_class = SCons_XML_to_man
-elif output_type == '--xml':
-    processor_class = SCons_XML_to_XML
+def write_output_files(h, buildersfiles, functionsfiles,
+                         toolsfiles, variablesfiles, write_func):
+    if buildersfiles:
+        g = processor_class([ Builder(b) for b in sorted(h.builders.values()) ],
+                            env_signatures=True)
+        write_func(g, buildersfiles)
+    
+    if functionsfiles:
+        g = processor_class([ Function(b) for b in sorted(h.functions.values()) ],
+                            env_signatures=True)
+        write_func(g, functionsfiles)
+    
+    if toolsfiles:
+        g = processor_class([ Tool(t) for t in sorted(h.tools.values()) ],
+                            env_signatures=False)
+        write_func(g, toolsfiles)
+    
+    if variablesfiles:
+        g = processor_class([ Variable(v) for v in sorted(h.cvars.values()) ],
+                            env_signatures=False)
+        write_func(g, variablesfiles)
+
+processor_class = SCons_XML
+
+# Step 1: Creating entity files for builders, functions,...
+h = parse_docs(args, False)
+write_output_files(h, buildersfiles, functionsfiles, toolsfiles,
+                   variablesfiles, SCons_XML.write_mod)
+
+# Step 2: Patching the include paths for entity definitions in XML files
+os.system('python bin/docs-correct-mod-paths.py')
+
+# Step 3: Validating all input files
+print "Validating files against SCons XSD..."
+if SConsDoc.validate_all_xml():
+    print "OK"
 else:
-    sys.stderr.write("Unknown output type '%s'\n" % output_type)
-    sys.exit(1)
-
-if buildersfiles:
-    g = processor_class([ Builder(b) for b in sorted(h.builders.values()) ],
-                        env_signatures=True)
-    g.write(buildersfiles)
-
-if functionsfiles:
-    g = processor_class([ Function(b) for b in sorted(h.functions.values()) ],
-                        env_signatures=True)
-    g.write(functionsfiles)
-
-if toolsfiles:
-    g = processor_class([ Tool(t) for t in sorted(h.tools.values()) ],
-                        env_signatures=False)
-    g.write(toolsfiles)
+    print "Validation failed! Please correct the errors above and try again."
 
-if variablesfiles:
-    g = processor_class([ Variable(v) for v in sorted(h.cvars.values()) ],
-                        env_signatures=False)
-    g.write(variablesfiles)
+# Step 4: Creating actual documentation snippets, using the
+#         fully resolved and updated entities from the *.mod files.
+h = parse_docs(args, True)
+write_output_files(h, buildersfiles, functionsfiles, toolsfiles,
+                   variablesfiles, SCons_XML.write)
 
 # Local Variables:
 # tab-width:4