Accumulated documentation changes.

3 files changed, 475 insertions, 24 deletions

diff --git a/bin/SConsDoc.py b/bin/SConsDoc.py
new file mode 100644
index 00000000..df28b76d
--- /dev/null
+++ b/bin/SConsDoc.py
@@ -0,0 +1,241 @@
+#!/usr/bin/env python
+#
+# Module for handling SCons documentation processing.
+#
+import os.path
+import imp
+import sys
+import xml.sax.handler
+
+class Item:
+    def __init__(self, name):
+        self.name = name
+        self.sort_name = name.lower()
+        if self.sort_name[0] == '_':
+            self.sort_name = self.sort_name[1:]
+        self.summary = []
+        self.uses = None
+    def cmp_name(self, name):
+        if name[0] == '_':
+            name = name[1:]
+        return name.lower()
+    def __cmp__(self, other):
+        return cmp(self.sort_name, other.sort_name)
+
+class Builder(Item):
+    pass
+
+class Tool(Item):
+    def __init__(self, name):
+        Item.__init__(self, name)
+        self.entity = self.name.replace('+', 'X')
+
+class ConstructionVariable(Item):
+    pass
+
+class Chunk:
+    def __init__(self, tag, body=None):
+        self.tag = tag
+        if not body:
+            body = []
+        self.body = body
+    def __str__(self):
+        body = ''.join(self.body)
+        return "<%s>%s</%s>\n" % (self.tag, body, self.tag)
+    def append(self, data):
+        self.body.append(data)
+
+class Summary:
+    def __init__(self):
+        self.body = []
+        self.collect = []
+    def append(self, data):
+        self.collect.append(data)
+    def end_para(self):
+        text = ''.join(self.collect)
+        paras = text.split('\n\n')
+        if paras == ['\n']:
+            return
+        if paras[0] == '':
+            self.body.append('\n')
+            paras = paras[1:]
+            paras[0] = '\n' + paras[0]
+        if paras[-1] == '':
+            paras = paras[:-1]
+            paras[-1] = paras[-1] + '\n'
+            last = '\n'
+        else:
+            last = None
+        sep = None
+        for p in paras:
+            c = Chunk("para", p)
+            if sep:
+                self.body.append(sep)
+            self.body.append(c)
+            sep = '\n'
+        if last:
+            self.body.append(last)
+    def begin_chunk(self, chunk):
+        self.end_para()
+        self.collect = chunk
+    def end_chunk(self):
+        self.body.append(self.collect)
+        self.collect = []
+
+class SConsDocHandler(xml.sax.handler.ContentHandler,
+                      xml.sax.handler.ErrorHandler):
+    def __init__(self):
+        self._start_dispatch = {}
+        self._end_dispatch = {}
+        keys = self.__class__.__dict__.keys()
+        start_tag_method_names = filter(lambda k: k[:6] == 'start_', keys)
+        end_tag_method_names = filter(lambda k: k[:4] == 'end_', keys)
+        for method_name in start_tag_method_names:
+            tag = method_name[6:]
+            self._start_dispatch[tag] = getattr(self, method_name)
+        for method_name in end_tag_method_names:
+            tag = method_name[4:]
+            self._end_dispatch[tag] = getattr(self, method_name)
+        self.stack = []
+        self.collect = []
+        self.current_object = []
+        self.builders = {}
+        self.tools = {}
+        self.cvars = {}
+
+    def startElement(self, name, attrs):
+        try:
+            start_element_method = self._start_dispatch[name]
+        except KeyError:
+            self.characters('<%s>' % name)
+        else:
+            start_element_method(attrs)
+
+    def endElement(self, name):
+        try:
+            end_element_method = self._end_dispatch[name]
+        except KeyError:
+            self.characters('</%s>' % name)
+        else:
+            end_element_method()
+
+    #
+    #
+    def characters(self, chars):
+        self.collect.append(chars)
+
+    def begin_collecting(self, chunk):
+        self.collect = chunk
+    def end_collecting(self):
+        self.collect = []
+
+    def begin_chunk(self):
+        pass
+    def end_chunk(self):
+        pass
+
+    #
+    #
+    #
+
+    def begin_xxx(self, obj):
+        self.stack.append(self.current_object)
+        self.current_object = obj
+    def end_xxx(self):
+        self.current_object = self.stack.pop()
+
+    #
+    #
+    #
+    def start_scons_doc(self, attrs):
+        pass
+    def end_scons_doc(self):
+        pass
+
+    def start_builder(self, attrs):
+        name = attrs.get('name')
+        try:
+            builder = self.builders[name]
+        except KeyError:
+            builder = Builder(name)
+            self.builders[name] = builder
+        self.begin_xxx(builder)
+    def end_builder(self):
+        self.end_xxx()
+
+    def start_tool(self, attrs):
+        name = attrs.get('name')
+        try:
+            tool = self.tools[name]
+        except KeyError:
+            tool = Tool(name)
+            self.tools[name] = tool
+        self.begin_xxx(tool)
+    def end_tool(self):
+        self.end_xxx()
+
+    def start_cvar(self, attrs):
+        name = attrs.get('name')
+        try:
+            cvar = self.cvars[name]
+        except KeyError:
+            cvar = ConstructionVariable(name)
+            self.cvars[name] = cvar
+        self.begin_xxx(cvar)
+    def end_cvar(self):
+        self.end_xxx()
+
+    def start_summary(self, attrs):
+        summary = Summary()
+        self.current_object.summary = summary
+        self.begin_xxx(summary)
+        self.begin_collecting(summary)
+    def end_summary(self):
+        self.current_object.end_para()
+        self.end_xxx()
+
+    def start_example(self, attrs):
+        example = Chunk("programlisting")
+        self.current_object.begin_chunk(example)
+    def end_example(self):
+        self.current_object.end_chunk()
+
+    def start_uses(self, attrs):
+        self.begin_collecting([])
+    def end_uses(self):
+        self.current_object.uses = ''.join(self.collect).split()
+        self.end_collecting()
+
+    # Stuff for the ErrorHandler portion.
+    def error(self, exception):
+        linenum = exception._linenum - self.preamble_lines
+        sys.stderr.write('%s:%d:%d: %s (error)\n' % (self.filename, linenum, exception._colnum, ''.join(exception.args)))
+
+    def fatalError(self, exception):
+        linenum = exception._linenum - self.preamble_lines
+        sys.stderr.write('%s:%d:%d: %s (fatalError)\n' % (self.filename, linenum, exception._colnum, ''.join(exception.args)))
+
+    def set_file_info(self, filename, preamble_lines):
+        self.filename = filename
+        self.preamble_lines = preamble_lines
+
+# lifted from Ka-Ping Yee's way cool pydoc module.
+def importfile(path):
+    """Import a Python source file or compiled file given its path."""
+    magic = imp.get_magic()
+    file = open(path, 'r')
+    if file.read(len(magic)) == magic:
+        kind = imp.PY_COMPILED
+    else:
+        kind = imp.PY_SOURCE
+    file.close()
+    filename = os.path.basename(path)
+    name, ext = os.path.splitext(filename)
+    file = open(path, 'r')
+    try:
+        module = imp.load_module(name, file, path, (ext, 'r', kind))
+    except ImportError, e:
+        sys.stderr.write("Could not import %s: %s\n" % (path, e))
+        return None
+    file.close()
+    return module
diff --git a/bin/scons-proc.py b/bin/scons-proc.py
new file mode 100644
index 00000000..809c3d04
--- /dev/null
+++ b/bin/scons-proc.py
@@ -0,0 +1,195 @@
+#!/usr/bin/env python
+#
+# Process a list of Python and/or XML files containing SCons documentation.
+#
+# Depending on the options, this script creates DocBook-formatted lists
+# of the Builders, Tools or construction variables in generated SGML
+# files containing the summary text and/or .mod files contining the
+# ENTITY definitions for each item.
+#
+import getopt
+import os.path
+import re
+import string
+import StringIO
+import sys
+import xml.sax
+
+import SConsDoc
+
+base_sys_path = [os.getcwd() + '/build/test-tar-gz/lib/scons'] + sys.path
+
+helpstr = """\
+Usage: scons-varlist.py [-b .gen,.mod] [-t .gen,.mod] [-v .gen,.mod] [infile]
+Options:
+  -m, --modfile               .mod file to hold Builder entities
+"""
+
+opts, args = getopt.getopt(sys.argv[1:],
+                           "b:t:v:",
+                           ['builders=', 'tools=', 'variables='])
+
+buildersfiles = None
+toolsfiles = None
+variablesfiles = None
+
+for o, a in opts:
+    if o == '-b' or o == '--builders':
+        buildersfiles = a
+    elif o == '-t' or o == '--tools':
+        toolsfiles = a
+    elif o == '-v' or o == '--variables':
+        variablesfiles = a
+
+h = SConsDoc.SConsDocHandler()
+saxparser = xml.sax.make_parser()
+saxparser.setContentHandler(h)
+saxparser.setErrorHandler(h)
+
+preamble = """\
+<?xml version="1.0"?>
+<scons_doc>
+"""
+
+postamble = """\
+</scons_doc>
+"""
+
+for f in args:
+    _, ext = os.path.splitext(f)
+    if ext == '.py':
+        dir, _ = os.path.split(f)
+        if dir:
+            sys.path = [dir] + base_sys_path
+        module = SConsDoc.importfile(f)
+        h.set_file_info(f, len(preamble.split('\n')))
+        try:
+            content = module.__scons_doc__
+        except AttributeError:
+            content = None
+        else:
+            del module.__scons_doc__
+    else:
+        h.set_file_info(f, len(preamble.split('\n')))
+        content = open(f).read()
+    if content:
+        content = content.replace('&', '&amp;')
+        input = preamble + content + postamble
+        try:
+            saxparser.parse(StringIO.StringIO(input))
+        except:
+            sys.stderr.write("error in %s\n" % f)
+            raise
+
+Warning = """\
+<!--
+THIS IS AN AUTOMATICALLY-GENERATED FILE.  DO NOT EDIT.
+-->
+"""
+
+Regular_Entities_Header = """\
+<!--
+
+  Regular %s entities.
+
+-->
+"""
+
+Link_Entities_Header = """\
+<!--
+
+  Entities that are links to the %s entries in the appendix.
+
+-->
+"""
+
+class XXX:
+    def __init__(self, entries, **kw):
+        values = entries.values()
+        values.sort()
+        self.values = values
+        for k, v in kw.items():
+            setattr(self, k, v)
+    def write_gen(self, filename):
+        if not filename:
+            return
+        f = open(filename, 'w')
+        for v in self.values:
+            f.write('\n<varlistentry id="%s%s">\n' %
+                        (self.prefix, self.idfunc(v.name)))
+            for term in self.termfunc(v.name):
+                f.write('<term><%s>%s</%s></term>\n' %
+                        (self.tag, term, self.tag))
+            f.write('<listitem>\n')
+            for chunk in v.summary.body:
+                f.write(str(chunk))
+            #if v.uses:
+            #    u = map(lambda x, s: '&%slink-%s;' % (s.prefix, x), v.uses)
+            #    f.write('<para>\n')
+            #    f.write('Uses:  ' + ', '.join(u) + '.\n')
+            #    f.write('</para>\n')
+            f.write('</listitem>\n')
+            f.write('</varlistentry>\n')
+    def write_mod(self, filename):
+        if not filename:
+            return
+        f = open(filename, 'w')
+        f.write(Warning)
+        f.write('\n')
+        f.write(Regular_Entities_Header % self.description)
+        f.write('\n')
+        for v in self.values:
+            f.write('<!ENTITY %s%s "<%s>%s</%s>">\n' %
+                        (self.prefix, self.idfunc(v.name),
+                         self.tag, self.entityfunc(v.name), self.tag))
+        f.write('\n')
+        f.write(Warning)
+        f.write('\n')
+        f.write(Link_Entities_Header % self.description)
+        f.write('\n')
+        for v in self.values:
+            f.write('<!ENTITY %slink-%s \'<link linkend="%s%s"><%s>%s</%s></link>\'>\n' %
+                        (self.prefix, self.idfunc(v.name),
+                         self.prefix, self.idfunc(v.name),
+                         self.tag, self.entityfunc(v.name), self.tag))
+        f.write('\n')
+        f.write(Warning)
+
+if buildersfiles:
+    g = XXX(h.builders,
+            description = 'builder',
+            prefix = 'b-',
+            tag = 'function',
+            idfunc = lambda x: x,
+            termfunc = lambda x: [x+'()', 'env.'+x+'()'],
+            entityfunc = lambda x: x)
+
+    gen, mod = string.split(buildersfiles, ',')
+    g.write_gen(gen)
+    g.write_mod(mod)
+
+if toolsfiles:
+    g = XXX(h.tools,
+            description = 'tool',
+            prefix = 't-',
+            tag = 'literal',
+            idfunc = lambda x: string.replace(x, '+', 'X'),
+            termfunc = lambda x: [x],
+            entityfunc = lambda x: x)
+
+    gen, mod = string.split(toolsfiles, ',')
+    g.write_gen(gen)
+    g.write_mod(mod)
+
+if variablesfiles:
+    g = XXX(h.cvars,
+            description = 'construction variable',
+            prefix = 'cv-',
+            tag = 'envar',
+            idfunc = lambda x: x,
+            termfunc = lambda x: [x],
+            entityfunc = lambda x: '$'+x)
+
+    gen, mod = string.split(variablesfiles, ',')
+    g.write_gen(gen)
+    g.write_mod(mod)
diff --git a/bin/sconsoutput.py b/bin/sconsoutput.py
index 3ca8a32d..974646c4 100644
--- a/bin/sconsoutput.py
+++ b/bin/sconsoutput.py
@@ -23,7 +23,7 @@
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #
 
-__revision__ = "src/sconsoutput.py 0.D003 2003/09/22 22:17:34 software"
+__revision__ = "/home/scons/sconsoutput/branch.0/baseline/src/sconsoutput.py 0.4.D001 2004/11/27 18:44:37 knight"
 
 #
 # sconsoutput.py -   an SGML preprocessor for capturing SCons output
@@ -76,14 +76,14 @@ __revision__ = "src/sconsoutput.py 0.D003 2003/09/22 22:17:34 software"
 # SCons output is generated from the following sort of tag:
 #
 #       <scons_output example="ex1" os="posix">
-#         <command>scons -Q foo</command>
-#         <command>scons -Q foo</command>
+#         <scons_output_command>scons -Q foo</scons_output_command>
+#         <scons_output_command>scons -Q foo</scons_output_command>
 #       </scons_output>
 #
-# You tell it which example to use with the "example" attribute, and
-# then give it a list of <command> tags to execute.  You can also supply
-# an "os" tag, which specifies the type of operating system this example
-# is intended to show; if you omit this, default value is "posix".
+# You tell it which example to use with the "example" attribute, and then
+# give it a list of <scons_output_command> tags to execute.  You can also
+# supply an "os" tag, which specifies the type of operating system this
+# example is intended to show; if you omit this, default value is "posix".
 #
 # The generated SGML will show the command line (with the appropriate
 # command-line prompt for the operating system), execute the command in
@@ -194,7 +194,7 @@ import SCons.Action
 import SCons.Defaults
 import SCons.Node.FS
 
-platform = '%s'
+platform = '%(osname)s'
 
 Sep = {
     'posix' : '/',
@@ -263,6 +263,10 @@ class ToolSurrogate:
             env[v] = SCons.Action.Action(self.func,
                                          strfunction=strfunction,
                                          varlist=self.varlist)
+    def __repr__(self):
+        # This is for the benefit of printing the 'TOOLS'
+        # variable through env.Dump().
+        return repr(self.tool)
 
 def Null(target, source, env):
     pass
@@ -365,7 +369,12 @@ ToolList = {
                 ],
 }
 
-tools = map(lambda t: apply(ToolSurrogate, t), ToolList[platform])
+toollist = ToolList[platform]
+filter_tools = string.split('%(tools)s')
+if filter_tools:
+    toollist = filter(lambda x, ft=filter_tools: x[0] in ft, toollist)
+
+toollist = map(lambda t: apply(ToolSurrogate, t), toollist)
 
 def surrogate_spawn(sh, escape, cmd, args, env):
     pass
@@ -375,7 +384,7 @@ def surrogate_pspawn(sh, escape, cmd, args, env, stdout, stderr):
 
 SCons.Defaults.ConstructionEnvironment.update({
     'PLATFORM' : platform,
-    'TOOLS'    : tools,
+    'TOOLS'    : toollist,
     'SPAWN'    : surrogate_spawn,
     'PSPAWN'   : surrogate_pspawn,
 })
@@ -384,7 +393,7 @@ SConscript('SConstruct')
 """
 
 # "Commands" that we will execute in our examples.
-def command_scons(args, c, test, osname):
+def command_scons(args, c, test, dict):
     save_vals = {}
     delete_keys = []
     try:
@@ -403,7 +412,7 @@ def command_scons(args, c, test, osname):
              program = scons_py,
              arguments = '-f - ' + string.join(args),
              chdir = test.workpath('WORK'),
-             stdin = Stdin % osname)
+             stdin = Stdin % dict)
     os.environ.update(save_vals)
     for key in delete_keys:
         del(os.environ[key])
@@ -417,7 +426,7 @@ def command_scons(args, c, test, osname):
     #    sys.stderr.write(err)
     return lines
 
-def command_touch(args, c, test, osname):
+def command_touch(args, c, test, dict):
     time.sleep(1)
     for file in args:
         if not os.path.isabs(file):
@@ -427,7 +436,7 @@ def command_touch(args, c, test, osname):
         os.utime(file, None)
     return []
 
-def command_edit(args, c, test, osname):
+def command_edit(args, c, test, dict):
     try:
         add_string = c.edit[:]
     except AttributeError:
@@ -441,7 +450,7 @@ def command_edit(args, c, test, osname):
         open(file, 'wb').write(contents + add_string)
     return []
 
-def command_ls(args, c, test, osname):
+def command_ls(args, c, test, dict):
     def ls(a):
         files = os.listdir(a)
         files = filter(lambda x: x[0] != '.', files)
@@ -462,12 +471,12 @@ CommandDict = {
     'ls'    : command_ls,
 }
 
-def ExecuteCommand(args, c, t, osname):
+def ExecuteCommand(args, c, t, dict):
     try:
         func = CommandDict[args[0]]
     except KeyError:
-        func = lambda args, c, t, osname: []
-    return func(args[1:], c, t, osname)
+        func = lambda args, c, t, dict: []
+    return func(args[1:], c, t, dict)
 
 class MySGML(sgmllib.SGMLParser):
     """A subclass of the standard Python 2.2 sgmllib SGML parser.
@@ -647,6 +656,7 @@ class MySGML(sgmllib.SGMLParser):
         o = Output()
         o.preserve = None
         o.os = 'posix'
+        o.tools = ''
         o.e = e
         # Locally-set.
         for name, value in attrs:
@@ -706,11 +716,16 @@ class MySGML(sgmllib.SGMLParser):
 
             e = string.replace(c.data, '__ROOT__', t.workpath('ROOT'))
             args = string.split(e)
-            lines = ExecuteCommand(args, c, t, o.os)
+            lines = ExecuteCommand(args, c, t, {'osname':o.os, 'tools':o.tools})
+            content = None
             if c.output:
-                sys.stdout.write(p + c.output + '\n')
+                content = c.output
             elif lines:
-                sys.stdout.write(p + string.join(lines, '\n' + p) + '\n')
+                content = string.join(lines, '\n' + p)
+            if content:
+                content = string.replace(content, '<', '&lt;')
+                content = string.replace(content, '>', '&gt;')
+                sys.stdout.write(p + content + '\n')
 
         if o.data[0] == '\n':
             o.data = o.data[1:]
@@ -718,11 +733,11 @@ class MySGML(sgmllib.SGMLParser):
         delattr(self, 'o')
         self.afunclist = self.afunclist[:-1]
 
-    def start_command(self, attrs):
+    def start_scons_output_command(self, attrs):
         try:
             o = self.o
         except AttributeError:
-            self.error("<command> tag outside of <scons_output>")
+            self.error("<scons_output_command> tag outside of <scons_output>")
         try:
             o.prefix
         except AttributeError:
@@ -734,7 +749,7 @@ class MySGML(sgmllib.SGMLParser):
         o.commandlist.append(c)
         self.afunclist.append(c.afunc)
 
-    def end_command(self):
+    def end_scons_output_command(self):
         self.o.data = ""
         self.afunclist = self.afunclist[:-1]