separated default (universal) transforms into two stages so that no

transform is applied twice;
do not delete document.transformer in publish_doctree (Martin, this may be
important for the pickle writer -- you may need to delete document.transformer);
regenerate reporter object unconditionally; do so in publish_from_doctree,
not the doctree reader;
added tests;
I will post about all that on Docutils-develop in one or two days

git-svn-id: http://svn.code.sf.net/p/docutils/code/trunk/docutils@3663 929543f6-e4f2-0310-98a6-ba3bd3dd1d04

4 files changed, 106 insertions, 54 deletions

diff --git a/docutils/core.py b/docutils/core.py
index 694696f18..8375fbe89 100644
--- a/docutils/core.py
+++ b/docutils/core.py
@@ -193,11 +193,16 @@ class Publisher:
 
     def publish(self, argv=None, usage=None, description=None,
                 settings_spec=None, settings_overrides=None,
-                config_section=None, enable_exit_status=None):
+                config_section=None, enable_exit_status=None, stage=None):
         """
         Process command line options and arguments (if `self.settings` not
         already set), run `self.reader` and then `self.writer`.  Return
         `self.writer`'s output.
+
+        Pass ``stage=1`` to set transformer.default_transforms to the
+        stage-1 transforms; dito for ``stage=2`` and stage-2
+        transforms, resp.  See the documentation in the
+        `transforms.Transformer` class.
         """
         if self.settings is None:
             self.process_command_line(
@@ -208,6 +213,13 @@ class Publisher:
         try:
             self.document = self.reader.read(self.source, self.parser,
                                              self.settings)
+            assert stage in (1, 2, None)
+            if stage == 1:
+                self.document.transformer.default_transforms = (
+                    self.document.transformer.stage1_transforms)
+            elif stage == 2:
+                self.document.transformer.default_transforms = (
+                    self.document.transformer.stage2_transforms)
             self.apply_transforms()
             output = self.writer.write(self.document, self.destination)
             self.writer.assemble_parts()
@@ -449,20 +461,16 @@ def publish_doctree(source, source_path=None,
 
     Parameters: see `publish_programmatically`.
     """
-    output, pub = publish_programmatically(
-        source_class=source_class, source=source, source_path=source_path,
-        destination_class=io.NullOutput,
-        destination=None, destination_path=None,
-        reader=reader, reader_name=reader_name,
-        parser=parser, parser_name=parser_name,
-        writer=None, writer_name='null',
-        settings=settings, settings_spec=settings_spec,
-        settings_overrides=settings_overrides,
-        config_section=config_section,
-        enable_exit_status=enable_exit_status)
-    # The transformer is not needed any more
-    # (a new transformer will be created in `publish_from_doctree`):
-    del pub.document.transformer
+    pub = Publisher(reader=reader, parser=parser, writer=None,
+                    settings=settings,
+                    source_class=source_class,
+                    destination_class=io.NullOutput)
+    pub.set_components(reader_name, parser_name, 'null')
+    pub.process_programmatic_settings(
+        settings_spec, settings_overrides, config_section)
+    pub.set_source(source, source_path)
+    pub.set_destination(None, None)
+    output = pub.publish(enable_exit_status=enable_exit_status, stage=1)
     return pub.document
 
 def publish_from_doctree(document, destination_path=None,
@@ -478,6 +486,9 @@ def publish_from_doctree(document, destination_path=None,
     Note that document.settings is overridden; if you want to use the settings
     of the original `document`, pass settings=document.settings.
 
+    Also, new document.transformer and document.reporter objects are
+    generated.
+
     For encoded string output, be sure to set the 'output_encoding' setting to
     the desired encoding.  Set it to 'unicode' for unencoded Unicode string
     output.  Here's one way::
@@ -490,11 +501,6 @@ def publish_from_doctree(document, destination_path=None,
 
     Other parameters: see `publish_programmatically`.
     """
-    # Create fresh Transformer object, to be populated from Writer component.
-    document.transformer = Transformer(document)
-    # Don't apply default transforms twice:
-    document.transformer.default_transforms = (
-        document.transformer.reprocess_transforms)
     reader = docutils.readers.doctree.Reader(parser_name='null')
     pub = Publisher(reader, None, writer,
                     source=io.DocTreeInput(document),
@@ -503,8 +509,15 @@ def publish_from_doctree(document, destination_path=None,
         pub.set_writer(writer_name)
     pub.process_programmatic_settings(
         settings_spec, settings_overrides, config_section)
+    # Create fresh Transformer object, to be populated from Writer component.
+    document.transformer = Transformer(document)
+    # Create fresh Reporter object because it is dependent on (new) settings.
+    document.reporter = utils.new_reporter(document.get('source', ''),
+                                           pub.settings)
+    # Replace existing settings object with new one.
+    document.settings = pub.settings
     pub.set_destination(None, destination_path)
-    output = pub.publish(enable_exit_status=enable_exit_status)
+    output = pub.publish(enable_exit_status=enable_exit_status, stage=2)
     return output, pub.writer.parts
 
 def publish_programmatically(source_class, source, source_path,
diff --git a/docutils/readers/doctree.py b/docutils/readers/doctree.py
index 5a4371983..ec56e4099 100644
--- a/docutils/readers/doctree.py
+++ b/docutils/readers/doctree.py
@@ -37,9 +37,3 @@ class Reader(readers.Reader):
         Overrides the inherited method.
         """
         self.document = self.input
-        # Restore the reporter after document serialization:
-        if self.document.reporter is None:
-            self.document.reporter = utils.new_reporter(
-                self.source.source_path, self.settings) 
-        # Override document settings with new settings:
-        self.document.settings = self.settings
diff --git a/docutils/transforms/__init__.py b/docutils/transforms/__init__.py
index e575f42be..0bc302b77 100644
--- a/docutils/transforms/__init__.py
+++ b/docutils/transforms/__init__.py
@@ -74,17 +74,20 @@ class Transformer(TransformSpec):
 
     from docutils.transforms import universal
 
-    default_transforms = (universal.Decorations,
-                          universal.ExposeInternals,
-                          universal.Messages,
-                          universal.FilterMessages)
+    stage1_transforms = (universal.Decorations,
+                         universal.ExposeInternals)
+    """Suggested replacement for `default_transforms` when generating
+    a document tree without writing it."""
+
+    stage2_transforms = (universal.Messages,
+                         universal.FilterMessages)
+    """Suggested replacement for `default_transforms` when writing a
+    previously-parsed document tree.  Only transforms which *must* be applied
+    after writer-specific transforms should be added to this list."""
+
+    default_transforms = stage1_transforms + stage2_transforms
     """These transforms are applied to all document trees."""
 
-    reprocess_transforms = (universal.Messages,
-                            universal.FilterMessages)
-    """This set of transforms is a suggested replacement for
-    `default_transforms` when reprocessing a document tree."""
-
     def __init__(self, document):
         self.transforms = []
         """List of transforms to apply.  Each item is a 3-tuple:
diff --git a/test/test_publisher.py b/test/test_publisher.py
index d5460694c..39b68cf37 100755
--- a/test/test_publisher.py
+++ b/test/test_publisher.py
@@ -10,61 +10,101 @@
 Test the `Publisher` facade and the ``publish_*`` convenience functions.
 """
 
-import unittest
-from types import DictType, StringType
-from docutils import core, nodes
 import pickle
+from types import DictType, StringType
+
+import docutils
+from docutils import core, nodes, io
+
+import DocutilsTestSupport
 
 
 test_document = """\
 Test Document
 =============
 
-This is a test document.
+This is a test document with a broken reference: nonexistent_
 """
 pseudoxml_output = """\
 <document ids="test-document" names="test document" source="<string>" title="Test Document">
     <title>
         Test Document
     <paragraph>
-        This is a test document.
+        This is a test document with a broken reference: \n\
+        <problematic ids="id2" refid="id1">
+            nonexistent_
+    <section classes="system-messages">
+        <title>
+            Docutils System Messages
+        <system_message backrefs="id2" ids="id1" level="3" line="4" source="<string>" type="ERROR">
+            <paragraph>
+                Unknown target name: "nonexistent".
+"""
+exposed_pseudoxml_output = """\
+<document ids="test-document" internal:refnames="{u\'nonexistent\': [<reference: <#text: u\'nonexistent\'>>]}" names="test document" source="<string>" title="Test Document">
+    <title>
+        Test Document
+    <paragraph>
+        This is a test document with a broken reference: \n\
+        <problematic ids="id2" refid="id1">
+            nonexistent_
+    <section classes="system-messages">
+        <title>
+            Docutils System Messages
+        <system_message backrefs="id2" ids="id1" level="3" line="4" source="<string>" type="ERROR">
+            <paragraph>
+                Unknown target name: "nonexistent".
 """
 
 
-class PublishDoctreeTestCase(unittest.TestCase):
+class PublishDoctreeTestCase(DocutilsTestSupport.StandardTestCase, docutils.SettingsSpec):
+
+    settings_default_overrides = {
+        '_disable_config': 1,
+        'warning_stream': io.NullOutput()}
 
     def test_publish_doctree(self):
-        """Test `publish_doctree` and `publish_from_doctree`."""
+        # Test `publish_doctree` and `publish_from_doctree`.
 
         # Produce the document tree.
         doctree = core.publish_doctree(
-            source=test_document,
-            reader_name='standalone',
-            parser_name='restructuredtext',
-            settings_overrides={'_disable_config': 1})
+            source=test_document, reader_name='standalone',
+            parser_name='restructuredtext', settings_spec=self,
+            settings_overrides={'expose_internals':
+                                ['refnames', 'do_not_expose'],
+                                'report_level': 5})
         self.assert_(isinstance(doctree, nodes.document))
-        
+
         # Confirm that transforms have been applied (in this case, the
         # DocTitle transform):
         self.assert_(isinstance(doctree[0], nodes.title))
         self.assert_(isinstance(doctree[1], nodes.paragraph))
+        # Confirm that the Messages transform has not yet been applied:
+        self.assertEquals(len(doctree), 2)
 
+        # The `do_not_expose` attribute may not show up in the
+        # pseudoxml output because the expose_internals transform may
+        # not be applied twice.
+        doctree.do_not_expose = 'test'
         # Write out the document:
         output, parts = core.publish_from_doctree(
             doctree, writer_name='pseudoxml',
-            settings_overrides={'_disable_config': 1})
-        self.assertEquals(output, pseudoxml_output)
+            settings_spec=self,
+            settings_overrides={'expose_internals':
+                                ['refnames', 'do_not_expose'],
+                                'report_level': 1})
+        self.assertEquals(output, exposed_pseudoxml_output)
         self.assert_(isinstance(parts, DictType))
 
     def test_publish_pickle(self):
-        """Test publishing a document tree with pickling and unpickling."""
+        # Test publishing a document tree with pickling and unpickling.
         
         # Produce the document tree.
         doctree = core.publish_doctree(
             source=test_document,
             reader_name='standalone',
             parser_name='restructuredtext',
-            settings_overrides={'_disable_config': 1})
+            settings_spec=self)
         self.assert_(isinstance(doctree, nodes.document))
 
         # Pickle the document.  Note: if this fails, some unpickleable
@@ -75,8 +115,9 @@ class PublishDoctreeTestCase(unittest.TestCase):
         # requirement, applications will be built on the assumption
         # that we can pickle the document.
 
-        # Remove the reporter before pickling.
+        # Remove the reporter and the transformer before pickling.
         doctree.reporter = None
+        doctree.transformer = None
 
         doctree_pickled = pickle.dumps(doctree)
         self.assert_(isinstance(doctree_pickled, StringType))
@@ -89,10 +130,11 @@ class PublishDoctreeTestCase(unittest.TestCase):
         # Write out the document:
         output, parts = core.publish_from_doctree(
             doctree_zombie, writer_name='pseudoxml',
-            settings_overrides={'_disable_config': 1})
+            settings_spec=self)
         self.assertEquals(output, pseudoxml_output)
         self.assert_(isinstance(parts, DictType))
 
 
 if __name__ == '__main__':
+    import unittest
     unittest.main()