Close #6698: doctest: Add :no-trim-doctest-flags: options

To control trimming doctest flags manually, this adds new options
:trim-doctest-flags: and :no-trim-doctest-flags: to doctest
directives.  It helps to describes doctest module itself in python
doc (see #6698).

6 files changed, 69 insertions, 12 deletions

diff --git a/CHANGES b/CHANGES
index d8a3cb915..c6d998363 100644
--- a/CHANGES
+++ b/CHANGES
@@ -42,6 +42,8 @@ Features added
 * #7768: i18n: :confval:`figure_language_filename` supports ``docpath`` token
 * #5208: linkcheck: Support checks for local links
 * #5090: setuptools: Link verbosity to distutils' -v and -q option
+* #6698: doctest: Add ``:trim-doctest-flags:`` and ``:no-trim-doctest-flags:``
+  options to doctest, testcode and testoutput directives
 * #7052: add ``:noindexentry:`` to the Python, C, C++, and Javascript domains.
   Update the documentation to better reflect the relationship between this option
   and the ``:noindex:`` option.
diff --git a/doc/usage/extensions/doctest.rst b/doc/usage/extensions/doctest.rst
index 33d6cf016..0fe9b535d 100644
--- a/doc/usage/extensions/doctest.rst
+++ b/doc/usage/extensions/doctest.rst
@@ -67,7 +67,7 @@ a comma-separated list of group names.
    default set of flags is specified by the :confval:`doctest_default_flags`
    configuration variable.
 
-   This directive supports three options:
+   This directive supports five options:
 
    * ``hide``, a flag option, hides the doctest block in other builders.  By
      default it is shown as a highlighted doctest block.
@@ -102,6 +102,11 @@ a comma-separated list of group names.
 
         Supported PEP-440 operands and notations
 
+   * ``trim-doctest-flags`` and ``no-trim-doctest-flags``, a flag option,
+     doctest flags (comments looking like ``# doctest: FLAG, ...``) at the
+     ends of lines and ``<BLANKLINE>`` markers are removed (or not removed)
+     individually.  Default is ``trim-doctest-flags``.
+
    Note that like with standard doctests, you have to use ``<BLANKLINE>`` to
    signal a blank line in the expected output.  The ``<BLANKLINE>`` is removed
    when building presentation output (HTML, LaTeX etc.).
@@ -119,11 +124,16 @@ a comma-separated list of group names.
 
    A code block for a code-output-style test.
 
-   This directive supports one option:
+   This directive supports three options:
 
    * ``hide``, a flag option, hides the code block in other builders.  By
      default it is shown as a highlighted code block.
 
+   * ``trim-doctest-flags`` and ``no-trim-doctest-flags``, a flag option,
+     doctest flags (comments looking like ``# doctest: FLAG, ...``) at the
+     ends of lines and ``<BLANKLINE>`` markers are removed (or not removed)
+     individually.  Default is ``trim-doctest-flags``.
+
    .. note::
 
       Code in a ``testcode`` block is always executed all at once, no matter how
@@ -149,7 +159,7 @@ a comma-separated list of group names.
    The corresponding output, or the exception message, for the last
    :rst:dir:`testcode` block.
 
-   This directive supports two options:
+   This directive supports four options:
 
    * ``hide``, a flag option, hides the output block in other builders.  By
      default it is shown as a literal block without highlighting.
@@ -157,6 +167,11 @@ a comma-separated list of group names.
    * ``options``, a string option, can be used to give doctest flags
      (comma-separated) just like in normal doctest blocks.
 
+   * ``trim-doctest-flags`` and ``no-trim-doctest-flags``, a flag option,
+     doctest flags (comments looking like ``# doctest: FLAG, ...``) at the
+     ends of lines and ``<BLANKLINE>`` markers are removed (or not removed)
+     individually.  Default is ``trim-doctest-flags``.
+
    Example::
 
       .. testcode::
diff --git a/sphinx/ext/doctest.py b/sphinx/ext/doctest.py
index 26966016d..7cd89ad32 100644
--- a/sphinx/ext/doctest.py
+++ b/sphinx/ext/doctest.py
@@ -91,7 +91,7 @@ class TestDirective(SphinxDirective):
                 # convert <BLANKLINE>s to ordinary blank lines for presentation
                 test = code
                 code = blankline_re.sub('', code)
-            if doctestopt_re.search(code):
+            if doctestopt_re.search(code) and 'no-trim-doctest-flags' not in self.options:
                 if not test:
                     test = code
                 code = doctestopt_re.sub('', code)
@@ -151,6 +151,10 @@ class TestDirective(SphinxDirective):
                     line=self.lineno)
         if 'skipif' in self.options:
             node['skipif'] = self.options['skipif']
+        if 'trim-doctest-flags' in self.options:
+            node['trim_flags'] = True
+        elif 'no-trim-doctest-flags' in self.options:
+            node['trim_flags'] = False
         return [node]
 
 
@@ -165,26 +169,32 @@ class TestcleanupDirective(TestDirective):
 class DoctestDirective(TestDirective):
     option_spec = {
         'hide': directives.flag,
+        'no-trim-doctest-flags': directives.flag,
         'options': directives.unchanged,
         'pyversion': directives.unchanged_required,
         'skipif': directives.unchanged_required,
+        'trim-doctest-flags': directives.flag,
     }
 
 
 class TestcodeDirective(TestDirective):
     option_spec = {
         'hide': directives.flag,
+        'no-trim-doctest-flags': directives.flag,
         'pyversion': directives.unchanged_required,
         'skipif': directives.unchanged_required,
+        'trim-doctest-flags': directives.flag,
     }
 
 
 class TestoutputDirective(TestDirective):
     option_spec = {
         'hide': directives.flag,
+        'no-trim-doctest-flags': directives.flag,
         'options': directives.unchanged,
         'pyversion': directives.unchanged_required,
         'skipif': directives.unchanged_required,
+        'trim-doctest-flags': directives.flag,
     }
 
 
diff --git a/sphinx/transforms/post_transforms/code.py b/sphinx/transforms/post_transforms/code.py
index add35647b..2012d6e11 100644
--- a/sphinx/transforms/post_transforms/code.py
+++ b/sphinx/transforms/post_transforms/code.py
@@ -9,10 +9,10 @@
 """
 
 import sys
-from typing import Any, Dict, List, NamedTuple, Union
+from typing import Any, Dict, List, NamedTuple
 
 from docutils import nodes
-from docutils.nodes import Node
+from docutils.nodes import Node, TextElement
 from pygments.lexers import PythonConsoleLexer, guess_lexer
 
 from sphinx import addnodes
@@ -93,9 +93,6 @@ class TrimDoctestFlagsTransform(SphinxTransform):
     default_priority = HighlightLanguageTransform.default_priority + 1
 
     def apply(self, **kwargs: Any) -> None:
-        if not self.config.trim_doctest_flags:
-            return
-
         for lbnode in self.document.traverse(nodes.literal_block):  # type: nodes.literal_block
             if self.is_pyconsole(lbnode):
                 self.strip_doctest_flags(lbnode)
@@ -103,8 +100,10 @@ class TrimDoctestFlagsTransform(SphinxTransform):
         for dbnode in self.document.traverse(nodes.doctest_block):  # type: nodes.doctest_block
             self.strip_doctest_flags(dbnode)
 
-    @staticmethod
-    def strip_doctest_flags(node: Union[nodes.literal_block, nodes.doctest_block]) -> None:
+    def strip_doctest_flags(self, node: TextElement) -> None:
+        if not node.get('trim_flags', self.config.trim_doctest_flags):
+            return
+
         source = node.rawsource
         source = doctest.blankline_re.sub('', source)
         source = doctest.doctestopt_re.sub('', source)
diff --git a/tests/roots/test-trim_doctest_flags/index.rst b/tests/roots/test-trim_doctest_flags/index.rst
index 63ce234cd..d63251a5a 100644
--- a/tests/roots/test-trim_doctest_flags/index.rst
+++ b/tests/roots/test-trim_doctest_flags/index.rst
@@ -22,7 +22,19 @@ test-trim_doctest_flags
    >>> datetime.date.now()   # doctest: +QUX
    datetime.date(2008, 1, 1)
 
-.. doctest_block::
+.. doctest::
 
    >>> datetime.date.now()   # doctest: +QUUX
    datetime.date(2008, 1, 1)
+
+.. doctest::
+   :trim-doctest-flags:
+
+   >>> datetime.date.now()   # doctest: +CORGE
+   datetime.date(2008, 1, 1)
+
+.. doctest::
+   :no-trim-doctest-flags:
+
+   >>> datetime.date.now()   # doctest: +GRAULT
+   datetime.date(2008, 1, 1)
diff --git a/tests/test_transforms_post_transforms_code.py b/tests/test_transforms_post_transforms_code.py
index 880f4711b..2e5da05ae 100644
--- a/tests/test_transforms_post_transforms_code.py
+++ b/tests/test_transforms_post_transforms_code.py
@@ -19,6 +19,23 @@ def test_trim_doctest_flags_html(app, status, warning):
     assert 'BAZ' not in result
     assert 'QUX' not in result
     assert 'QUUX' not in result
+    assert 'CORGE' not in result
+    assert 'GRAULT' in result
+
+
+@pytest.mark.sphinx('html', testroot='trim_doctest_flags',
+                    confoverrides={'trim_doctest_flags': False})
+def test_trim_doctest_flags_disabled(app, status, warning):
+    app.build()
+
+    result = (app.outdir / 'index.html').read_text()
+    assert 'FOO' in result
+    assert 'BAR' in result
+    assert 'BAZ' in result
+    assert 'QUX' in result
+    assert 'QUUX' not in result
+    assert 'CORGE' not in result
+    assert 'GRAULT' in result
 
 
 @pytest.mark.sphinx('latex', testroot='trim_doctest_flags')
@@ -31,3 +48,5 @@ def test_trim_doctest_flags_latex(app, status, warning):
     assert 'BAZ' not in result
     assert 'QUX' not in result
     assert 'QUUX' not in result
+    assert 'CORGE' not in result
+    assert 'GRAULT' in result