Adding --validate option __main__ and run new validation (#240)

* Adding --validate option __main__ and run new validation

* Fixing main tests, and minor improvements to the code

* Fixing travis negation of exit status

6 files changed, 144 insertions, 89 deletions

diff --git a/.travis.yml b/.travis.yml
index c1aa853..f6dd4cc 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -23,6 +23,12 @@ script:
     cd dist
     pip install numpydoc* -v
   - pytest -v --pyargs numpydoc
+  # Making sure the command line options work
+  - python -m numpydoc numpydoc.tests.test_main._capture_stdout
+  - echo '! python -m numpydoc numpydoc.tests.test_main._invalid_docstring' | bash
+  - python -m numpydoc --validate numpydoc.tests.test_main._capture_stdout
+  - echo '! python -m numpydoc --validate numpydoc.tests.test_main._docstring_with_errors' | bash
+  # Build documentation
   - |
     cd ../doc
     make SPHINXOPTS=$SPHINXOPTS html
diff --git a/doc/validation.rst b/doc/validation.rst
index c851668..3672d83 100644
--- a/doc/validation.rst
+++ b/doc/validation.rst
@@ -2,11 +2,16 @@
 Validating NumpyDoc docstrings
 ==============================
 
-One tool for validating docstrings is to see how an object's dosctring
-translates to Restructured Text.  Using numpydoc as a command-line tool
-facilitates this. For example to see the Restructured Text generated
-for ``numpy.ndarray``, use:
+To see the Restructured Text generated for an object, the ``numpydoc`` module
+can be called. For example, to do it for ``numpy.ndarray``, use:
 
 .. code-block:: bash
 
     $ python -m numpydoc numpy.ndarray
+
+This will validate that the docstring can be built.
+
+For an exhaustive validation of the formatting of the docstring, use the
+``--validate`` parameter. This will report the errors detected, such as
+incorrect capitalization, wrong order of the sections, and many other
+issues.
diff --git a/numpydoc/__init__.py b/numpydoc/__init__.py
index da8b80a..8058bb5 100644
--- a/numpydoc/__init__.py
+++ b/numpydoc/__init__.py
@@ -1,3 +1,7 @@
+"""
+This package provides the numpydoc Sphinx extension for handling docstrings
+formatted according to the NumPy documentation format.
+"""
 __version__ = '1.0.0.dev0'
 
 
diff --git a/numpydoc/__main__.py b/numpydoc/__main__.py
index 25f58cf..534e9e0 100644
--- a/numpydoc/__main__.py
+++ b/numpydoc/__main__.py
@@ -1,13 +1,32 @@
+"""
+Implementing `python -m numpydoc` functionality.
+"""
+import sys
 import argparse
-import importlib
 import ast
 
 from .docscrape_sphinx import get_doc_object
+from .validate import validate, Docstring
 
 
-def main(argv=None):
+def render_object(import_path, config=None):
     """Test numpydoc docstring generation for a given object"""
+    # TODO: Move Docstring._load_obj to a better place than validate
+    print(get_doc_object(Docstring(import_path).obj,
+                         config=dict(config or [])))
+    return 0
 
+
+def validate_object(import_path):
+    exit_status = 0
+    results = validate(import_path)
+    for err_code, err_desc in results["errors"]:
+        exit_status += 1
+        print(':'.join([import_path, err_code, err_desc]))
+    return exit_status
+
+
+if __name__ == '__main__':
     ap = argparse.ArgumentParser(description=__doc__)
     ap.add_argument('import_path', help='e.g. numpy.ndarray')
 
@@ -20,25 +39,13 @@ def main(argv=None):
                     action='append',
                     help='key=val where val will be parsed by literal_eval, '
                          'e.g. -c use_plots=True. Multiple -c can be used.')
-    args = ap.parse_args(argv)
+    ap.add_argument('--validate', action='store_true',
+                    help='validate the object and report errors')
+    args = ap.parse_args()
 
-    parts = args.import_path.split('.')
-
-    for split_point in range(len(parts), 0, -1):
-        try:
-            path = '.'.join(parts[:split_point])
-            obj = importlib.import_module(path)
-        except ImportError:
-            continue
-        break
+    if args.validate:
+        exit_code = validate_object(args.import_path)
     else:
-        raise ImportError('Could not resolve {!r} to an importable object'
-                          ''.format(args.import_path))
-
-    for part in parts[split_point:]:
-        obj = getattr(obj, part)
+        exit_code = render_object(args.import_path, args.config)
 
-    print(get_doc_object(obj, config=dict(args.config or [])))
-
-if __name__ == '__main__':
-    main()
+    sys.exit(exit_code)
diff --git a/numpydoc/tests/test_main.py b/numpydoc/tests/test_main.py
index e565bb2..d7f6657 100644
--- a/numpydoc/tests/test_main.py
+++ b/numpydoc/tests/test_main.py
@@ -1,80 +1,113 @@
-from __future__ import print_function
-
-from contextlib import contextmanager
-import os
 import sys
-import tempfile
-try:
-    from StringIO import StringIO
-except ImportError:
-    from io import StringIO
+import io
+import pytest
+import numpydoc
+import numpydoc.__main__
 
-from numpydoc.__main__ import main
 
+def _capture_stdout(func_name, *args, **kwargs):
+    """
+    Return stdout of calling `func_name`.
 
-PACKAGE_CODE = """
-'''This package has test stuff'''
-"""
+    This docstring should be perfect, as it is used to test the
+    validation with a docstring without errors.
+
+    Parameters
+    ----------
+    func_name : callable
+        Function to be called.
+    *args, **kwargs
+        Will be passed to `func_name`.
+
+    Returns
+    -------
+    str
+        The content that the function printed.
+
+    See Also
+    --------
+    sys.stdout : Python's file handler for stdout.
+
+    Examples
+    --------
+    >>> _capture_stdout(print, 'hello world')
+    'hello world'
+    """
+    f = io.StringIO()
+    sys.stdout, old_stdout = f, sys.stdout
+    try:
+        func_name(*args, **kwargs)
+        return f.getvalue().strip('\n\r')
+    finally:
+        sys.stdout = old_stdout
 
-MODULE_CODE = """
-'''This module has test stuff'''
 
-def foo(a, b=5):
-    '''Hello world
+def _docstring_with_errors():
+    """
+    this docstring should report some errors
 
     Parameters
     ----------
-    something : foo
-        bar
-    something_else
-        bar
-    '''
-"""
+    made_up_param : str
+    """
+    pass
 
 
-@contextmanager
-def _mock_module(pkg_name):
-    try:
-        tempdir = tempfile.mkdtemp()
-        os.mkdir(os.path.join(tempdir, pkg_name))
-        with open(os.path.join(tempdir, pkg_name, '__init__.py'), 'w') as f:
-            print(PACKAGE_CODE, file=f)
-        with open(os.path.join(tempdir, pkg_name, 'module.py'), 'w') as f:
-            print(MODULE_CODE, file=f)
-
-        sys.path.insert(0, tempdir)
-        yield tempdir
-    finally:
-        try:
-            os.path.rmdir(tempdir)
-            sys.path.remove(tempdir)
-        except:
-            pass
+def _invalid_docstring():
+    """
+    This docstring should break the parsing.
 
+    See Also
+    --------
+    : this is invalid
+    """
+    pass
 
-def _capture_main(*args):
-    f = StringIO()
-    sys.stdout, old_stdout = f, sys.stdout
-    try:
-        main(args)
-        return f.getvalue().strip('\n\r')
-    finally:
-        sys.stdout = old_stdout
 
+def test_renders_package_docstring():
+    out = _capture_stdout(numpydoc.__main__.render_object,
+                          'numpydoc')
+    assert out.startswith('This package provides the numpydoc Sphinx')
+
+
+def test_renders_module_docstring():
+    out = _capture_stdout(numpydoc.__main__.render_object,
+                          'numpydoc.__main__')
+    assert out.startswith('Implementing `python -m numpydoc` functionality.')
+
+
+def test_renders_function_docstring():
+    out = _capture_stdout(numpydoc.__main__.render_object,
+                          'numpydoc.tests.test_main._capture_stdout')
+    assert out.startswith('Return stdout of calling')
+
+
+def test_render_object_returns_correct_exit_status():
+    exit_status = numpydoc.__main__.render_object(
+        'numpydoc.tests.test_main._capture_stdout')
+    assert exit_status == 0
+
+    with pytest.raises(numpydoc.docscrape.ParseError):
+        numpydoc.__main__.render_object(
+            'numpydoc.tests.test_main._invalid_docstring')
+
+
+def test_validate_detects_errors():
+    out = _capture_stdout(numpydoc.__main__.validate_object,
+                          'numpydoc.tests.test_main._docstring_with_errors')
+    assert 'SS02' in out
+    assert 'Summary does not start with a capital letter' in out
+
+    exit_status = numpydoc.__main__.validate_object(
+        'numpydoc.tests.test_main._docstring_with_errors')
+    assert exit_status > 0
 
-def test_main():
-    # TODO: does not currently check that numpydoc transformations are applied
 
-    assert (_capture_main('numpydoc.__main__.main') ==
-            main.__doc__.strip())
+def test_validate_perfect_docstring():
+    out = _capture_stdout(numpydoc.__main__.validate_object,
+                          'numpydoc.tests.test_main._capture_stdout')
+    assert out == ''
 
-    # check it works with modules not imported from __init__
-    with _mock_module('somepackage1'):
-        out = _capture_main('somepackage1.module.foo')
-    assert out.startswith('Hello world\n')
-    with _mock_module('somepackage2'):
-        out = _capture_main('somepackage2.module')
-    assert out.startswith('This module has test')
-    with _mock_module('somepackage3'):
-        out = _capture_main('somepackage3')
-    assert out.startswith('This package has test')
+    exit_status = numpydoc.__main__.validate_object(
+        'numpydoc.tests.test_main._capture_stdout')
+    assert exit_status == 0
diff --git a/numpydoc/validate.py b/numpydoc/validate.py
index fe0473c..1fd80e1 100644
--- a/numpydoc/validate.py
+++ b/numpydoc/validate.py
@@ -152,7 +152,7 @@ class Docstring:
         >>> Docstring._load_obj('datetime.datetime')
         <class 'datetime.datetime'>
         """
-        for maxsplit in range(1, name.count(".") + 1):
+        for maxsplit in range(0, name.count(".") + 1):
             module, *func_parts = name.rsplit(".", maxsplit)
             try:
                 obj = importlib.import_module(module)