Fix autoclass signature parsing

This fixes:
* Signatures defined by __new__
* Signatures defined by metaclasses
* Signatures defined by builtin base classes

All of these changes bring the sphinx docs inline with the behavior of `inspect.signature`.

Note that this changes autodoc to output `.. py:class: MyClass()` with parentheses even if no user-defined __init__ is present.
This is quite deliberate, as if no user-defined `__init__` is present the default is `object.__init__`, which indeed does not take arguments.

6 files changed, 214 insertions, 67 deletions

diff --git a/sphinx/ext/autodoc/__init__.py b/sphinx/ext/autodoc/__init__.py
index 027fb0869..89de4372a 100644
--- a/sphinx/ext/autodoc/__init__.py
+++ b/sphinx/ext/autodoc/__init__.py
@@ -1205,6 +1205,14 @@ class DecoratorDocumenter(FunctionDocumenter):
             return None
 
 
+# Types which have confusing metaclass signatures it would be best not to show.
+# These are listed by name, rather than storing the objects themselves, to avoid
+# needing to import the modules.
+_METACLASS_CALL_BLACKLIST = [
+    'enum.EnumMeta.__call__',
+]
+
+
 class ClassDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter):  # type: ignore
     """
     Specialized Documenter subclass for classes.
@@ -1243,22 +1251,72 @@ class ClassDocumenter(DocstringSignatureMixin, ModuleLevelDocumenter):  # type:
         if self.env.config.autodoc_typehints in ('none', 'description'):
             kwargs.setdefault('show_annotation', False)
 
-        # for classes, the relevant signature is the __init__ method's
-        initmeth = self.get_attr(self.object, '__init__', None)
-        # classes without __init__ method, default __init__ or
-        # __init__ written in C?
-        if initmeth is None or \
-                inspect.is_builtin_class_method(self.object, '__init__') or \
-                not(inspect.ismethod(initmeth) or inspect.isfunction(initmeth)):
-            return None
-        try:
-            self.env.app.emit('autodoc-before-process-signature', initmeth, True)
-            sig = inspect.signature(initmeth, bound_method=True)
+        def get_user_defined_function_or_method(obj: Any, attr: str) -> Any:
+            """ Get the `attr` function or method from `obj`, if it is user-defined. """
+            if inspect.is_builtin_class_method(obj, attr):
+                return None
+            attr = self.get_attr(obj, attr, None)
+            if not (inspect.ismethod(attr) or inspect.isfunction(attr)):
+                return None
+            return attr
+
+        sig = None
+
+        # this sequence is copied from inspect._signature_from_callable
+
+        # First, let's see if it has an overloaded __call__ defined
+        # in its metaclass
+        if sig is None:
+            call = get_user_defined_function_or_method(type(self.object), '__call__')
+
+            if call is not None:
+                if "{0.__module__}.{0.__qualname__}".format(call) in _METACLASS_CALL_BLACKLIST:
+                    call = None
+
+            if call is not None:
+                self.env.app.emit('autodoc-before-process-signature', call, True)
+                try:
+                    sig = inspect.signature(call, bound_method=True)
+                except ValueError:
+                    pass
+
+        # Now we check if the 'obj' class has a '__new__' method
+        if sig is None:
+            new = get_user_defined_function_or_method(self.object, '__new__')
+            if new is not None:
+                self.env.app.emit('autodoc-before-process-signature', new, True)
+                try:
+                    sig = inspect.signature(new, bound_method=True)
+                except ValueError:
+                    pass
+
+        # Finally, we should have at least __init__ implemented
+        if sig is None:
+            init = get_user_defined_function_or_method(self.object, '__init__')
+            if init is not None:
+                self.env.app.emit('autodoc-before-process-signature', init, True)
+                try:
+                    sig = inspect.signature(init, bound_method=True)
+                except ValueError:
+                    pass
+
+        # None of the attributes are user-defined, so fall back to let inspect
+        # handle it.
+        if sig is None:
+            # We don't know the exact method that inspect.signature will read
+            # the signature from, so just pass the object itself to our hook.
+            self.env.app.emit('autodoc-before-process-signature', self.object, False)
+            try:
+                sig = inspect.signature(self.object, bound_method=False)
+            except ValueError:
+                pass
+
+        if sig is not None:
             return stringify_signature(sig, show_return_annotation=False, **kwargs)
-        except TypeError:
-            # still not possible: happens e.g. for old-style classes
-            # with __init__ in C
-            return None
+
+        # Still no signature: happens e.g. for old-style classes
+        # with __init__ in C and no `__text_signature__`.
+        return None
 
     def format_signature(self, **kwargs: Any) -> str:
         if self.doc_as_attr:
diff --git a/tests/roots/test-ext-autodoc/target/typehints.py b/tests/roots/test-ext-autodoc/target/typehints.py
index 4503d41e4..1a70eca67 100644
--- a/tests/roots/test-ext-autodoc/target/typehints.py
+++ b/tests/roots/test-ext-autodoc/target/typehints.py
@@ -37,6 +37,26 @@ def tuple_args(x: Tuple[int, Union[int, str]]) -> Tuple[int, int]:
     pass
 
 
+class NewAnnotation:
+    def __new__(cls, i: int) -> 'NewAnnotation':
+        pass
+
+
+class NewComment:
+    def __new__(cls, i):
+        # type: (int) -> NewComment
+        pass
+
+
+class _MetaclassWithCall(type):
+    def __call__(cls, a: int):
+        pass
+
+
+class SignatureFromMetaclass(metaclass=_MetaclassWithCall):
+    pass
+
+
 def complex_func(arg1, arg2, arg3=None, *args, **kwargs):
     # type: (str, List[int], Tuple[int, Union[str, Unknown]], *str, **str) -> None
     pass
@@ -48,4 +68,3 @@ def missing_attr(c,
                  ):
     # type: (...) -> str
     return a + (b or "")
-
diff --git a/tests/test_ext_autodoc.py b/tests/test_ext_autodoc.py
index 8a3afa43d..c1799778c 100644
--- a/tests/test_ext_autodoc.py
+++ b/tests/test_ext_autodoc.py
@@ -169,21 +169,64 @@ def test_format_signature(app):
         pass
 
     class E:
-        pass
-    # no signature for classes without __init__
+        def __init__(self):
+            pass
+
+    # an empty init and no init are the same
     for C in (D, E):
-        assert formatsig('class', 'D', C, None, None) == ''
+        assert formatsig('class', 'D', C, None, None) == '()'
+
 
+    class SomeMeta(type):
+        def __call__(cls, a, b=None):
+            return type.__call__(cls, a, b)
+
+    # these three are all equivalent
     class F:
         def __init__(self, a, b=None):
             pass
 
+    class FNew:
+        def __new__(cls, a, b=None):
+            return super().__new__(cls)
+
+    class FMeta(metaclass=SomeMeta):
+        pass
+
+    # and subclasses should always inherit
     class G(F):
         pass
-    for C in (F, G):
+
+    class GNew(FNew):
+        pass
+
+    class GMeta(FMeta):
+        pass
+
+    # subclasses inherit
+    for C in (F, FNew, FMeta, G, GNew, GMeta):
         assert formatsig('class', 'C', C, None, None) == '(a, b=None)'
     assert formatsig('class', 'C', D, 'a, b', 'X') == '(a, b) -> X'
 
+
+    class ListSubclass(list):
+        pass
+
+    # only supported if the python implementation decides to document it
+    if getattr(list, '__text_signature__', None) is not None:
+        assert formatsig('class', 'C', ListSubclass, None, None) == '(iterable=(), /)'
+    else:
+        assert formatsig('class', 'C', ListSubclass, None, None) == ''
+
+
+    class ExceptionSubclass(Exception):
+        pass
+
+    # Exception has no __text_signature__ at least in Python 3.8
+    if getattr(Exception, '__text_signature__', None) is None:
+        assert formatsig('class', 'C', ExceptionSubclass, None, None) == ''
+
+
     # __init__ have signature at first line of docstring
     directive.env.config.autoclass_content = 'both'
 
@@ -497,14 +540,14 @@ def test_autodoc_members(app):
     # default (no-members)
     actual = do_autodoc(app, 'class', 'target.inheritance.Base')
     assert list(filter(lambda l: '::' in l, actual)) == [
-        '.. py:class:: Base',
+        '.. py:class:: Base()',
     ]
 
     # default ALL-members
     options = {"members": None}
     actual = do_autodoc(app, 'class', 'target.inheritance.Base', options)
     assert list(filter(lambda l: '::' in l, actual)) == [
-        '.. py:class:: Base',
+        '.. py:class:: Base()',
         '   .. py:method:: Base.inheritedclassmeth()',
         '   .. py:method:: Base.inheritedmeth()',
         '   .. py:method:: Base.inheritedstaticmeth(cls)'
@@ -514,7 +557,7 @@ def test_autodoc_members(app):
     options = {"members": "inheritedmeth,inheritedstaticmeth"}
     actual = do_autodoc(app, 'class', 'target.inheritance.Base', options)
     assert list(filter(lambda l: '::' in l, actual)) == [
-        '.. py:class:: Base',
+        '.. py:class:: Base()',
         '   .. py:method:: Base.inheritedmeth()',
         '   .. py:method:: Base.inheritedstaticmeth(cls)'
     ]
@@ -526,7 +569,7 @@ def test_autodoc_exclude_members(app):
                "exclude-members": "inheritedmeth,inheritedstaticmeth"}
     actual = do_autodoc(app, 'class', 'target.inheritance.Base', options)
     assert list(filter(lambda l: '::' in l, actual)) == [
-        '.. py:class:: Base',
+        '.. py:class:: Base()',
         '   .. py:method:: Base.inheritedclassmeth()'
     ]
 
@@ -535,7 +578,7 @@ def test_autodoc_exclude_members(app):
                "exclude-members": "inheritedmeth"}
     actual = do_autodoc(app, 'class', 'target.inheritance.Base', options)
     assert list(filter(lambda l: '::' in l, actual)) == [
-        '.. py:class:: Base',
+        '.. py:class:: Base()',
     ]
 
 
@@ -679,10 +722,10 @@ def test_autodoc_ignore_module_all(app):
     assert list(filter(lambda l: 'class::' in l, actual)) == [
         '.. py:class:: Class(arg)',
         '.. py:class:: CustomDict',
-        '.. py:class:: InnerChild',
+        '.. py:class:: InnerChild()',
         '.. py:class:: InstAttCls()',
-        '.. py:class:: Outer',
-        '   .. py:class:: Outer.Inner',
+        '.. py:class:: Outer()',
+        '   .. py:class:: Outer.Inner()',
         '.. py:class:: StrRepr'
     ]
 
@@ -703,7 +746,7 @@ def test_autodoc_noindex(app):
     actual = do_autodoc(app, 'class', 'target.inheritance.Base', options)
     assert list(actual) == [
         '',
-        '.. py:class:: Base',
+        '.. py:class:: Base()',
         '   :noindex:',
         '   :module: target.inheritance',
         ''
@@ -730,13 +773,13 @@ def test_autodoc_inner_class(app):
     actual = do_autodoc(app, 'class', 'target.Outer', options)
     assert list(actual) == [
         '',
-        '.. py:class:: Outer',
+        '.. py:class:: Outer()',
         '   :module: target',
         '',
         '   Foo',
         '',
         '',
-        '   .. py:class:: Outer.Inner',
+        '   .. py:class:: Outer.Inner()',
         '      :module: target',
         '',
         '      Foo',
@@ -757,7 +800,7 @@ def test_autodoc_inner_class(app):
     actual = do_autodoc(app, 'class', 'target.Outer.Inner', options)
     assert list(actual) == [
         '',
-        '.. py:class:: Outer.Inner',
+        '.. py:class:: Outer.Inner()',
         '   :module: target',
         '',
         '   Foo',
@@ -774,7 +817,7 @@ def test_autodoc_inner_class(app):
     actual = do_autodoc(app, 'class', 'target.InnerChild', options)
     assert list(actual) == [
         '',
-        '.. py:class:: InnerChild',
+        '.. py:class:: InnerChild()',
         '   :module: target', '',
         '   Bases: :class:`target.Outer.Inner`',
         '',
@@ -818,7 +861,7 @@ def test_autodoc_descriptor(app):
     actual = do_autodoc(app, 'class', 'target.descriptor.Class', options)
     assert list(actual) == [
         '',
-        '.. py:class:: Class',
+        '.. py:class:: Class()',
         '   :module: target.descriptor',
         '',
         '',
@@ -925,8 +968,8 @@ def test_autodoc_module_member_order(app):
         '.. py:module:: target.sort_by_all',
         '.. py:function:: baz()',
         '.. py:function:: foo()',
-        '.. py:class:: Bar',
-        '.. py:class:: Quux',
+        '.. py:class:: Bar()',
+        '.. py:class:: Quux()',
         '.. py:function:: foobar()',
         '.. py:function:: qux()',
     ]
@@ -940,10 +983,10 @@ def test_autodoc_module_member_order(app):
     assert list(filter(lambda l: '::' in l, actual)) == [
         '.. py:module:: target.sort_by_all',
         '.. py:function:: foo()',
-        '.. py:class:: Bar',
+        '.. py:class:: Bar()',
         '.. py:function:: baz()',
         '.. py:function:: qux()',
-        '.. py:class:: Quux',
+        '.. py:class:: Quux()',
         '.. py:function:: foobar()',
     ]
 
@@ -986,7 +1029,7 @@ def test_class_attributes(app):
     actual = do_autodoc(app, 'class', 'target.AttCls', options)
     assert list(actual) == [
         '',
-        '.. py:class:: AttCls',
+        '.. py:class:: AttCls()',
         '   :module: target',
         '',
         '',
@@ -1106,7 +1149,7 @@ def test_slots(app):
         '      :module: target.slots',
         '',
         '',
-        '.. py:class:: Foo',
+        '.. py:class:: Foo()',
         '   :module: target.slots',
         '',
         '',
@@ -1122,7 +1165,7 @@ def test_enum_class(app):
     actual = do_autodoc(app, 'class', 'target.enum.EnumCls', options)
     assert list(actual) == [
         '',
-        '.. py:class:: EnumCls',
+        '.. py:class:: EnumCls(value)',
         '   :module: target.enum',
         '',
         '   this is enum class',
@@ -1239,7 +1282,7 @@ def test_abstractmethods(app):
         '.. py:module:: target.abstractmethods',
         '',
         '',
-        '.. py:class:: Base',
+        '.. py:class:: Base()',
         '   :module: target.abstractmethods',
         '',
         '',
@@ -1356,7 +1399,7 @@ def test_coroutine(app):
     actual = do_autodoc(app, 'class', 'target.coroutine.AsyncClass', options)
     assert list(actual) == [
         '',
-        '.. py:class:: AsyncClass',
+        '.. py:class:: AsyncClass()',
         '   :module: target.coroutine',
         '',
         '',
@@ -1398,7 +1441,7 @@ def test_coroutine(app):
 def test_partialmethod(app):
     expected = [
         '',
-        '.. py:class:: Cell',
+        '.. py:class:: Cell()',
         '   :module: target.partialmethod',
         '',
         '   An example for partialmethod.',
@@ -1428,7 +1471,7 @@ def test_partialmethod(app):
 def test_partialmethod_undoc_members(app):
     expected = [
         '',
-        '.. py:class:: Cell',
+        '.. py:class:: Cell()',
         '   :module: target.partialmethod',
         '',
         '   An example for partialmethod.',
@@ -1615,7 +1658,7 @@ def test_singledispatchmethod(app):
         '.. py:module:: target.singledispatchmethod',
         '',
         '',
-        '.. py:class:: Foo',
+        '.. py:class:: Foo()',
         '   :module: target.singledispatchmethod',
         '',
         '   docstring',
@@ -1660,7 +1703,7 @@ def test_cython(app):
         '.. py:module:: target.cython',
         '',
         '',
-        '.. py:class:: Class',
+        '.. py:class:: Class()',
         '   :module: target.cython',
         '',
         '   Docstring.',
@@ -1691,7 +1734,7 @@ def test_final(app):
         '.. py:module:: target.final',
         '',
         '',
-        '.. py:class:: Class',
+        '.. py:class:: Class()',
         '   :module: target.final',
         '   :final:',
         '',
diff --git a/tests/test_ext_autodoc_configs.py b/tests/test_ext_autodoc_configs.py
index 6821c6264..22558885b 100644
--- a/tests/test_ext_autodoc_configs.py
+++ b/tests/test_ext_autodoc_configs.py
@@ -9,6 +9,7 @@
 """
 
 import platform
+import sys
 
 import pytest
 
@@ -27,7 +28,7 @@ def test_autoclass_content_class(app):
         '.. py:module:: target.autoclass_content',
         '',
         '',
-        '.. py:class:: A',
+        '.. py:class:: A()',
         '   :module: target.autoclass_content',
         '',
         '   A class having no __init__, no __new__',
@@ -45,13 +46,13 @@ def test_autoclass_content_class(app):
         '   A class having __init__, no __new__',
         '',
         '',
-        '.. py:class:: D',
+        '.. py:class:: D()',
         '   :module: target.autoclass_content',
         '',
         '   A class having no __init__, __new__(no docstring)',
         '',
         '',
-        '.. py:class:: E',
+        '.. py:class:: E()',
         '   :module: target.autoclass_content',
         '',
         '   A class having no __init__, __new__',
@@ -87,7 +88,7 @@ def test_autoclass_content_init(app):
         '.. py:module:: target.autoclass_content',
         '',
         '',
-        '.. py:class:: A',
+        '.. py:class:: A()',
         '   :module: target.autoclass_content',
         '',
         '   A class having no __init__, no __new__',
@@ -105,13 +106,13 @@ def test_autoclass_content_init(app):
         '   __init__ docstring',
         '',
         '',
-        '.. py:class:: D',
+        '.. py:class:: D()',
         '   :module: target.autoclass_content',
         '',
         '   A class having no __init__, __new__(no docstring)',
         '',
         '',
-        '.. py:class:: E',
+        '.. py:class:: E()',
         '   :module: target.autoclass_content',
         '',
         '   __new__ docstring',
@@ -147,7 +148,7 @@ def test_autoclass_content_both(app):
         '.. py:module:: target.autoclass_content',
         '',
         '',
-        '.. py:class:: A',
+        '.. py:class:: A()',
         '   :module: target.autoclass_content',
         '',
         '   A class having no __init__, no __new__',
@@ -167,13 +168,13 @@ def test_autoclass_content_both(app):
         '   __init__ docstring',
         '',
         '',
-        '.. py:class:: D',
+        '.. py:class:: D()',
         '   :module: target.autoclass_content',
         '',
         '   A class having no __init__, __new__(no docstring)',
         '',
         '',
-        '.. py:class:: E',
+        '.. py:class:: E()',
         '   :module: target.autoclass_content',
         '',
         '   A class having no __init__, __new__',
@@ -237,7 +238,7 @@ def test_autodoc_docstring_signature(app):
     actual = do_autodoc(app, 'class', 'target.DocstringSig', options)
     assert list(actual) == [
         '',
-        '.. py:class:: DocstringSig',
+        '.. py:class:: DocstringSig()',
         '   :module: target',
         '',
         '',
@@ -279,7 +280,7 @@ def test_autodoc_docstring_signature(app):
     actual = do_autodoc(app, 'class', 'target.DocstringSig', options)
     assert list(actual) == [
         '',
-        '.. py:class:: DocstringSig',
+        '.. py:class:: DocstringSig()',
         '   :module: target',
         '',
         '',
@@ -435,7 +436,7 @@ def test_mocked_module_imports(app, warning):
         '.. py:module:: target.need_mocks',
         '',
         '',
-        '.. py:class:: TestAutodoc',
+        '.. py:class:: TestAutodoc()',
         '   :module: target.need_mocks',
         '',
         '   TestAutodoc docstring.',
@@ -493,6 +494,18 @@ def test_autodoc_typehints_signature(app):
         '      :module: target.typehints',
         '',
         '',
+        '.. py:class:: NewAnnotation(i: int)',
+        '   :module: target.typehints',
+        '',
+        '',
+        '.. py:class:: NewComment(i: int)',
+        '   :module: target.typehints',
+        '',
+        '',
+        '.. py:class:: SignatureFromMetaclass(a: int)',
+        '   :module: target.typehints',
+        '',
+        '',
         '.. py:function:: complex_func(arg1: str, arg2: List[int], arg3: Tuple[int, '
         'Union[str, Unknown]] = None, *args: str, **kwargs: str) -> None',
         '   :module: target.typehints',
@@ -547,6 +560,18 @@ def test_autodoc_typehints_none(app):
         '      :module: target.typehints',
         '',
         '',
+        '.. py:class:: NewAnnotation(i)',
+        '   :module: target.typehints',
+        '',
+        '',
+        '.. py:class:: NewComment(i)',
+        '   :module: target.typehints',
+        '',
+        '',
+        '.. py:class:: SignatureFromMetaclass(a)',
+        '   :module: target.typehints',
+        '',
+        '',
         '.. py:function:: complex_func(arg1, arg2, arg3=None, *args, **kwargs)',
         '   :module: target.typehints',
         '',
diff --git a/tests/test_ext_autosummary.py b/tests/test_ext_autosummary.py
index 281ba141e..a65826141 100644
--- a/tests/test_ext_autosummary.py
+++ b/tests/test_ext_autosummary.py
@@ -292,7 +292,7 @@ def test_autosummary_generate(app, status, warning):
     assert len(doctree[3][0][0][2]) == 5
     assert doctree[3][0][0][2][0].astext() == 'autosummary_dummy_module\n\n'
     assert doctree[3][0][0][2][1].astext() == 'autosummary_dummy_module.Foo()\n\n'
-    assert doctree[3][0][0][2][2].astext() == 'autosummary_dummy_module.Foo.Bar\n\n'
+    assert doctree[3][0][0][2][2].astext() == 'autosummary_dummy_module.Foo.Bar()\n\n'
     assert doctree[3][0][0][2][3].astext() == 'autosummary_dummy_module.bar(x[, y])\n\n'
     assert doctree[3][0][0][2][4].astext() == 'autosummary_dummy_module.qux\n\na module-level attribute'
 
diff --git a/tests/test_util_inspect.py b/tests/test_util_inspect.py
index 4da61df47..f16feb698 100644
--- a/tests/test_util_inspect.py
+++ b/tests/test_util_inspect.py
@@ -29,12 +29,14 @@ def test_signature():
     with pytest.raises(TypeError):
         inspect.signature('')
 
-    # builitin classes
-    with pytest.raises(ValueError):
-        inspect.signature(int)
-
-    with pytest.raises(ValueError):
-        inspect.signature(str)
+    # builtins are supported on a case-by-case basis, depending on whether
+    # they define __text_signature__
+    if getattr(list, '__text_signature__', None):
+        sig = inspect.stringify_signature(inspect.signature(list))
+        assert sig == '(iterable=(), /)'
+    else:
+        with pytest.raises(ValueError):
+            inspect.signature(list)
 
     # normal function
     def func(a, b, c=1, d=2, *e, **f):