Merge branch '3.x' into preprocess-other-sections

2 files changed, 68 insertions, 9 deletions

diff --git a/sphinx/ext/napoleon/docstring.py b/sphinx/ext/napoleon/docstring.py
index a41e9fdd4..1612a2d90 100644
--- a/sphinx/ext/napoleon/docstring.py
+++ b/sphinx/ext/napoleon/docstring.py
@@ -37,7 +37,7 @@ _xref_or_code_regex = re.compile(
     r'((?::(?:[a-zA-Z0-9]+[\-_+:.])*[a-zA-Z0-9]+:`.+?`)|'
     r'(?:``.+``))')
 _xref_regex = re.compile(
-    r'(?::(?:[a-zA-Z0-9]+[\-_+:.])*[a-zA-Z0-9]+:`.+?`)'
+    r'(?:(?::(?:[a-zA-Z0-9]+[\-_+:.])*[a-zA-Z0-9]+:)?`.+?`)'
 )
 _bullet_list_regex = re.compile(r'^(\*|\+|\-)(\s+\S|\s*$)')
 _enumerated_list_regex = re.compile(
@@ -45,10 +45,15 @@ _enumerated_list_regex = re.compile(
     r'(\d+|#|[ivxlcdm]+|[IVXLCDM]+|[a-zA-Z])'
     r'(?(paren)\)|\.)(\s+\S|\s*$)')
 _token_regex = re.compile(
-    r"(\sor\s|\sof\s|:\s|,\s|[{]|[}]"
+    r"(,\sor\s|\sor\s|\sof\s|:\s|\sto\s|,\sand\s|\sand\s|,\s"
+    r"|[{]|[}]"
     r'|"(?:\\"|[^"])*"'
     r"|'(?:\\'|[^'])*')"
 )
+_default_regex = re.compile(
+    r"^default[^_0-9A-Za-z].*$",
+)
+_SINGLETONS = ("None", "True", "False", "Ellipsis")
 
 
 class GoogleDocstring:
@@ -808,6 +813,9 @@ def _recombine_set_tokens(tokens: List[str]) -> List[str]:
                 previous_token = token
                 continue
 
+            if not token.strip():
+                continue
+
             if token in keywords:
                 tokens.appendleft(token)
                 if previous_token is not None:
@@ -846,8 +854,13 @@ def _recombine_set_tokens(tokens: List[str]) -> List[str]:
 
 def _tokenize_type_spec(spec: str) -> List[str]:
     def postprocess(item):
-        if item.startswith("default"):
-            return [item[:7], item[7:]]
+        if _default_regex.match(item):
+            default = item[:7]
+            # can't be separated by anything other than a single space
+            # for now
+            other = item[8:]
+
+            return [default, " ", other]
         else:
             return [item]
 
@@ -861,10 +874,19 @@ def _tokenize_type_spec(spec: str) -> List[str]:
 
 
 def _token_type(token: str, location: str = None) -> str:
+    def is_numeric(token):
+        try:
+            # use complex to make sure every numeric value is detected as literal
+            complex(token)
+        except ValueError:
+            return False
+        else:
+            return True
+
     if token.startswith(" ") or token.endswith(" "):
         type_ = "delimiter"
     elif (
-            token.isnumeric() or
+            is_numeric(token) or
             (token.startswith("{") and token.endswith("}")) or
             (token.startswith('"') and token.endswith('"')) or
             (token.startswith("'") and token.endswith("'"))
@@ -914,9 +936,12 @@ def _convert_numpy_type_spec(_type: str, location: str = None, translations: dic
     def convert_obj(obj, translations, default_translation):
         translation = translations.get(obj, obj)
 
-        # use :class: (the default) only if obj is not a standard singleton (None, True, False)
-        if translation in ("None", "True", "False") and default_translation == ":class:`%s`":
+        # use :class: (the default) only if obj is not a standard singleton
+        if translation in _SINGLETONS and default_translation == ":class:`%s`":
             default_translation = ":obj:`%s`"
+        elif translation == "..." and default_translation == ":class:`%s`":
+            # allow referencing the builtin ...
+            default_translation = ":obj:`%s <Ellipsis>`"
 
         if _xref_regex.match(translation) is None:
             translation = default_translation % translation
diff --git a/tests/test_ext_napoleon_docstring.py b/tests/test_ext_napoleon_docstring.py
index 5900f37d1..74e54eded 100644
--- a/tests/test_ext_napoleon_docstring.py
+++ b/tests/test_ext_napoleon_docstring.py
@@ -2062,6 +2062,8 @@ definition_after_normal_text : int
     def test_token_type(self):
         tokens = (
             ("1", "literal"),
+            ("-4.6", "literal"),
+            ("2j", "literal"),
             ("'string'", "literal"),
             ('"another_string"', "literal"),
             ("{1, 2}", "literal"),
@@ -2085,20 +2087,32 @@ definition_after_normal_text : int
     def test_tokenize_type_spec(self):
         specs = (
             "str",
+            "defaultdict",
+            "int, float, or complex",
             "int or float or None, optional",
             '{"F", "C", "N"}',
             "{'F', 'C', 'N'}, default: 'F'",
             "{'F', 'C', 'N or C'}, default 'F'",
+            "str, default: 'F or C'",
+            "int, default: None",
+            "int, default None",
+            "int, default :obj:`None`",
             '"ma{icious"',
             r"'with \'quotes\''",
         )
 
         tokens = (
             ["str"],
+            ["defaultdict"],
+            ["int", ", ", "float", ", or ", "complex"],
             ["int", " or ", "float", " or ", "None", ", ", "optional"],
             ["{", '"F"', ", ", '"C"', ", ", '"N"', "}"],
             ["{", "'F'", ", ", "'C'", ", ", "'N'", "}", ", ", "default", ": ", "'F'"],
             ["{", "'F'", ", ", "'C'", ", ", "'N or C'", "}", ", ", "default", " ", "'F'"],
+            ["str", ", ", "default", ": ", "'F or C'"],
+            ["int", ", ", "default", ": ", "None"],
+            ["int", ", " , "default", " ", "None"],
+            ["int", ", ", "default", " ", ":obj:`None`"],
             ['"ma{icious"'],
             [r"'with \'quotes\''"],
         )
@@ -2112,12 +2126,14 @@ definition_after_normal_text : int
             ["{", "1", ", ", "2", "}"],
             ["{", '"F"', ", ", '"C"', ", ", '"N"', "}", ", ", "optional"],
             ["{", "'F'", ", ", "'C'", ", ", "'N'", "}", ", ", "default", ": ", "None"],
+            ["{", "'F'", ", ", "'C'", ", ", "'N'", "}", ", ", "default", " ", "None"],
         )
 
         combined_tokens = (
             ["{1, 2}"],
             ['{"F", "C", "N"}', ", ", "optional"],
             ["{'F', 'C', 'N'}", ", ", "default", ": ", "None"],
+            ["{'F', 'C', 'N'}", ", ", "default", " ", "None"],
         )
 
         for tokens_, expected in zip(tokens, combined_tokens):
@@ -2150,8 +2166,10 @@ definition_after_normal_text : int
             "optional",
             "str, optional",
             "int or float or None, default: None",
+            "int, default None",
             '{"F", "C", "N"}',
             "{'F', 'C', 'N'}, default: 'N'",
+            "{'F', 'C', 'N'}, default 'N'",
             "DataFrame, optional",
         )
 
@@ -2160,8 +2178,10 @@ definition_after_normal_text : int
             "*optional*",
             ":class:`str`, *optional*",
             ":class:`int` or :class:`float` or :obj:`None`, *default*: :obj:`None`",
+            ":class:`int`, *default* :obj:`None`",
             '``{"F", "C", "N"}``',
             "``{'F', 'C', 'N'}``, *default*: ``'N'``",
+            "``{'F', 'C', 'N'}``, *default* ``'N'``",
             ":class:`pandas.DataFrame`, *optional*",
         )
 
@@ -2175,7 +2195,7 @@ definition_after_normal_text : int
             ----------
             param1 : DataFrame
                 the data to work on
-            param2 : int or float or None
+            param2 : int or float or None, optional
                 a parameter with different types
             param3 : dict-like, optional
                 a optional mapping
@@ -2183,21 +2203,35 @@ definition_after_normal_text : int
                 a optional parameter with different types
             param5 : {"F", "C", "N"}, optional
                 a optional parameter with fixed values
+            param6 : int, default None
+                different default format
+            param7 : mapping of hashable to str, optional
+                a optional mapping
+            param8 : ... or Ellipsis
+                ellipsis
         """)
         expected = dedent("""\
             :param param1: the data to work on
             :type param1: DataFrame
             :param param2: a parameter with different types
-            :type param2: :class:`int` or :class:`float` or :obj:`None`
+            :type param2: :class:`int` or :class:`float` or :obj:`None`, *optional*
             :param param3: a optional mapping
             :type param3: :term:`dict-like <mapping>`, *optional*
             :param param4: a optional parameter with different types
             :type param4: :class:`int` or :class:`float` or :obj:`None`, *optional*
             :param param5: a optional parameter with fixed values
             :type param5: ``{"F", "C", "N"}``, *optional*
+            :param param6: different default format
+            :type param6: :class:`int`, *default* :obj:`None`
+            :param param7: a optional mapping
+            :type param7: :term:`mapping` of :term:`hashable` to :class:`str`, *optional*
+            :param param8: ellipsis
+            :type param8: :obj:`... <Ellipsis>` or :obj:`Ellipsis`
         """)
         translations = {
             "dict-like": ":term:`dict-like <mapping>`",
+            "mapping": ":term:`mapping`",
+            "hashable": ":term:`hashable`",
         }
         config = Config(
             napoleon_use_param=True,