1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
|
"""
sphinx.pycode
~~~~~~~~~~~~~
Utilities parsing and analyzing Python code.
:copyright: Copyright 2007-2020 by the Sphinx team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
import re
import tokenize
import warnings
from importlib import import_module
from io import StringIO
from os import path
from typing import Any, Dict, IO, List, Tuple, Optional
from zipfile import ZipFile
from sphinx.deprecation import RemovedInSphinx40Warning
from sphinx.errors import PycodeError
from sphinx.pycode.parser import Parser
class ModuleAnalyzer:
# cache for analyzer objects -- caches both by module and file name
cache = {} # type: Dict[Tuple[str, str], Any]
@staticmethod
def get_module_source(modname: str) -> Tuple[Optional[str], Optional[str]]:
"""Try to find the source code for a module.
Returns ('filename', 'source'). One of it can be None if
no filename or source found
"""
try:
mod = import_module(modname)
except Exception as err:
raise PycodeError('error importing %r' % modname, err)
loader = getattr(mod, '__loader__', None)
filename = getattr(mod, '__file__', None)
if loader and getattr(loader, 'get_source', None):
# prefer Native loader, as it respects #coding directive
try:
source = loader.get_source(modname)
if source:
# no exception and not None - it must be module source
return filename, source
except ImportError:
pass # Try other "source-mining" methods
if filename is None and loader and getattr(loader, 'get_filename', None):
# have loader, but no filename
try:
filename = loader.get_filename(modname)
except ImportError as err:
raise PycodeError('error getting filename for %r' % modname, err)
if filename is None:
# all methods for getting filename failed, so raise...
raise PycodeError('no source found for module %r' % modname)
filename = path.normpath(path.abspath(filename))
if filename.lower().endswith(('.pyo', '.pyc')):
filename = filename[:-1]
if not path.isfile(filename) and path.isfile(filename + 'w'):
filename += 'w'
elif not filename.lower().endswith(('.py', '.pyw')):
raise PycodeError('source is not a .py file: %r' % filename)
elif ('.egg' + path.sep) in filename:
pat = '(?<=\\.egg)' + re.escape(path.sep)
eggpath, _ = re.split(pat, filename, 1)
if path.isfile(eggpath):
return filename, None
if not path.isfile(filename):
raise PycodeError('source file is not present: %r' % filename)
return filename, None
@classmethod
def for_string(cls, string: str, modname: str, srcname: str = '<string>'
) -> "ModuleAnalyzer":
return cls(StringIO(string), modname, srcname, decoded=True)
@classmethod
def for_file(cls, filename: str, modname: str) -> "ModuleAnalyzer":
if ('file', filename) in cls.cache:
return cls.cache['file', filename]
try:
with tokenize.open(filename) as f:
obj = cls(f, modname, filename, decoded=True)
cls.cache['file', filename] = obj
except Exception as err:
if '.egg' + path.sep in filename:
obj = cls.cache['file', filename] = cls.for_egg(filename, modname)
else:
raise PycodeError('error opening %r' % filename, err)
return obj
@classmethod
def for_egg(cls, filename: str, modname: str) -> "ModuleAnalyzer":
SEP = re.escape(path.sep)
eggpath, relpath = re.split('(?<=\\.egg)' + SEP, filename)
try:
with ZipFile(eggpath) as egg:
code = egg.read(relpath).decode()
return cls.for_string(code, modname, filename)
except Exception as exc:
raise PycodeError('error opening %r' % filename, exc)
@classmethod
def for_module(cls, modname: str) -> "ModuleAnalyzer":
if ('module', modname) in cls.cache:
entry = cls.cache['module', modname]
if isinstance(entry, PycodeError):
raise entry
return entry
try:
filename, source = cls.get_module_source(modname)
if source is not None:
obj = cls.for_string(source, modname, filename or '<string>')
elif filename is not None:
obj = cls.for_file(filename, modname)
except PycodeError as err:
cls.cache['module', modname] = err
raise
cls.cache['module', modname] = obj
return obj
def __init__(self, source: IO, modname: str, srcname: str, decoded: bool = False) -> None:
self.modname = modname # name of the module
self.srcname = srcname # name of the source file
# cache the source code as well
pos = source.tell()
if not decoded:
warnings.warn('decode option for ModuleAnalyzer is deprecated.',
RemovedInSphinx40Warning)
self._encoding, _ = tokenize.detect_encoding(source.readline)
source.seek(pos)
self.code = source.read().decode(self._encoding)
else:
self._encoding = None
self.code = source.read()
# will be filled by parse()
self.annotations = None # type: Dict[Tuple[str, str], str]
self.attr_docs = None # type: Dict[Tuple[str, str], List[str]]
self.finals = None # type: List[str]
self.tagorder = None # type: Dict[str, int]
self.tags = None # type: Dict[str, Tuple[str, int, int]]
def parse(self) -> None:
"""Parse the source code."""
try:
parser = Parser(self.code, self._encoding)
parser.parse()
self.attr_docs = {}
for (scope, comment) in parser.comments.items():
if comment:
self.attr_docs[scope] = comment.splitlines() + ['']
else:
self.attr_docs[scope] = ['']
self.annotations = parser.annotations
self.finals = parser.finals
self.tags = parser.definitions
self.tagorder = parser.deforders
except Exception as exc:
raise PycodeError('parsing %r failed: %r' % (self.srcname, exc))
def find_attr_docs(self) -> Dict[Tuple[str, str], List[str]]:
"""Find class and module-level attributes and their documentation."""
if self.attr_docs is None:
self.parse()
return self.attr_docs
def find_tags(self) -> Dict[str, Tuple[str, int, int]]:
"""Find class, function and method definitions and their location."""
if self.tags is None:
self.parse()
return self.tags
@property
def encoding(self) -> str:
warnings.warn('ModuleAnalyzer.encoding is deprecated.',
RemovedInSphinx40Warning)
return self._encoding
|
