diff options
Diffstat (limited to 'Cython')
248 files changed, 25683 insertions, 8917 deletions
diff --git a/Cython/Build/BuildExecutable.py b/Cython/Build/BuildExecutable.py index 2db9e5d74..334fbf069 100644 --- a/Cython/Build/BuildExecutable.py +++ b/Cython/Build/BuildExecutable.py @@ -1,10 +1,10 @@ """ -Compile a Python script into an executable that embeds CPython and run it. +Compile a Python script into an executable that embeds CPython. Requires CPython to be built as a shared library ('libpythonX.Y'). Basic usage: - python cythonrun somefile.py [ARGS] + python -m Cython.Build.BuildExecutable [ARGS] somefile.py """ from __future__ import absolute_import @@ -28,7 +28,7 @@ if PYLIB_DYN == PYLIB: # no shared library PYLIB_DYN = '' else: - PYLIB_DYN = os.path.splitext(PYLIB_DYN[3:])[0] # 'lib(XYZ).so' -> XYZ + PYLIB_DYN = os.path.splitext(PYLIB_DYN[3:])[0] # 'lib(XYZ).so' -> XYZ CC = get_config_var('CC', os.environ.get('CC', '')) CFLAGS = get_config_var('CFLAGS') + ' ' + os.environ.get('CFLAGS', '') @@ -38,12 +38,14 @@ LIBS = get_config_var('LIBS') SYSLIBS = get_config_var('SYSLIBS') EXE_EXT = sysconfig.get_config_var('EXE') + def _debug(msg, *args): if DEBUG: if args: msg = msg % args sys.stderr.write(msg + '\n') + def dump_config(): _debug('INCDIR: %s', INCDIR) _debug('LIBDIR1: %s', LIBDIR1) @@ -58,6 +60,26 @@ def dump_config(): _debug('SYSLIBS: %s', SYSLIBS) _debug('EXE_EXT: %s', EXE_EXT) + +def _parse_args(args): + cy_args = [] + last_arg = None + for i, arg in enumerate(args): + if arg.startswith('-'): + cy_args.append(arg) + elif last_arg in ('-X', '--directive'): + cy_args.append(arg) + else: + input_file = arg + args = args[i+1:] + break + last_arg = arg + else: + raise ValueError('no input file provided') + + return input_file, cy_args, args + + def runcmd(cmd, shell=True): if shell: cmd = ' '.join(cmd) @@ -65,24 +87,23 @@ def runcmd(cmd, shell=True): else: _debug(' '.join(cmd)) - try: - import subprocess - except ImportError: # Python 2.3 ... - returncode = os.system(cmd) - else: - returncode = subprocess.call(cmd, shell=shell) + import subprocess + returncode = subprocess.call(cmd, shell=shell) if returncode: sys.exit(returncode) + def clink(basename): runcmd([LINKCC, '-o', basename + EXE_EXT, basename+'.o', '-L'+LIBDIR1, '-L'+LIBDIR2] + [PYLIB_DYN and ('-l'+PYLIB_DYN) or os.path.join(LIBDIR1, PYLIB)] + LIBS.split() + SYSLIBS.split() + LINKFORSHARED.split()) + def ccompile(basename): runcmd([CC, '-c', '-o', basename+'.o', basename+'.c', '-I' + INCDIR] + CFLAGS.split()) + def cycompile(input_file, options=()): from ..Compiler import Version, CmdLine, Main options, sources = CmdLine.parse_command_line(list(options or ()) + ['--embed', input_file]) @@ -91,9 +112,11 @@ def cycompile(input_file, options=()): if result.num_errors > 0: sys.exit(1) + def exec_file(program_name, args=()): runcmd([os.path.abspath(program_name)] + list(args), shell=False) + def build(input_file, compiler_args=(), force=False): """ Build an executable program from a Cython module. @@ -105,7 +128,7 @@ def build(input_file, compiler_args=(), force=False): if not force and os.path.abspath(exe_file) == os.path.abspath(input_file): raise ValueError("Input and output file names are the same, refusing to overwrite") if (not force and os.path.exists(exe_file) and os.path.exists(input_file) - and os.path.getmtime(input_file) <= os.path.getmtime(exe_file)): + and os.path.getmtime(input_file) <= os.path.getmtime(exe_file)): _debug("File is up to date, not regenerating %s", exe_file) return exe_file cycompile(input_file, compiler_args) @@ -113,30 +136,22 @@ def build(input_file, compiler_args=(), force=False): clink(basename) return exe_file + def build_and_run(args): """ - Build an executable program from a Cython module and runs it. + Build an executable program from a Cython module and run it. - Arguments after the module name will be passed verbatimely to the - program. + Arguments after the module name will be passed verbatimly to the program. """ - cy_args = [] - last_arg = None - for i, arg in enumerate(args): - if arg.startswith('-'): - cy_args.append(arg) - elif last_arg in ('-X', '--directive'): - cy_args.append(arg) - else: - input_file = arg - args = args[i+1:] - break - last_arg = arg - else: - raise ValueError('no input file provided') + program_name, args = _build(args) + exec_file(program_name, args) + +def _build(args): + input_file, cy_args, args = _parse_args(args) program_name = build(input_file, cy_args) - exec_file(program_name, args) + return program_name, args + if __name__ == '__main__': - build_and_run(sys.argv[1:]) + _build(sys.argv[1:]) diff --git a/Cython/Build/Cythonize.py b/Cython/Build/Cythonize.py index c85b6eaba..179c04060 100644 --- a/Cython/Build/Cythonize.py +++ b/Cython/Build/Cythonize.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -from __future__ import absolute_import +from __future__ import absolute_import, print_function import os import shutil @@ -38,46 +38,19 @@ class _FakePool(object): pass -def parse_directives(option, name, value, parser): - dest = option.dest - old_directives = dict(getattr(parser.values, dest, - Options.get_directive_defaults())) - directives = Options.parse_directive_list( - value, relaxed_bool=True, current_settings=old_directives) - setattr(parser.values, dest, directives) - - -def parse_options(option, name, value, parser): - dest = option.dest - options = dict(getattr(parser.values, dest, {})) - for opt in value.split(','): - if '=' in opt: - n, v = opt.split('=', 1) - v = v.lower() not in ('false', 'f', '0', 'no') - else: - n, v = opt, True - options[n] = v - setattr(parser.values, dest, options) - - -def parse_compile_time_env(option, name, value, parser): - dest = option.dest - old_env = dict(getattr(parser.values, dest, {})) - new_env = Options.parse_compile_time_env(value, current_settings=old_env) - setattr(parser.values, dest, new_env) - - def find_package_base(path): base_dir, package_path = os.path.split(path) - while os.path.isfile(os.path.join(base_dir, '__init__.py')): + while is_package_dir(base_dir): base_dir, parent = os.path.split(base_dir) package_path = '%s/%s' % (parent, package_path) return base_dir, package_path - def cython_compile(path_pattern, options): - pool = None all_paths = map(os.path.abspath, extended_iglob(path_pattern)) + _cython_compile_files(all_paths, options) + +def _cython_compile_files(all_paths, options): + pool = None try: for path in all_paths: if options.build_inplace: @@ -149,55 +122,89 @@ def run_distutils(args): shutil.rmtree(temp_dir) -def parse_args(args): - from optparse import OptionParser - parser = OptionParser(usage='%prog [options] [sources and packages]+') +def create_args_parser(): + from argparse import ArgumentParser, RawDescriptionHelpFormatter + from ..Compiler.CmdLine import ParseDirectivesAction, ParseOptionsAction, ParseCompileTimeEnvAction - parser.add_option('-X', '--directive', metavar='NAME=VALUE,...', - dest='directives', default={}, type="str", - action='callback', callback=parse_directives, + parser = ArgumentParser( + formatter_class=RawDescriptionHelpFormatter, + epilog="""\ +Environment variables: + CYTHON_FORCE_REGEN: if set to 1, forces cythonize to regenerate the output files regardless + of modification times and changes. + Environment variables accepted by setuptools are supported to configure the C compiler and build: + https://setuptools.pypa.io/en/latest/userguide/ext_modules.html#compiler-and-linker-options""" + ) + + parser.add_argument('-X', '--directive', metavar='NAME=VALUE,...', + dest='directives', default={}, type=str, + action=ParseDirectivesAction, help='set a compiler directive') - parser.add_option('-E', '--compile-time-env', metavar='NAME=VALUE,...', - dest='compile_time_env', default={}, type="str", - action='callback', callback=parse_compile_time_env, + parser.add_argument('-E', '--compile-time-env', metavar='NAME=VALUE,...', + dest='compile_time_env', default={}, type=str, + action=ParseCompileTimeEnvAction, help='set a compile time environment variable') - parser.add_option('-s', '--option', metavar='NAME=VALUE', - dest='options', default={}, type="str", - action='callback', callback=parse_options, + parser.add_argument('-s', '--option', metavar='NAME=VALUE', + dest='options', default={}, type=str, + action=ParseOptionsAction, help='set a cythonize option') - parser.add_option('-2', dest='language_level', action='store_const', const=2, default=None, + parser.add_argument('-2', dest='language_level', action='store_const', const=2, default=None, help='use Python 2 syntax mode by default') - parser.add_option('-3', dest='language_level', action='store_const', const=3, + parser.add_argument('-3', dest='language_level', action='store_const', const=3, help='use Python 3 syntax mode by default') - parser.add_option('--3str', dest='language_level', action='store_const', const='3str', + parser.add_argument('--3str', dest='language_level', action='store_const', const='3str', help='use Python 3 syntax mode by default') - parser.add_option('-a', '--annotate', dest='annotate', action='store_true', - help='generate annotated HTML page for source files') - - parser.add_option('-x', '--exclude', metavar='PATTERN', dest='excludes', + parser.add_argument('-a', '--annotate', action='store_const', const='default', dest='annotate', + help='Produce a colorized HTML version of the source.') + parser.add_argument('--annotate-fullc', action='store_const', const='fullc', dest='annotate', + help='Produce a colorized HTML version of the source ' + 'which includes entire generated C/C++-code.') + parser.add_argument('-x', '--exclude', metavar='PATTERN', dest='excludes', action='append', default=[], help='exclude certain file patterns from the compilation') - parser.add_option('-b', '--build', dest='build', action='store_true', + parser.add_argument('-b', '--build', dest='build', action='store_true', default=None, help='build extension modules using distutils') - parser.add_option('-i', '--inplace', dest='build_inplace', action='store_true', + parser.add_argument('-i', '--inplace', dest='build_inplace', action='store_true', default=None, help='build extension modules in place using distutils (implies -b)') - parser.add_option('-j', '--parallel', dest='parallel', metavar='N', + parser.add_argument('-j', '--parallel', dest='parallel', metavar='N', type=int, default=parallel_compiles, help=('run builds in N parallel jobs (default: %d)' % parallel_compiles or 1)) - parser.add_option('-f', '--force', dest='force', action='store_true', + parser.add_argument('-f', '--force', dest='force', action='store_true', default=None, help='force recompilation') - parser.add_option('-q', '--quiet', dest='quiet', action='store_true', + parser.add_argument('-q', '--quiet', dest='quiet', action='store_true', default=None, help='be less verbose during compilation') - parser.add_option('--lenient', dest='lenient', action='store_true', + parser.add_argument('--lenient', dest='lenient', action='store_true', default=None, help='increase Python compatibility by ignoring some compile time errors') - parser.add_option('-k', '--keep-going', dest='keep_going', action='store_true', + parser.add_argument('-k', '--keep-going', dest='keep_going', action='store_true', default=None, help='compile as much as possible, ignore compilation failures') - parser.add_option('-M', '--depfile', action='store_true', help='produce depfiles for the sources') + parser.add_argument('--no-docstrings', dest='no_docstrings', action='store_true', default=None, + help='strip docstrings') + parser.add_argument('-M', '--depfile', action='store_true', help='produce depfiles for the sources') + parser.add_argument('sources', nargs='*') + return parser + + +def parse_args_raw(parser, args): + options, unknown = parser.parse_known_args(args) + sources = options.sources + # if positional arguments were interspersed + # some of them are in unknown + for option in unknown: + if option.startswith('-'): + parser.error("unknown option "+option) + else: + sources.append(option) + del options.sources + return (options, sources) + + +def parse_args(args): + parser = create_args_parser() + options, args = parse_args_raw(parser, args) - options, args = parser.parse_args(args) if not args: parser.error("no source files provided") if options.build_inplace: @@ -207,11 +214,6 @@ def parse_args(args): if options.language_level: assert options.language_level in (2, 3, '3str') options.options['language_level'] = options.language_level - return options, args - - -def main(args=None): - options, paths = parse_args(args) if options.lenient: # increase Python compatibility by ignoring compile time errors @@ -219,10 +221,26 @@ def main(args=None): Options.error_on_uninitialized = False if options.annotate: - Options.annotate = True + Options.annotate = options.annotate + + if options.no_docstrings: + Options.docstrings = False + + return options, args + + +def main(args=None): + options, paths = parse_args(args) + all_paths = [] for path in paths: - cython_compile(path, options) + expanded_path = [os.path.abspath(p) for p in extended_iglob(path)] + if not expanded_path: + import sys + print("{}: No such file or directory: '{}'".format(sys.argv[0], path), file=sys.stderr) + sys.exit(1) + all_paths.extend(expanded_path) + _cython_compile_files(all_paths, options) if __name__ == '__main__': diff --git a/Cython/Build/Dependencies.py b/Cython/Build/Dependencies.py index 28c48ed8c..c60cbf34a 100644 --- a/Cython/Build/Dependencies.py +++ b/Cython/Build/Dependencies.py @@ -10,7 +10,6 @@ import os import shutil import subprocess import re, sys, time -import warnings from glob import iglob from io import open as io_open from os.path import relpath as _relpath @@ -43,9 +42,12 @@ except: pythran = None from .. import Utils -from ..Utils import (cached_function, cached_method, path_exists, write_depfile, - safe_makedirs, copy_file_to_dir_if_newer, is_package_dir, replace_suffix) -from ..Compiler.Main import Context, CompilationOptions, default_options +from ..Utils import (cached_function, cached_method, path_exists, + safe_makedirs, copy_file_to_dir_if_newer, is_package_dir, write_depfile) +from ..Compiler import Errors +from ..Compiler.Main import Context +from ..Compiler.Options import (CompilationOptions, default_options, + get_directive_defaults) join_path = cached_function(os.path.join) copy_once_if_newer = cached_function(copy_file_to_dir_if_newer) @@ -84,11 +86,14 @@ def extended_iglob(pattern): for path in extended_iglob(before + case + after): yield path return - if '**/' in pattern: + + # We always accept '/' and also '\' on Windows, + # because '/' is generally common for relative paths. + if '**/' in pattern or os.sep == '\\' and '**\\' in pattern: seen = set() - first, rest = pattern.split('**/', 1) + first, rest = re.split(r'\*\*[%s]' % ('/\\\\' if os.sep == '\\' else '/'), pattern, 1) if first: - first = iglob(first+'/') + first = iglob(first + os.sep) else: first = [''] for root in first: @@ -96,7 +101,7 @@ def extended_iglob(pattern): if path not in seen: seen.add(path) yield path - for path in extended_iglob(join_path(root, '*', '**/' + rest)): + for path in extended_iglob(join_path(root, '*', '**', rest)): if path not in seen: seen.add(path) yield path @@ -118,7 +123,7 @@ def nonempty(it, error_msg="expected non-empty iterator"): def file_hash(filename): path = os.path.normpath(filename) prefix = ('%d:%s' % (len(path), path)).encode("UTF-8") - m = hashlib.md5(prefix) + m = hashlib.sha1(prefix) with open(path, 'rb') as f: data = f.read(65000) while data: @@ -532,14 +537,14 @@ class DependencyTree(object): for include in self.parse_dependencies(filename)[1]: include_path = join_path(os.path.dirname(filename), include) if not path_exists(include_path): - include_path = self.context.find_include_file(include, None) + include_path = self.context.find_include_file(include, source_file_path=filename) if include_path: if '.' + os.path.sep in include_path: include_path = os.path.normpath(include_path) all.add(include_path) all.update(self.included_files(include_path)) elif not self.quiet: - print("Unable to locate '%s' referenced from '%s'" % (filename, include)) + print(u"Unable to locate '%s' referenced from '%s'" % (filename, include)) return all @cached_method @@ -586,17 +591,18 @@ class DependencyTree(object): return None # FIXME: error? module_path.pop(0) relative = '.'.join(package_path + module_path) - pxd = self.context.find_pxd_file(relative, None) + pxd = self.context.find_pxd_file(relative, source_file_path=filename) if pxd: return pxd if is_relative: return None # FIXME: error? - return self.context.find_pxd_file(module, None) + return self.context.find_pxd_file(module, source_file_path=filename) @cached_method def cimported_files(self, filename): - if filename[-4:] == '.pyx' and path_exists(filename[:-4] + '.pxd'): - pxd_list = [filename[:-4] + '.pxd'] + filename_root, filename_ext = os.path.splitext(filename) + if filename_ext in ('.pyx', '.py') and path_exists(filename_root + '.pxd'): + pxd_list = [filename_root + '.pxd'] else: pxd_list = [] # Cimports generates all possible combinations package.module @@ -611,10 +617,10 @@ class DependencyTree(object): @cached_method def immediate_dependencies(self, filename): - all = set([filename]) - all.update(self.cimported_files(filename)) - all.update(self.included_files(filename)) - return all + all_deps = {filename} + all_deps.update(self.cimported_files(filename)) + all_deps.update(self.included_files(filename)) + return all_deps def all_dependencies(self, filename): return self.transitive_merge(filename, self.immediate_dependencies, set.union) @@ -638,7 +644,7 @@ class DependencyTree(object): incorporate everything that has an influence on the generated code. """ try: - m = hashlib.md5(__version__.encode('UTF-8')) + m = hashlib.sha1(__version__.encode('UTF-8')) m.update(file_hash(filename).encode('UTF-8')) for x in sorted(self.all_dependencies(filename)): if os.path.splitext(x)[1] not in ('.c', '.cpp', '.h'): @@ -726,7 +732,8 @@ def create_dependency_tree(ctx=None, quiet=False): global _dep_tree if _dep_tree is None: if ctx is None: - ctx = Context(["."], CompilationOptions(default_options)) + ctx = Context(["."], get_directive_defaults(), + options=CompilationOptions(default_options)) _dep_tree = DependencyTree(ctx, quiet=quiet) return _dep_tree @@ -757,7 +764,7 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet= return [], {} elif isinstance(patterns, basestring) or not isinstance(patterns, Iterable): patterns = [patterns] - explicit_modules = set([m.name for m in patterns if isinstance(m, Extension)]) + explicit_modules = {m.name for m in patterns if isinstance(m, Extension)} seen = set() deps = create_dependency_tree(ctx, quiet=quiet) to_exclude = set() @@ -783,6 +790,8 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet= create_extension = ctx.options.create_extension or default_create_extension for pattern in patterns: + if not isinstance(pattern, (Extension_distutils, Extension_setuptools)): + pattern = encode_filename_in_py2(pattern) if isinstance(pattern, str): filepattern = pattern template = Extension(pattern, []) # Fake Extension without sources @@ -795,9 +804,9 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet= if cython_sources: filepattern = cython_sources[0] if len(cython_sources) > 1: - print("Warning: Multiple cython sources found for extension '%s': %s\n" - "See http://cython.readthedocs.io/en/latest/src/userguide/sharing_declarations.html " - "for sharing declarations among Cython files." % (pattern.name, cython_sources)) + print(u"Warning: Multiple cython sources found for extension '%s': %s\n" + u"See https://cython.readthedocs.io/en/latest/src/userguide/sharing_declarations.html " + u"for sharing declarations among Cython files." % (pattern.name, cython_sources)) else: # ignore non-cython modules module_list.append(pattern) @@ -871,15 +880,15 @@ def create_extension_list(patterns, exclude=None, ctx=None, aliases=None, quiet= m.sources.remove(target_file) except ValueError: # never seen this in the wild, but probably better to warn about this unexpected case - print("Warning: Cython source file not found in sources list, adding %s" % file) + print(u"Warning: Cython source file not found in sources list, adding %s" % file) m.sources.insert(0, file) seen.add(name) return module_list, module_metadata # This is the user-exposed entry point. -def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, force=False, language=None, - exclude_failures=False, **options): +def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, force=None, language=None, + exclude_failures=False, show_all_warnings=False, **options): """ Compile a set of source modules into C/C++ files and return a list of distutils Extension objects for them. @@ -929,6 +938,9 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, really makes sense for compiling ``.py`` files which can also be used without compilation. + :param show_all_warnings: By default, not all Cython warnings are printed. + Set to true to show all warnings. + :param annotate: If ``True``, will produce a HTML file for each of the ``.pyx`` or ``.py`` files compiled. The HTML file gives an indication of how much Python interaction there is in @@ -941,6 +953,11 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, See examples in :ref:`determining_where_to_add_types` or :ref:`primes`. + + :param annotate-fullc: If ``True`` will produce a colorized HTML version of + the source which includes entire generated C/C++-code. + + :param compiler_directives: Allow to set compiler directives in the ``setup.py`` like this: ``compiler_directives={'embedsignature': True}``. See :ref:`compiler-directives`. @@ -968,7 +985,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, c_options = CompilationOptions(**options) cpp_options = CompilationOptions(**options); cpp_options.cplus = True - ctx = c_options.create_context() + ctx = Context.from_options(c_options) options = c_options module_list, module_metadata = create_extension_list( module_list, @@ -978,6 +995,9 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, exclude_failures=exclude_failures, language=language, aliases=aliases) + + fix_windows_unicode_modules(module_list) + deps = create_dependency_tree(ctx, quiet=quiet) build_dir = getattr(options, 'build_dir', None) @@ -1025,7 +1045,8 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, # setup for out of place build directory if enabled if build_dir: if os.path.isabs(c_file): - warnings.warn("build_dir has no effect for absolute source paths") + c_file = os.path.splitdrive(c_file)[1] + c_file = c_file.split(os.sep, 1)[1] c_file = os.path.join(build_dir, c_file) dir = os.path.dirname(c_file) safe_makedirs_once(dir) @@ -1035,7 +1056,8 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, dependencies = deps.all_dependencies(source) write_depfile(c_file, source, dependencies) - if os.path.exists(c_file): + # Missing files and those generated by other Cython versions should always be recreated. + if Utils.file_generated_by_this_cython(c_file): c_timestamp = os.path.getmtime(c_file) else: c_timestamp = -1 @@ -1051,9 +1073,12 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, if force or c_timestamp < dep_timestamp: if not quiet and not force: if source == dep: - print("Compiling %s because it changed." % source) + print(u"Compiling %s because it changed." % Utils.decode_filename(source)) else: - print("Compiling %s because it depends on %s." % (source, dep)) + print(u"Compiling %s because it depends on %s." % ( + Utils.decode_filename(source), + Utils.decode_filename(dep), + )) if not force and options.cache: fingerprint = deps.transitive_fingerprint(source, m, options) else: @@ -1061,7 +1086,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, to_compile.append(( priority, source, c_file, fingerprint, quiet, options, not exclude_failures, module_metadata.get(m.name), - full_module_name)) + full_module_name, show_all_warnings)) new_sources.append(c_file) modules_by_cfile[c_file].append(m) else: @@ -1085,32 +1110,26 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, if N <= 1: nthreads = 0 if nthreads: - # Requires multiprocessing (or Python >= 2.6) + import multiprocessing + pool = multiprocessing.Pool( + nthreads, initializer=_init_multiprocessing_helper) + # This is a bit more involved than it should be, because KeyboardInterrupts + # break the multiprocessing workers when using a normal pool.map(). + # See, for example: + # https://noswap.com/blog/python-multiprocessing-keyboardinterrupt try: - import multiprocessing - pool = multiprocessing.Pool( - nthreads, initializer=_init_multiprocessing_helper) - except (ImportError, OSError): - print("multiprocessing required for parallel cythonization") - nthreads = 0 - else: - # This is a bit more involved than it should be, because KeyboardInterrupts - # break the multiprocessing workers when using a normal pool.map(). - # See, for example: - # http://noswap.com/blog/python-multiprocessing-keyboardinterrupt - try: - result = pool.map_async(cythonize_one_helper, to_compile, chunksize=1) - pool.close() - while not result.ready(): - try: - result.get(99999) # seconds - except multiprocessing.TimeoutError: - pass - except KeyboardInterrupt: - pool.terminate() - raise - pool.join() - if not nthreads: + result = pool.map_async(cythonize_one_helper, to_compile, chunksize=1) + pool.close() + while not result.ready(): + try: + result.get(99999) # seconds + except multiprocessing.TimeoutError: + pass + except KeyboardInterrupt: + pool.terminate() + raise + pool.join() + else: for args in to_compile: cythonize_one(*args) @@ -1130,7 +1149,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, if failed_modules: for module in failed_modules: module_list.remove(module) - print("Failed compilations: %s" % ', '.join(sorted([ + print(u"Failed compilations: %s" % ', '.join(sorted([ module.name for module in failed_modules]))) if options.cache: @@ -1141,6 +1160,41 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, return module_list +def fix_windows_unicode_modules(module_list): + # Hack around a distutils 3.[5678] bug on Windows for unicode module names. + # https://bugs.python.org/issue39432 + if sys.platform != "win32": + return + if sys.version_info < (3, 5) or sys.version_info >= (3, 8, 2): + return + + def make_filtered_list(ignored_symbol, old_entries): + class FilteredExportSymbols(list): + # export_symbols for unicode filename cause link errors on Windows + # Cython doesn't need them (it already defines PyInit with the correct linkage) + # so use this class as a temporary fix to stop them from being generated + def __contains__(self, val): + # so distutils doesn't "helpfully" add PyInit_<name> + return val == ignored_symbol or list.__contains__(self, val) + + filtered_list = FilteredExportSymbols(old_entries) + if old_entries: + filtered_list.extend(name for name in old_entries if name != ignored_symbol) + return filtered_list + + for m in module_list: + # TODO: use m.name.isascii() in Py3.7+ + try: + m.name.encode("ascii") + continue + except UnicodeEncodeError: + pass + m.export_symbols = make_filtered_list( + "PyInit_" + m.name.rsplit(".", 1)[-1], + m.export_symbols, + ) + + if os.environ.get('XML_RESULTS'): compile_result_dir = os.environ['XML_RESULTS'] def record_results(func): @@ -1180,7 +1234,8 @@ else: # TODO: Share context? Issue: pyx processing leaks into pxd module @record_results def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, - raise_on_failure=True, embedded_metadata=None, full_module_name=None, + raise_on_failure=True, embedded_metadata=None, + full_module_name=None, show_all_warnings=False, progress=""): from ..Compiler.Main import compile_single, default_options from ..Compiler.Errors import CompileError, PyrexError @@ -1196,7 +1251,7 @@ def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, zip_fingerprint_file = fingerprint_file_base + '.zip' if os.path.exists(gz_fingerprint_file) or os.path.exists(zip_fingerprint_file): if not quiet: - print("%sFound compiled %s in cache" % (progress, pyx_file)) + print(u"%sFound compiled %s in cache" % (progress, pyx_file)) if os.path.exists(gz_fingerprint_file): os.utime(gz_fingerprint_file, None) with contextlib.closing(gzip_open(gz_fingerprint_file, 'rb')) as g: @@ -1210,12 +1265,16 @@ def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, z.extract(artifact, os.path.join(dirname, artifact)) return if not quiet: - print("%sCythonizing %s" % (progress, pyx_file)) + print(u"%sCythonizing %s" % (progress, Utils.decode_filename(pyx_file))) if options is None: options = CompilationOptions(default_options) options.output_file = c_file options.embedded_metadata = embedded_metadata + old_warning_level = Errors.LEVEL + if show_all_warnings: + Errors.LEVEL = 0 + any_failures = 0 try: result = compile_single(pyx_file, options, full_module_name=full_module_name) @@ -1233,6 +1292,10 @@ def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, import traceback traceback.print_exc() any_failures = 1 + finally: + if show_all_warnings: + Errors.LEVEL = old_warning_level + if any_failures: if raise_on_failure: raise CompileError(None, pyx_file) @@ -1250,7 +1313,7 @@ def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, else: fingerprint_file = zip_fingerprint_file with contextlib.closing(zipfile.ZipFile( - fingerprint_file + '.tmp', 'w', zipfile_compression_mode)) as zip: + fingerprint_file + '.tmp', 'w', zipfile_compression_mode)) as zip: for artifact in artifacts: zip.write(artifact, os.path.basename(artifact)) os.rename(fingerprint_file + '.tmp', fingerprint_file) @@ -1274,9 +1337,10 @@ def _init_multiprocessing_helper(): def cleanup_cache(cache, target_size, ratio=.85): try: p = subprocess.Popen(['du', '-s', '-k', os.path.abspath(cache)], stdout=subprocess.PIPE) + stdout, _ = p.communicate() res = p.wait() if res == 0: - total_size = 1024 * int(p.stdout.read().strip().split()[0]) + total_size = 1024 * int(stdout.strip().split()[0]) if total_size < target_size: return except (OSError, ValueError): diff --git a/Cython/Build/Inline.py b/Cython/Build/Inline.py index 69684e03f..abb891265 100644 --- a/Cython/Build/Inline.py +++ b/Cython/Build/Inline.py @@ -10,7 +10,9 @@ from distutils.core import Distribution, Extension from distutils.command.build_ext import build_ext import Cython -from ..Compiler.Main import Context, default_options +from ..Compiler.Main import Context +from ..Compiler.Options import (default_options, CompilationOptions, + get_directive_defaults) from ..Compiler.Visitor import CythonTransform, EnvTransform from ..Compiler.ParseTreeTransforms import SkipDeclarations @@ -34,6 +36,7 @@ if not IS_PY3: else: to_unicode = lambda x: x + if sys.version_info < (3, 5): import imp def load_dynamic(name, module_path): @@ -48,9 +51,10 @@ else: spec.loader.exec_module(module) return module + class UnboundSymbols(EnvTransform, SkipDeclarations): def __init__(self): - CythonTransform.__init__(self, None) + super(EnvTransform, self).__init__(context=None) self.unbound = set() def visit_NameNode(self, node): if not self.current_env().lookup(node.name): @@ -65,7 +69,8 @@ class UnboundSymbols(EnvTransform, SkipDeclarations): def unbound_symbols(code, context=None): code = to_unicode(code) if context is None: - context = Context([], default_options) + context = Context([], get_directive_defaults(), + options=CompilationOptions(default_options)) from ..Compiler.ParseTreeTransforms import AnalyseDeclarationsTransform tree = parse_from_strings('(tree fragment)', code) for phase in Pipeline.create_pipeline(context, 'pyx'): @@ -126,7 +131,11 @@ def _get_build_extension(): @cached_function def _create_context(cython_include_dirs): - return Context(list(cython_include_dirs), default_options) + return Context( + list(cython_include_dirs), + get_directive_defaults(), + options=CompilationOptions(default_options) + ) _cython_inline_cache = {} @@ -170,6 +179,8 @@ def cython_inline(code, get_type=unsafe_type, if language_level is not None: cython_compiler_directives['language_level'] = language_level + key_hash = None + # Fast path if this has been called in this session. _unbound_symbols = _cython_inline_cache.get(code) if _unbound_symbols is not None: @@ -205,7 +216,8 @@ def cython_inline(code, get_type=unsafe_type, del kwds[name] arg_names = sorted(kwds) arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names]) - key_hash = _inline_key(orig_code, arg_sigs, language_level) + if key_hash is None: + key_hash = _inline_key(orig_code, arg_sigs, language_level) module_name = "_cython_inline_" + key_hash if module_name in sys.modules: @@ -224,6 +236,7 @@ def cython_inline(code, get_type=unsafe_type, os.makedirs(lib_dir) if force or not os.path.isfile(module_path): cflags = [] + define_macros = [] c_include_dirs = [] qualified = re.compile(r'([.\w]+)[.]') for type, _ in arg_sigs: @@ -234,6 +247,7 @@ def cython_inline(code, get_type=unsafe_type, if m.groups()[0] == 'numpy': import numpy c_include_dirs.append(numpy.get_include()) + define_macros.append(("NPY_NO_DEPRECATED_API", "NPY_1_7_API_VERSION")) # cflags.append('-Wno-unused') module_body, func_body = extract_func_code(code) params = ', '.join(['%s %s' % a for a in arg_sigs]) @@ -256,10 +270,12 @@ def __invoke(%(params)s): finally: fh.close() extension = Extension( - name = module_name, - sources = [pyx_file], - include_dirs = c_include_dirs, - extra_compile_args = cflags) + name=module_name, + sources=[pyx_file], + include_dirs=c_include_dirs or None, + extra_compile_args=cflags or None, + define_macros=define_macros or None, + ) if build_extension is None: build_extension = _get_build_extension() build_extension.extensions = cythonize( @@ -322,37 +338,6 @@ def extract_func_code(code): return '\n'.join(module), ' ' + '\n '.join(function) -try: - from inspect import getcallargs -except ImportError: - def getcallargs(func, *arg_values, **kwd_values): - all = {} - args, varargs, kwds, defaults = inspect.getargspec(func) - if varargs is not None: - all[varargs] = arg_values[len(args):] - for name, value in zip(args, arg_values): - all[name] = value - for name, value in list(kwd_values.items()): - if name in args: - if name in all: - raise TypeError("Duplicate argument %s" % name) - all[name] = kwd_values.pop(name) - if kwds is not None: - all[kwds] = kwd_values - elif kwd_values: - raise TypeError("Unexpected keyword arguments: %s" % list(kwd_values)) - if defaults is None: - defaults = () - first_default = len(args) - len(defaults) - for ix, name in enumerate(args): - if name not in all: - if ix >= first_default: - all[name] = defaults[ix - first_default] - else: - raise TypeError("Missing argument: %s" % name) - return all - - def get_body(source): ix = source.index(':') if source[:5] == 'lambda': @@ -370,7 +355,7 @@ class RuntimeCompiledFunction(object): self._body = get_body(inspect.getsource(f)) def __call__(self, *args, **kwds): - all = getcallargs(self._f, *args, **kwds) + all = inspect.getcallargs(self._f, *args, **kwds) if IS_PY3: return cython_inline(self._body, locals=self._f.__globals__, globals=self._f.__globals__, **all) else: diff --git a/Cython/Build/IpythonMagic.py b/Cython/Build/IpythonMagic.py index b3a9d7c58..3fa43c96d 100644 --- a/Cython/Build/IpythonMagic.py +++ b/Cython/Build/IpythonMagic.py @@ -58,16 +58,7 @@ import textwrap IO_ENCODING = sys.getfilesystemencoding() IS_PY2 = sys.version_info[0] < 3 -try: - reload -except NameError: # Python 3 - from imp import reload - -try: - import hashlib -except ImportError: - import md5 as hashlib - +import hashlib from distutils.core import Distribution, Extension from distutils.command.build_ext import build_ext @@ -85,6 +76,7 @@ from ..Shadow import __version__ as cython_version from ..Compiler.Errors import CompileError from .Inline import cython_inline, load_dynamic from .Dependencies import cythonize +from ..Utils import captured_fd, print_captured PGO_CONFIG = { @@ -191,10 +183,15 @@ class CythonMagics(Magics): @magic_arguments.magic_arguments() @magic_arguments.argument( - '-a', '--annotate', action='store_true', default=False, + '-a', '--annotate', action='store_const', const='default', dest='annotate', help="Produce a colorized HTML version of the source." ) @magic_arguments.argument( + '--annotate-fullc', action='store_const', const='fullc', dest='annotate', + help="Produce a colorized HTML version of the source " + "which includes entire generated C/C++-code." + ) + @magic_arguments.argument( '-+', '--cplus', action='store_true', default=False, help="Output a C++ rather than C file." ) @@ -316,7 +313,7 @@ class CythonMagics(Magics): if args.name: module_name = str(args.name) # no-op in Py3 else: - module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest() + module_name = "_cython_magic_" + hashlib.sha1(str(key).encode('utf-8')).hexdigest() html_file = os.path.join(lib_dir, module_name + '.html') module_path = os.path.join(lib_dir, module_name + self.so_ext) @@ -340,13 +337,25 @@ class CythonMagics(Magics): if args.pgo: self._profile_pgo_wrapper(extension, lib_dir) + def print_compiler_output(stdout, stderr, where): + # On windows, errors are printed to stdout, we redirect both to sys.stderr. + print_captured(stdout, where, u"Content of stdout:\n") + print_captured(stderr, where, u"Content of stderr:\n") + + get_stderr = get_stdout = None try: - self._build_extension(extension, lib_dir, pgo_step_name='use' if args.pgo else None, - quiet=args.quiet) - except distutils.errors.CompileError: - # Build failed and printed error message + with captured_fd(1) as get_stdout: + with captured_fd(2) as get_stderr: + self._build_extension( + extension, lib_dir, pgo_step_name='use' if args.pgo else None, quiet=args.quiet) + except (distutils.errors.CompileError, distutils.errors.LinkError): + # Build failed, print error message from compiler/linker + print_compiler_output(get_stdout(), get_stderr(), sys.stderr) return None + # Build seems ok, but we might still want to show any warnings that occurred + print_compiler_output(get_stdout(), get_stderr(), sys.stdout) + module = load_dynamic(module_name, module_path) self._import_all(module) @@ -438,12 +447,11 @@ class CythonMagics(Magics): quiet=quiet, annotate=args.annotate, force=True, + language_level=min(3, sys.version_info[0]), ) if args.language_level is not None: assert args.language_level in (2, 3) opts['language_level'] = args.language_level - elif sys.version_info[0] >= 3: - opts['language_level'] = 3 return cythonize([extension], **opts) except CompileError: return None diff --git a/Cython/Build/Tests/TestCyCache.py b/Cython/Build/Tests/TestCyCache.py index a3224b417..7a44d89e9 100644 --- a/Cython/Build/Tests/TestCyCache.py +++ b/Cython/Build/Tests/TestCyCache.py @@ -33,25 +33,31 @@ class TestCyCache(CythonTest): a_pyx = os.path.join(self.src_dir, 'a.pyx') a_c = a_pyx[:-4] + '.c' - open(a_pyx, 'w').write(content1) + with open(a_pyx, 'w') as f: + f.write(content1) self.fresh_cythonize(a_pyx, cache=self.cache_dir) self.fresh_cythonize(a_pyx, cache=self.cache_dir) self.assertEqual(1, len(self.cache_files('a.c*'))) - a_contents1 = open(a_c).read() + with open(a_c) as f: + a_contents1 = f.read() os.unlink(a_c) - open(a_pyx, 'w').write(content2) + with open(a_pyx, 'w') as f: + f.write(content2) self.fresh_cythonize(a_pyx, cache=self.cache_dir) - a_contents2 = open(a_c).read() + with open(a_c) as f: + a_contents2 = f.read() os.unlink(a_c) self.assertNotEqual(a_contents1, a_contents2, 'C file not changed!') self.assertEqual(2, len(self.cache_files('a.c*'))) - open(a_pyx, 'w').write(content1) + with open(a_pyx, 'w') as f: + f.write(content1) self.fresh_cythonize(a_pyx, cache=self.cache_dir) self.assertEqual(2, len(self.cache_files('a.c*'))) - a_contents = open(a_c).read() + with open(a_c) as f: + a_contents = f.read() self.assertEqual( a_contents, a_contents1, msg='\n'.join(list(difflib.unified_diff( @@ -60,13 +66,15 @@ class TestCyCache(CythonTest): def test_cycache_uses_cache(self): a_pyx = os.path.join(self.src_dir, 'a.pyx') a_c = a_pyx[:-4] + '.c' - open(a_pyx, 'w').write('pass') + with open(a_pyx, 'w') as f: + f.write('pass') self.fresh_cythonize(a_pyx, cache=self.cache_dir) a_cache = os.path.join(self.cache_dir, os.listdir(self.cache_dir)[0]) gzip.GzipFile(a_cache, 'wb').write('fake stuff'.encode('ascii')) os.unlink(a_c) self.fresh_cythonize(a_pyx, cache=self.cache_dir) - a_contents = open(a_c).read() + with open(a_c) as f: + a_contents = f.read() self.assertEqual(a_contents, 'fake stuff', 'Unexpected contents: %s...' % a_contents[:100]) @@ -75,7 +83,8 @@ class TestCyCache(CythonTest): a_c = a_pyx[:-4] + '.c' a_h = a_pyx[:-4] + '.h' a_api_h = a_pyx[:-4] + '_api.h' - open(a_pyx, 'w').write('cdef public api int foo(int x): return x\n') + with open(a_pyx, 'w') as f: + f.write('cdef public api int foo(int x): return x\n') self.fresh_cythonize(a_pyx, cache=self.cache_dir) expected = [a_c, a_h, a_api_h] for output in expected: @@ -89,7 +98,8 @@ class TestCyCache(CythonTest): hash_pyx = os.path.join(self.src_dir, 'options.pyx') hash_c = hash_pyx[:-len('.pyx')] + '.c' - open(hash_pyx, 'w').write('pass') + with open(hash_pyx, 'w') as f: + f.write('pass') self.fresh_cythonize(hash_pyx, cache=self.cache_dir, cplus=False) self.assertEqual(1, len(self.cache_files('options.c*'))) diff --git a/Cython/Build/Tests/TestCythonizeArgsParser.py b/Cython/Build/Tests/TestCythonizeArgsParser.py new file mode 100644 index 000000000..c5a682dd6 --- /dev/null +++ b/Cython/Build/Tests/TestCythonizeArgsParser.py @@ -0,0 +1,482 @@ +from Cython.Build.Cythonize import ( + create_args_parser, parse_args_raw, parse_args, + parallel_compiles +) + +from Cython.Compiler import Options +from Cython.Compiler.Tests.Utils import backup_Options, restore_Options, check_global_options + +from unittest import TestCase + +import sys +try: + from StringIO import StringIO +except ImportError: + from io import StringIO # doesn't accept 'str' in Py2 + + +class TestCythonizeArgsParser(TestCase): + + def setUp(self): + TestCase.setUp(self) + self.parse_args = lambda x, parser=create_args_parser() : parse_args_raw(parser, x) + + + def are_default(self, options, skip): + # empty containers + empty_containers = ['directives', 'compile_time_env', 'options', 'excludes'] + are_none = ['language_level', 'annotate', 'build', 'build_inplace', 'force', 'quiet', 'lenient', 'keep_going', 'no_docstrings'] + for opt_name in empty_containers: + if len(getattr(options, opt_name))!=0 and (opt_name not in skip): + self.assertEqual(opt_name,"", msg="For option "+opt_name) + return False + for opt_name in are_none: + if (getattr(options, opt_name) is not None) and (opt_name not in skip): + self.assertEqual(opt_name,"", msg="For option "+opt_name) + return False + if options.parallel!=parallel_compiles and ('parallel' not in skip): + return False + return True + + # testing directives: + def test_directive_short(self): + options, args = self.parse_args(['-X', 'cdivision=True']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], True) + + def test_directive_long(self): + options, args = self.parse_args(['--directive', 'cdivision=True']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], True) + + def test_directive_multiple(self): + options, args = self.parse_args(['-X', 'cdivision=True', '-X', 'c_string_type=bytes']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], True) + self.assertEqual(options.directives['c_string_type'], 'bytes') + + def test_directive_multiple_v2(self): + options, args = self.parse_args(['-X', 'cdivision=True,c_string_type=bytes']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], True) + self.assertEqual(options.directives['c_string_type'], 'bytes') + + def test_directive_value_yes(self): + options, args = self.parse_args(['-X', 'cdivision=YeS']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], True) + + def test_directive_value_no(self): + options, args = self.parse_args(['-X', 'cdivision=no']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives'])) + self.assertEqual(options.directives['cdivision'], False) + + def test_directive_value_invalid(self): + with self.assertRaises(ValueError) as context: + options, args = self.parse_args(['-X', 'cdivision=sadfasd']) + + def test_directive_key_invalid(self): + with self.assertRaises(ValueError) as context: + options, args = self.parse_args(['-X', 'abracadabra']) + + def test_directive_no_value(self): + with self.assertRaises(ValueError) as context: + options, args = self.parse_args(['-X', 'cdivision']) + + def test_directives_types(self): + directives = { + 'auto_pickle': True, + 'c_string_type': 'bytearray', + 'c_string_type': 'bytes', + 'c_string_type': 'str', + 'c_string_type': 'bytearray', + 'c_string_type': 'unicode', + 'c_string_encoding' : 'ascii', + 'language_level' : 2, + 'language_level' : 3, + 'language_level' : '3str', + 'set_initial_path' : 'my_initial_path', + } + for key, value in directives.items(): + cmd = '{key}={value}'.format(key=key, value=str(value)) + options, args = self.parse_args(['-X', cmd]) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['directives']), msg = "Error for option: "+cmd) + self.assertEqual(options.directives[key], value, msg = "Error for option: "+cmd) + + def test_directives_wrong(self): + directives = { + 'auto_pickle': 42, # for bool type + 'auto_pickle': 'NONONO', # for bool type + 'c_string_type': 'bites', + #'c_string_encoding' : 'a', + #'language_level' : 4, + } + for key, value in directives.items(): + cmd = '{key}={value}'.format(key=key, value=str(value)) + with self.assertRaises(ValueError, msg = "Error for option: "+cmd) as context: + options, args = self.parse_args(['-X', cmd]) + + def test_compile_time_env_short(self): + options, args = self.parse_args(['-E', 'MYSIZE=10']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['compile_time_env'])) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + + def test_compile_time_env_long(self): + options, args = self.parse_args(['--compile-time-env', 'MYSIZE=10']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['compile_time_env'])) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + + def test_compile_time_env_multiple(self): + options, args = self.parse_args(['-E', 'MYSIZE=10', '-E', 'ARRSIZE=11']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['compile_time_env'])) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + self.assertEqual(options.compile_time_env['ARRSIZE'], 11) + + def test_compile_time_env_multiple_v2(self): + options, args = self.parse_args(['-E', 'MYSIZE=10,ARRSIZE=11']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['compile_time_env'])) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + self.assertEqual(options.compile_time_env['ARRSIZE'], 11) + + #testing options + def test_option_short(self): + options, args = self.parse_args(['-s', 'docstrings=True']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_long(self): + options, args = self.parse_args(['--option', 'docstrings=True']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_multiple(self): + options, args = self.parse_args(['-s', 'docstrings=True', '-s', 'buffer_max_dims=8']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + self.assertEqual(options.options['buffer_max_dims'], True) # really? + + def test_option_multiple_v2(self): + options, args = self.parse_args(['-s', 'docstrings=True,buffer_max_dims=8']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + self.assertEqual(options.options['buffer_max_dims'], True) # really? + + def test_option_value_yes(self): + options, args = self.parse_args(['-s', 'docstrings=YeS']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_value_4242(self): + options, args = self.parse_args(['-s', 'docstrings=4242']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_value_0(self): + options, args = self.parse_args(['-s', 'docstrings=0']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], False) + + def test_option_value_emptystr(self): + options, args = self.parse_args(['-s', 'docstrings=']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_value_a_str(self): + options, args = self.parse_args(['-s', 'docstrings=BB']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_value_no(self): + options, args = self.parse_args(['-s', 'docstrings=nO']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], False) + + def test_option_no_value(self): + options, args = self.parse_args(['-s', 'docstrings']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['docstrings'], True) + + def test_option_any_key(self): + options, args = self.parse_args(['-s', 'abracadabra']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['options'])) + self.assertEqual(options.options['abracadabra'], True) + + def test_language_level_2(self): + options, args = self.parse_args(['-2']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['language_level'])) + self.assertEqual(options.language_level, 2) + + def test_language_level_3(self): + options, args = self.parse_args(['-3']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['language_level'])) + self.assertEqual(options.language_level, 3) + + def test_language_level_3str(self): + options, args = self.parse_args(['--3str']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['language_level'])) + self.assertEqual(options.language_level, '3str') + + def test_annotate_short(self): + options, args = self.parse_args(['-a']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['annotate'])) + self.assertEqual(options.annotate, 'default') + + def test_annotate_long(self): + options, args = self.parse_args(['--annotate']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['annotate'])) + self.assertEqual(options.annotate, 'default') + + def test_annotate_fullc(self): + options, args = self.parse_args(['--annotate-fullc']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['annotate'])) + self.assertEqual(options.annotate, 'fullc') + + def test_annotate_and_positional(self): + options, args = self.parse_args(['-a', 'foo.pyx']) + self.assertEqual(args, ['foo.pyx']) + self.assertTrue(self.are_default(options, ['annotate'])) + self.assertEqual(options.annotate, 'default') + + def test_annotate_and_optional(self): + options, args = self.parse_args(['-a', '--3str']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['annotate', 'language_level'])) + self.assertEqual(options.annotate, 'default') + self.assertEqual(options.language_level, '3str') + + def test_exclude_short(self): + options, args = self.parse_args(['-x', '*.pyx']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['excludes'])) + self.assertTrue('*.pyx' in options.excludes) + + def test_exclude_long(self): + options, args = self.parse_args(['--exclude', '*.pyx']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['excludes'])) + self.assertTrue('*.pyx' in options.excludes) + + def test_exclude_multiple(self): + options, args = self.parse_args(['--exclude', '*.pyx', '--exclude', '*.py', ]) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['excludes'])) + self.assertEqual(options.excludes, ['*.pyx', '*.py']) + + def test_build_short(self): + options, args = self.parse_args(['-b']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['build'])) + self.assertEqual(options.build, True) + + def test_build_long(self): + options, args = self.parse_args(['--build']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['build'])) + self.assertEqual(options.build, True) + + def test_inplace_short(self): + options, args = self.parse_args(['-i']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['build_inplace'])) + self.assertEqual(options.build_inplace, True) + + def test_inplace_long(self): + options, args = self.parse_args(['--inplace']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['build_inplace'])) + self.assertEqual(options.build_inplace, True) + + def test_parallel_short(self): + options, args = self.parse_args(['-j', '42']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['parallel'])) + self.assertEqual(options.parallel, 42) + + def test_parallel_long(self): + options, args = self.parse_args(['--parallel', '42']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['parallel'])) + self.assertEqual(options.parallel, 42) + + def test_force_short(self): + options, args = self.parse_args(['-f']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['force'])) + self.assertEqual(options.force, True) + + def test_force_long(self): + options, args = self.parse_args(['--force']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['force'])) + self.assertEqual(options.force, True) + + def test_quite_short(self): + options, args = self.parse_args(['-q']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['quiet'])) + self.assertEqual(options.quiet, True) + + def test_quite_long(self): + options, args = self.parse_args(['--quiet']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['quiet'])) + self.assertEqual(options.quiet, True) + + def test_lenient_long(self): + options, args = self.parse_args(['--lenient']) + self.assertTrue(self.are_default(options, ['lenient'])) + self.assertFalse(args) + self.assertEqual(options.lenient, True) + + def test_keep_going_short(self): + options, args = self.parse_args(['-k']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['keep_going'])) + self.assertEqual(options.keep_going, True) + + def test_keep_going_long(self): + options, args = self.parse_args(['--keep-going']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['keep_going'])) + self.assertEqual(options.keep_going, True) + + def test_no_docstrings_long(self): + options, args = self.parse_args(['--no-docstrings']) + self.assertFalse(args) + self.assertTrue(self.are_default(options, ['no_docstrings'])) + self.assertEqual(options.no_docstrings, True) + + def test_file_name(self): + options, args = self.parse_args(['file1.pyx', 'file2.pyx']) + self.assertEqual(len(args), 2) + self.assertEqual(args[0], 'file1.pyx') + self.assertEqual(args[1], 'file2.pyx') + self.assertTrue(self.are_default(options, [])) + + def test_option_first(self): + options, args = self.parse_args(['-i', 'file.pyx']) + self.assertEqual(args, ['file.pyx']) + self.assertEqual(options.build_inplace, True) + self.assertTrue(self.are_default(options, ['build_inplace'])) + + def test_file_inbetween(self): + options, args = self.parse_args(['-i', 'file.pyx', '-a']) + self.assertEqual(args, ['file.pyx']) + self.assertEqual(options.build_inplace, True) + self.assertEqual(options.annotate, 'default') + self.assertTrue(self.are_default(options, ['build_inplace', 'annotate'])) + + def test_option_trailing(self): + options, args = self.parse_args(['file.pyx', '-i']) + self.assertEqual(args, ['file.pyx']) + self.assertEqual(options.build_inplace, True) + self.assertTrue(self.are_default(options, ['build_inplace'])) + + def test_interspersed_positional(self): + options, sources = self.parse_args([ + 'file1.pyx', '-a', + 'file2.pyx' + ]) + self.assertEqual(sources, ['file1.pyx', 'file2.pyx']) + self.assertEqual(options.annotate, 'default') + self.assertTrue(self.are_default(options, ['annotate'])) + + def test_interspersed_positional2(self): + options, sources = self.parse_args([ + 'file1.pyx', '-a', + 'file2.pyx', '-a', 'file3.pyx' + ]) + self.assertEqual(sources, ['file1.pyx', 'file2.pyx', 'file3.pyx']) + self.assertEqual(options.annotate, 'default') + self.assertTrue(self.are_default(options, ['annotate'])) + + def test_interspersed_positional3(self): + options, sources = self.parse_args([ + '-f', 'f1', 'f2', '-a', + 'f3', 'f4', '-a', 'f5' + ]) + self.assertEqual(sources, ['f1', 'f2', 'f3', 'f4', 'f5']) + self.assertEqual(options.annotate, 'default') + self.assertEqual(options.force, True) + self.assertTrue(self.are_default(options, ['annotate', 'force'])) + + def test_wrong_option(self): + old_stderr = sys.stderr + stderr = sys.stderr = StringIO() + try: + self.assertRaises(SystemExit, self.parse_args, + ['--unknown-option'] + ) + finally: + sys.stderr = old_stderr + self.assertTrue(stderr.getvalue()) + + +class TestParseArgs(TestCase): + def setUp(self): + self._options_backup = backup_Options() + + def tearDown(self): + restore_Options(self._options_backup) + + def check_default_global_options(self, white_list=[]): + self.assertEqual(check_global_options(self._options_backup, white_list), "") + + def test_build_set_for_inplace(self): + options, args = parse_args(['foo.pyx', '-i']) + self.assertEqual(options.build, True) + self.check_default_global_options() + + def test_lenient(self): + options, sources = parse_args(['foo.pyx', '--lenient']) + self.assertEqual(sources, ['foo.pyx']) + self.assertEqual(Options.error_on_unknown_names, False) + self.assertEqual(Options.error_on_uninitialized, False) + self.check_default_global_options(['error_on_unknown_names', 'error_on_uninitialized']) + + def test_annotate(self): + options, sources = parse_args(['foo.pyx', '--annotate']) + self.assertEqual(sources, ['foo.pyx']) + self.assertEqual(Options.annotate, 'default') + self.check_default_global_options(['annotate']) + + def test_annotate_fullc(self): + options, sources = parse_args(['foo.pyx', '--annotate-fullc']) + self.assertEqual(sources, ['foo.pyx']) + self.assertEqual(Options.annotate, 'fullc') + self.check_default_global_options(['annotate']) + + def test_no_docstrings(self): + options, sources = parse_args(['foo.pyx', '--no-docstrings']) + self.assertEqual(sources, ['foo.pyx']) + self.assertEqual(Options.docstrings, False) + self.check_default_global_options(['docstrings']) diff --git a/Cython/Build/Tests/TestDependencies.py b/Cython/Build/Tests/TestDependencies.py new file mode 100644 index 000000000..d3888117d --- /dev/null +++ b/Cython/Build/Tests/TestDependencies.py @@ -0,0 +1,142 @@ +import contextlib +import os.path +import sys +import tempfile +import unittest +from io import open +from os.path import join as pjoin + +from ..Dependencies import extended_iglob + + +@contextlib.contextmanager +def writable_file(dir_path, filename): + with open(pjoin(dir_path, filename), "w", encoding="utf8") as f: + yield f + + +class TestGlobbing(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls._orig_dir = os.getcwd() + if sys.version_info[0] < 3: + temp_path = cls._tmpdir = tempfile.mkdtemp() + else: + cls._tmpdir = tempfile.TemporaryDirectory() + temp_path = cls._tmpdir.name + os.chdir(temp_path) + + for dir1 in "abcd": + for dir1x in [dir1, dir1 + 'x']: + for dir2 in "xyz": + dir_path = pjoin(dir1x, dir2) + os.makedirs(dir_path) + with writable_file(dir_path, "file2_pyx.pyx") as f: + f.write(u'""" PYX """') + with writable_file(dir_path, "file2_py.py") as f: + f.write(u'""" PY """') + + with writable_file(dir1x, "file1_pyx.pyx") as f: + f.write(u'""" PYX """') + with writable_file(dir1x, "file1_py.py") as f: + f.write(u'""" PY """') + + @classmethod + def tearDownClass(cls): + os.chdir(cls._orig_dir) + if sys.version_info[0] < 3: + import shutil + shutil.rmtree(cls._tmpdir) + else: + cls._tmpdir.cleanup() + + def files_equal(self, pattern, expected_files): + expected_files = sorted(expected_files) + # It's the users's choice whether '/' will appear on Windows. + matched_files = sorted(path.replace('/', os.sep) for path in extended_iglob(pattern)) + self.assertListEqual(matched_files, expected_files) # / + + # Special case for Windows: also support '\' in patterns. + if os.sep == '\\' and '/' in pattern: + matched_files = sorted(extended_iglob(pattern.replace('/', '\\'))) + self.assertListEqual(matched_files, expected_files) # \ + + def test_extended_iglob_simple(self): + ax_files = [pjoin("a", "x", "file2_pyx.pyx"), pjoin("a", "x", "file2_py.py")] + self.files_equal("a/x/*", ax_files) + self.files_equal("a/x/*.c12", []) + self.files_equal("a/x/*.{py,pyx,c12}", ax_files) + self.files_equal("a/x/*.{py,pyx}", ax_files) + self.files_equal("a/x/*.{pyx}", ax_files[:1]) + self.files_equal("a/x/*.pyx", ax_files[:1]) + self.files_equal("a/x/*.{py}", ax_files[1:]) + self.files_equal("a/x/*.py", ax_files[1:]) + + def test_extended_iglob_simple_star(self): + for basedir in "ad": + files = [ + pjoin(basedir, dirname, filename) + for dirname in "xyz" + for filename in ["file2_pyx.pyx", "file2_py.py"] + ] + self.files_equal(basedir + "/*/*", files) + self.files_equal(basedir + "/*/*.c12", []) + self.files_equal(basedir + "/*/*.{py,pyx,c12}", files) + self.files_equal(basedir + "/*/*.{py,pyx}", files) + self.files_equal(basedir + "/*/*.{pyx}", files[::2]) + self.files_equal(basedir + "/*/*.pyx", files[::2]) + self.files_equal(basedir + "/*/*.{py}", files[1::2]) + self.files_equal(basedir + "/*/*.py", files[1::2]) + + for subdir in "xy*": + files = [ + pjoin(basedir, dirname, filename) + for dirname in "xyz" + if subdir in ('*', dirname) + for filename in ["file2_pyx.pyx", "file2_py.py"] + ] + path = basedir + '/' + subdir + '/' + self.files_equal(path + "*", files) + self.files_equal(path + "*.{py,pyx}", files) + self.files_equal(path + "*.{pyx}", files[::2]) + self.files_equal(path + "*.pyx", files[::2]) + self.files_equal(path + "*.{py}", files[1::2]) + self.files_equal(path + "*.py", files[1::2]) + + def test_extended_iglob_double_star(self): + basedirs = os.listdir(".") + files = [ + pjoin(basedir, dirname, filename) + for basedir in basedirs + for dirname in "xyz" + for filename in ["file2_pyx.pyx", "file2_py.py"] + ] + all_files = [ + pjoin(basedir, filename) + for basedir in basedirs + for filename in ["file1_pyx.pyx", "file1_py.py"] + ] + files + self.files_equal("*/*/*", files) + self.files_equal("*/*/**/*", files) + self.files_equal("*/**/*.*", all_files) + self.files_equal("**/*.*", all_files) + self.files_equal("*/**/*.c12", []) + self.files_equal("**/*.c12", []) + self.files_equal("*/*/*.{py,pyx,c12}", files) + self.files_equal("*/*/**/*.{py,pyx,c12}", files) + self.files_equal("*/**/*/*.{py,pyx,c12}", files) + self.files_equal("**/*/*/*.{py,pyx,c12}", files) + self.files_equal("**/*.{py,pyx,c12}", all_files) + self.files_equal("*/*/*.{py,pyx}", files) + self.files_equal("**/*/*/*.{py,pyx}", files) + self.files_equal("*/**/*/*.{py,pyx}", files) + self.files_equal("**/*.{py,pyx}", all_files) + self.files_equal("*/*/*.{pyx}", files[::2]) + self.files_equal("**/*.{pyx}", all_files[::2]) + self.files_equal("*/**/*/*.pyx", files[::2]) + self.files_equal("*/*/*.pyx", files[::2]) + self.files_equal("**/*.pyx", all_files[::2]) + self.files_equal("*/*/*.{py}", files[1::2]) + self.files_equal("**/*.{py}", all_files[1::2]) + self.files_equal("*/*/*.py", files[1::2]) + self.files_equal("**/*.py", all_files[1::2]) diff --git a/Cython/Build/Tests/TestInline.py b/Cython/Build/Tests/TestInline.py index d20948808..35d9a29cd 100644 --- a/Cython/Build/Tests/TestInline.py +++ b/Cython/Build/Tests/TestInline.py @@ -1,4 +1,6 @@ -import os, tempfile +import os +import tempfile +import unittest from Cython.Shadow import inline from Cython.Build.Inline import safe_type from Cython.TestUtils import CythonTest @@ -85,12 +87,26 @@ class TestInline(CythonTest): inline(inline_divcode, language_level=3)['f'](5,2), 2.5 ) + self.assertEqual( + inline(inline_divcode, language_level=2)['f'](5,2), + 2 + ) - if has_numpy: - - def test_numpy(self): - import numpy - a = numpy.ndarray((10, 20)) - a[0,0] = 10 - self.assertEqual(safe_type(a), 'numpy.ndarray[numpy.float64_t, ndim=2]') - self.assertEqual(inline("return a[0,0]", a=a, **self.test_kwds), 10.0) + def test_repeated_use(self): + inline_mulcode = "def f(int a, int b): return a * b" + self.assertEqual(inline(inline_mulcode)['f'](5, 2), 10) + self.assertEqual(inline(inline_mulcode)['f'](5, 3), 15) + self.assertEqual(inline(inline_mulcode)['f'](6, 2), 12) + self.assertEqual(inline(inline_mulcode)['f'](5, 2), 10) + + f = inline(inline_mulcode)['f'] + self.assertEqual(f(5, 2), 10) + self.assertEqual(f(5, 3), 15) + + @unittest.skipIf(not has_numpy, "NumPy is not available") + def test_numpy(self): + import numpy + a = numpy.ndarray((10, 20)) + a[0,0] = 10 + self.assertEqual(safe_type(a), 'numpy.ndarray[numpy.float64_t, ndim=2]') + self.assertEqual(inline("return a[0,0]", a=a, **self.test_kwds), 10.0) diff --git a/Cython/Build/Tests/TestIpythonMagic.py b/Cython/Build/Tests/TestIpythonMagic.py index 24213091b..febb480ac 100644 --- a/Cython/Build/Tests/TestIpythonMagic.py +++ b/Cython/Build/Tests/TestIpythonMagic.py @@ -6,10 +6,12 @@ from __future__ import absolute_import import os +import io import sys from contextlib import contextmanager from Cython.Build import IpythonMagic from Cython.TestUtils import CythonTest +from Cython.Compiler.Annotate import AnnotationCCodeWriter try: import IPython.testing.globalipapp @@ -28,6 +30,26 @@ try: except ImportError: pass + +@contextmanager +def capture_output(): + backup = sys.stdout, sys.stderr + try: + replacement = [ + io.TextIOWrapper(io.BytesIO(), encoding=sys.stdout.encoding), + io.TextIOWrapper(io.BytesIO(), encoding=sys.stderr.encoding), + ] + sys.stdout, sys.stderr = replacement + output = [] + yield output + finally: + sys.stdout, sys.stderr = backup + for wrapper in replacement: + wrapper.seek(0) # rewind + output.append(wrapper.read()) + wrapper.close() + + code = u"""\ def f(x): return 2*x @@ -47,6 +69,27 @@ def main(): main() """ +compile_error_code = u'''\ +cdef extern from *: + """ + xxx a=1; + """ + int a; +def doit(): + return a +''' + +compile_warning_code = u'''\ +cdef extern from *: + """ + #pragma message ( "CWarning" ) + int a = 42; + """ + int a; +def doit(): + return a +''' + if sys.platform == 'win32': # not using IPython's decorators here because they depend on "nose" @@ -142,6 +185,39 @@ class TestIPythonMagic(CythonTest): self.assertEqual(ip.user_ns['g'], 2 // 10) self.assertEqual(ip.user_ns['h'], 2 // 10) + def test_cython_compile_error_shown(self): + ip = self._ip + with capture_output() as out: + ip.run_cell_magic('cython', '-3', compile_error_code) + captured_out, captured_err = out + + # it could be that c-level output is captured by distutil-extension + # (and not by us) and is printed to stdout: + captured_all = captured_out + "\n" + captured_err + self.assertTrue("error" in captured_all, msg="error in " + captured_all) + + def test_cython_link_error_shown(self): + ip = self._ip + with capture_output() as out: + ip.run_cell_magic('cython', '-3 -l=xxxxxxxx', code) + captured_out, captured_err = out + + # it could be that c-level output is captured by distutil-extension + # (and not by us) and is printed to stdout: + captured_all = captured_out + "\n!" + captured_err + self.assertTrue("error" in captured_all, msg="error in " + captured_all) + + def test_cython_warning_shown(self): + ip = self._ip + with capture_output() as out: + # force rebuild, otherwise no warning as after the first success + # no build step is performed + ip.run_cell_magic('cython', '-3 -f', compile_warning_code) + captured_out, captured_err = out + + # check that warning was printed to stdout even if build hasn't failed + self.assertTrue("CWarning" in captured_out) + @skip_win32('Skip on Windows') def test_cython3_pgo(self): # The Cython cell defines the functions f() and call(). @@ -203,3 +279,29 @@ x = sin(0.0) ip.ex('g = f(10)') self.assertEqual(ip.user_ns['g'], 20.0) self.assertEqual([normal_log.INFO], normal_log.thresholds) + + def test_cython_no_annotate(self): + ip = self._ip + html = ip.run_cell_magic('cython', '', code) + self.assertTrue(html is None) + + def test_cython_annotate(self): + ip = self._ip + html = ip.run_cell_magic('cython', '--annotate', code) + # somewhat brittle way to differentiate between annotated htmls + # with/without complete source code: + self.assertTrue(AnnotationCCodeWriter.COMPLETE_CODE_TITLE not in html.data) + + def test_cython_annotate_default(self): + ip = self._ip + html = ip.run_cell_magic('cython', '-a', code) + # somewhat brittle way to differentiate between annotated htmls + # with/without complete source code: + self.assertTrue(AnnotationCCodeWriter.COMPLETE_CODE_TITLE not in html.data) + + def test_cython_annotate_complete_c_code(self): + ip = self._ip + html = ip.run_cell_magic('cython', '--annotate-fullc', code) + # somewhat brittle way to differentiate between annotated htmls + # with/without complete source code: + self.assertTrue(AnnotationCCodeWriter.COMPLETE_CODE_TITLE in html.data) diff --git a/Cython/Build/Tests/TestRecythonize.py b/Cython/Build/Tests/TestRecythonize.py new file mode 100644 index 000000000..eb87018cb --- /dev/null +++ b/Cython/Build/Tests/TestRecythonize.py @@ -0,0 +1,212 @@ +import shutil +import os +import tempfile +import time + +import Cython.Build.Dependencies +import Cython.Utils +from Cython.TestUtils import CythonTest + + +def fresh_cythonize(*args, **kwargs): + Cython.Utils.clear_function_caches() + Cython.Build.Dependencies._dep_tree = None # discard method caches + Cython.Build.Dependencies.cythonize(*args, **kwargs) + +class TestRecythonize(CythonTest): + + def setUp(self): + CythonTest.setUp(self) + self.temp_dir = ( + tempfile.mkdtemp( + prefix='recythonize-test', + dir='TEST_TMP' if os.path.isdir('TEST_TMP') else None + ) + ) + + def tearDown(self): + CythonTest.tearDown(self) + shutil.rmtree(self.temp_dir) + + def test_recythonize_pyx_on_pxd_change(self): + + src_dir = tempfile.mkdtemp(prefix='src', dir=self.temp_dir) + + a_pxd = os.path.join(src_dir, 'a.pxd') + a_pyx = os.path.join(src_dir, 'a.pyx') + a_c = os.path.join(src_dir, 'a.c') + dep_tree = Cython.Build.Dependencies.create_dependency_tree() + + with open(a_pxd, 'w') as f: + f.write('cdef int value\n') + + with open(a_pyx, 'w') as f: + f.write('value = 1\n') + + + # The dependencies for "a.pyx" are "a.pxd" and "a.pyx". + self.assertEqual({a_pxd, a_pyx}, dep_tree.all_dependencies(a_pyx)) + + # Cythonize to create a.c + fresh_cythonize(a_pyx) + + # Sleep to address coarse time-stamp precision. + time.sleep(1) + + with open(a_c) as f: + a_c_contents1 = f.read() + + with open(a_pxd, 'w') as f: + f.write('cdef double value\n') + + fresh_cythonize(a_pyx) + + with open(a_c) as f: + a_c_contents2 = f.read() + + self.assertTrue("__pyx_v_1a_value = 1;" in a_c_contents1) + self.assertFalse("__pyx_v_1a_value = 1;" in a_c_contents2) + self.assertTrue("__pyx_v_1a_value = 1.0;" in a_c_contents2) + self.assertFalse("__pyx_v_1a_value = 1.0;" in a_c_contents1) + + + def test_recythonize_py_on_pxd_change(self): + + src_dir = tempfile.mkdtemp(prefix='src', dir=self.temp_dir) + + a_pxd = os.path.join(src_dir, 'a.pxd') + a_py = os.path.join(src_dir, 'a.py') + a_c = os.path.join(src_dir, 'a.c') + dep_tree = Cython.Build.Dependencies.create_dependency_tree() + + with open(a_pxd, 'w') as f: + f.write('cdef int value\n') + + with open(a_py, 'w') as f: + f.write('value = 1\n') + + + # The dependencies for "a.py" are "a.pxd" and "a.py". + self.assertEqual({a_pxd, a_py}, dep_tree.all_dependencies(a_py)) + + # Cythonize to create a.c + fresh_cythonize(a_py) + + # Sleep to address coarse time-stamp precision. + time.sleep(1) + + with open(a_c) as f: + a_c_contents1 = f.read() + + with open(a_pxd, 'w') as f: + f.write('cdef double value\n') + + fresh_cythonize(a_py) + + with open(a_c) as f: + a_c_contents2 = f.read() + + + self.assertTrue("__pyx_v_1a_value = 1;" in a_c_contents1) + self.assertFalse("__pyx_v_1a_value = 1;" in a_c_contents2) + self.assertTrue("__pyx_v_1a_value = 1.0;" in a_c_contents2) + self.assertFalse("__pyx_v_1a_value = 1.0;" in a_c_contents1) + + def test_recythonize_pyx_on_dep_pxd_change(self): + src_dir = tempfile.mkdtemp(prefix='src', dir=self.temp_dir) + + a_pxd = os.path.join(src_dir, 'a.pxd') + a_pyx = os.path.join(src_dir, 'a.pyx') + b_pyx = os.path.join(src_dir, 'b.pyx') + b_c = os.path.join(src_dir, 'b.c') + dep_tree = Cython.Build.Dependencies.create_dependency_tree() + + with open(a_pxd, 'w') as f: + f.write('cdef int value\n') + + with open(a_pyx, 'w') as f: + f.write('value = 1\n') + + with open(b_pyx, 'w') as f: + f.write('cimport a\n' + 'a.value = 2\n') + + + # The dependencies for "b.pyx" are "a.pxd" and "b.pyx". + self.assertEqual({a_pxd, b_pyx}, dep_tree.all_dependencies(b_pyx)) + + + # Cythonize to create b.c + fresh_cythonize([a_pyx, b_pyx]) + + # Sleep to address coarse time-stamp precision. + time.sleep(1) + + with open(b_c) as f: + b_c_contents1 = f.read() + + with open(a_pxd, 'w') as f: + f.write('cdef double value\n') + + fresh_cythonize([a_pyx, b_pyx]) + + with open(b_c) as f: + b_c_contents2 = f.read() + + + + self.assertTrue("__pyx_v_1a_value = 2;" in b_c_contents1) + self.assertFalse("__pyx_v_1a_value = 2;" in b_c_contents2) + self.assertTrue("__pyx_v_1a_value = 2.0;" in b_c_contents2) + self.assertFalse("__pyx_v_1a_value = 2.0;" in b_c_contents1) + + + + def test_recythonize_py_on_dep_pxd_change(self): + + src_dir = tempfile.mkdtemp(prefix='src', dir=self.temp_dir) + + a_pxd = os.path.join(src_dir, 'a.pxd') + a_pyx = os.path.join(src_dir, 'a.pyx') + b_pxd = os.path.join(src_dir, 'b.pxd') + b_py = os.path.join(src_dir, 'b.py') + b_c = os.path.join(src_dir, 'b.c') + dep_tree = Cython.Build.Dependencies.create_dependency_tree() + + with open(a_pxd, 'w') as f: + f.write('cdef int value\n') + + with open(a_pyx, 'w') as f: + f.write('value = 1\n') + + with open(b_pxd, 'w') as f: + f.write('cimport a\n') + + with open(b_py, 'w') as f: + f.write('a.value = 2\n') + + + # The dependencies for b.py are "a.pxd", "b.pxd" and "b.py". + self.assertEqual({a_pxd, b_pxd, b_py}, dep_tree.all_dependencies(b_py)) + + + # Cythonize to create b.c + fresh_cythonize([a_pyx, b_py]) + + # Sleep to address coarse time-stamp precision. + time.sleep(1) + + with open(b_c) as f: + b_c_contents1 = f.read() + + with open(a_pxd, 'w') as f: + f.write('cdef double value\n') + + fresh_cythonize([a_pyx, b_py]) + + with open(b_c) as f: + b_c_contents2 = f.read() + + self.assertTrue("__pyx_v_1a_value = 2;" in b_c_contents1) + self.assertFalse("__pyx_v_1a_value = 2;" in b_c_contents2) + self.assertTrue("__pyx_v_1a_value = 2.0;" in b_c_contents2) + self.assertFalse("__pyx_v_1a_value = 2.0;" in b_c_contents1) diff --git a/Cython/Build/Tests/TestStripLiterals.py b/Cython/Build/Tests/TestStripLiterals.py index a7572a508..cbe5c65a9 100644 --- a/Cython/Build/Tests/TestStripLiterals.py +++ b/Cython/Build/Tests/TestStripLiterals.py @@ -54,4 +54,3 @@ class TestStripLiterals(CythonTest): def test_extern(self): self.t("cdef extern from 'a.h': # comment", "cdef extern from '_L1_': #_L2_") - diff --git a/Cython/CodeWriter.py b/Cython/CodeWriter.py index a5453b90b..f386da21c 100644 --- a/Cython/CodeWriter.py +++ b/Cython/CodeWriter.py @@ -1,7 +1,6 @@ """ Serializes a Cython code tree to Cython code. This is primarily useful for debugging and testing purposes. - The output is in a strict format, no whitespace or comments from the input is preserved (and it could not be as it is not present in the code tree). """ @@ -10,6 +9,7 @@ from __future__ import absolute_import, print_function from .Compiler.Visitor import TreeVisitor from .Compiler.ExprNodes import * +from .Compiler.Nodes import CSimpleBaseTypeNode class LinesResult(object): @@ -28,7 +28,11 @@ class LinesResult(object): self.put(s) self.newline() + class DeclarationWriter(TreeVisitor): + """ + A Cython code writer that is limited to declarations nodes. + """ indent_string = u" " @@ -76,6 +80,14 @@ class DeclarationWriter(TreeVisitor): self.visit(item.default) self.put(u", ") self.visit(items[-1]) + if output_rhs and items[-1].default is not None: + self.put(u" = ") + self.visit(items[-1].default) + + def _visit_indented(self, node): + self.indent() + self.visit(node) + self.dedent() def visit_Node(self, node): raise AssertionError("Node not handled by serializer: %r" % node) @@ -92,9 +104,7 @@ class DeclarationWriter(TreeVisitor): else: file = u'"%s"' % node.include_file self.putline(u"cdef extern from %s:" % file) - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) def visit_CPtrDeclaratorNode(self, node): self.put('*') @@ -111,13 +121,6 @@ class DeclarationWriter(TreeVisitor): self.visit(node.dimension) self.put(u']') - def visit_CArrayDeclaratorNode(self, node): - self.visit(node.base) - self.put(u'[') - if node.dimension is not None: - self.visit(node.dimension) - self.put(u']') - def visit_CFuncDeclaratorNode(self, node): # TODO: except, gil, etc. self.visit(node.base) @@ -136,13 +139,12 @@ class DeclarationWriter(TreeVisitor): self.put("short " * -node.longness) elif node.longness > 0: self.put("long " * node.longness) - self.put(node.name) + if node.name is not None: + self.put(node.name) def visit_CComplexBaseTypeNode(self, node): - self.put(u'(') self.visit(node.base_type) self.visit(node.declarator) - self.put(u')') def visit_CNestedBaseTypeNode(self, node): self.visit(node.base_type) @@ -162,7 +164,7 @@ class DeclarationWriter(TreeVisitor): self.comma_separated_list(node.declarators, output_rhs=True) self.endline() - def visit_container_node(self, node, decl, extras, attributes): + def _visit_container_node(self, node, decl, extras, attributes): # TODO: visibility self.startline(decl) if node.name: @@ -191,7 +193,7 @@ class DeclarationWriter(TreeVisitor): if node.packed: decl += u'packed ' decl += node.kind - self.visit_container_node(node, decl, None, node.attributes) + self._visit_container_node(node, decl, None, node.attributes) def visit_CppClassNode(self, node): extras = "" @@ -199,10 +201,10 @@ class DeclarationWriter(TreeVisitor): extras = u"[%s]" % ", ".join(node.templates) if node.base_classes: extras += "(%s)" % ", ".join(node.base_classes) - self.visit_container_node(node, u"cdef cppclass", extras, node.attributes) + self._visit_container_node(node, u"cdef cppclass", extras, node.attributes) def visit_CEnumDefNode(self, node): - self.visit_container_node(node, u"cdef enum", None, node.items) + self._visit_container_node(node, u"cdef enum", None, node.items) def visit_CEnumDefItemNode(self, node): self.startline(node.name) @@ -228,9 +230,7 @@ class DeclarationWriter(TreeVisitor): self.put(node.base_class_name) self.put(u")") self.endline(u":") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) def visit_CTypeDefNode(self, node): self.startline(u"ctypedef ") @@ -240,17 +240,49 @@ class DeclarationWriter(TreeVisitor): self.endline() def visit_FuncDefNode(self, node): + # TODO: support cdef + cpdef functions self.startline(u"def %s(" % node.name) self.comma_separated_list(node.args) self.endline(u"):") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) + + def visit_CFuncDefNode(self, node): + self.startline(u'cpdef ' if node.overridable else u'cdef ') + if node.modifiers: + self.put(' '.join(node.modifiers)) + self.put(' ') + if node.visibility != 'private': + self.put(node.visibility) + self.put(u' ') + if node.api: + self.put(u'api ') + + if node.base_type: + self.visit(node.base_type) + if node.base_type.name is not None: + self.put(u' ') + + # visit the CFuncDeclaratorNode, but put a `:` at the end of line + self.visit(node.declarator.base) + self.put(u'(') + self.comma_separated_list(node.declarator.args) + self.endline(u'):') + + self._visit_indented(node.body) def visit_CArgDeclNode(self, node): - if node.base_type.name is not None: + # For "CSimpleBaseTypeNode", the variable type may have been parsed as type. + # For other node types, the "name" is always None. + if not isinstance(node.base_type, CSimpleBaseTypeNode) or \ + node.base_type.name is not None: self.visit(node.base_type) - self.put(u" ") + + # If we printed something for "node.base_type", we may need to print an extra ' '. + # + # Special case: if "node.declarator" is a "CNameDeclaratorNode", + # its "name" might be an empty string, for example, for "cdef f(x)". + if node.declarator.declared_name(): + self.put(u" ") self.visit(node.declarator) if node.default is not None: self.put(u" = ") @@ -284,46 +316,20 @@ class DeclarationWriter(TreeVisitor): def visit_NameNode(self, node): self.put(node.name) - def visit_IntNode(self, node): - self.put(node.value) - - def visit_NoneNode(self, node): - self.put(u"None") - - def visit_NotNode(self, node): - self.put(u"(not ") - self.visit(node.operand) - self.put(u")") - def visit_DecoratorNode(self, node): self.startline("@") self.visit(node.decorator) self.endline() - def visit_BinopNode(self, node): - self.visit(node.operand1) - self.put(u" %s " % node.operator) - self.visit(node.operand2) - - def visit_AttributeNode(self, node): - self.visit(node.obj) - self.put(u".%s" % node.attribute) - - def visit_BoolNode(self, node): - self.put(str(node.value)) - - # FIXME: represent string nodes correctly - def visit_StringNode(self, node): - value = node.value - if value.encoding is not None: - value = value.encode(value.encoding) - self.put(repr(value)) - def visit_PassStatNode(self, node): self.startline(u"pass") self.endline() -class CodeWriter(DeclarationWriter): + +class StatementWriter(DeclarationWriter): + """ + A Cython code writer for most language statement features. + """ def visit_SingleAssignmentNode(self, node): self.startline() @@ -349,18 +355,17 @@ class CodeWriter(DeclarationWriter): def visit_ForInStatNode(self, node): self.startline(u"for ") - self.visit(node.target) + if node.target.is_sequence_constructor: + self.comma_separated_list(node.target.args) + else: + self.visit(node.target) self.put(u" in ") self.visit(node.iterator.sequence) self.endline(u":") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) if node.else_clause is not None: self.line(u"else:") - self.indent() - self.visit(node.else_clause) - self.dedent() + self._visit_indented(node.else_clause) def visit_IfStatNode(self, node): # The IfClauseNode is handled directly without a separate match @@ -368,50 +373,33 @@ class CodeWriter(DeclarationWriter): self.startline(u"if ") self.visit(node.if_clauses[0].condition) self.endline(":") - self.indent() - self.visit(node.if_clauses[0].body) - self.dedent() + self._visit_indented(node.if_clauses[0].body) for clause in node.if_clauses[1:]: self.startline("elif ") self.visit(clause.condition) self.endline(":") - self.indent() - self.visit(clause.body) - self.dedent() + self._visit_indented(clause.body) if node.else_clause is not None: self.line("else:") - self.indent() - self.visit(node.else_clause) - self.dedent() + self._visit_indented(node.else_clause) - def visit_SequenceNode(self, node): - self.comma_separated_list(node.args) # Might need to discover whether we need () around tuples...hmm... + def visit_WhileStatNode(self, node): + self.startline(u"while ") + self.visit(node.condition) + self.endline(u":") + self._visit_indented(node.body) + if node.else_clause is not None: + self.line("else:") + self._visit_indented(node.else_clause) - def visit_SimpleCallNode(self, node): - self.visit(node.function) - self.put(u"(") - self.comma_separated_list(node.args) - self.put(")") + def visit_ContinueStatNode(self, node): + self.line(u"continue") - def visit_GeneralCallNode(self, node): - self.visit(node.function) - self.put(u"(") - posarg = node.positional_args - if isinstance(posarg, AsTupleNode): - self.visit(posarg.arg) - else: - self.comma_separated_list(posarg.args) # TupleNode.args - if node.keyword_args: - if isinstance(node.keyword_args, DictNode): - for i, (name, value) in enumerate(node.keyword_args.key_value_pairs): - if i > 0: - self.put(', ') - self.visit(name) - self.put('=') - self.visit(value) - else: - raise Exception("Not implemented yet") - self.put(u")") + def visit_BreakStatNode(self, node): + self.line(u"break") + + def visit_SequenceNode(self, node): + self.comma_separated_list(node.args) # Might need to discover whether we need () around tuples...hmm... def visit_ExprStatNode(self, node): self.startline() @@ -433,25 +421,17 @@ class CodeWriter(DeclarationWriter): self.put(u" as ") self.visit(node.target) self.endline(u":") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) def visit_TryFinallyStatNode(self, node): self.line(u"try:") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) self.line(u"finally:") - self.indent() - self.visit(node.finally_clause) - self.dedent() + self._visit_indented(node.finally_clause) def visit_TryExceptStatNode(self, node): self.line(u"try:") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) for x in node.except_clauses: self.visit(x) if node.else_clause is not None: @@ -466,13 +446,13 @@ class CodeWriter(DeclarationWriter): self.put(u", ") self.visit(node.target) self.endline(":") - self.indent() - self.visit(node.body) - self.dedent() + self._visit_indented(node.body) def visit_ReturnStatNode(self, node): - self.startline("return ") - self.visit(node.value) + self.startline("return") + if node.value is not None: + self.put(u" ") + self.visit(node.value) self.endline() def visit_ReraiseStatNode(self, node): @@ -498,30 +478,10 @@ class CodeWriter(DeclarationWriter): self.put(self.tempnames[node.handle]) -class PxdWriter(DeclarationWriter): - def __call__(self, node): - print(u'\n'.join(self.write(node).lines)) - return node - - def visit_CFuncDefNode(self, node): - if 'inline' in node.modifiers: - return - if node.overridable: - self.startline(u'cpdef ') - else: - self.startline(u'cdef ') - if node.visibility != 'private': - self.put(node.visibility) - self.put(u' ') - if node.api: - self.put(u'api ') - self.visit(node.declarator) - - def visit_StatNode(self, node): - pass - - class ExpressionWriter(TreeVisitor): + """ + A Cython code writer that is intentionally limited to expressions. + """ def __init__(self, result=None): super(ExpressionWriter, self).__init__() @@ -551,12 +511,18 @@ class ExpressionWriter(TreeVisitor): def visit_Node(self, node): raise AssertionError("Node not handled by serializer: %r" % node) - def visit_NameNode(self, node): - self.put(node.name) + def visit_IntNode(self, node): + self.put(node.value) + + def visit_FloatNode(self, node): + self.put(node.value) def visit_NoneNode(self, node): self.put(u"None") + def visit_NameNode(self, node): + self.put(node.name) + def visit_EllipsisNode(self, node): self.put(u"...") @@ -817,3 +783,38 @@ class ExpressionWriter(TreeVisitor): # type(body) is Nodes.ExprStatNode body = body.expr.arg self.emit_comprehension(body, target, sequence, condition, u"()") + + +class PxdWriter(DeclarationWriter, ExpressionWriter): + """ + A Cython code writer for everything supported in pxd files. + (currently unused) + """ + + def __call__(self, node): + print(u'\n'.join(self.write(node).lines)) + return node + + def visit_CFuncDefNode(self, node): + if node.overridable: + self.startline(u'cpdef ') + else: + self.startline(u'cdef ') + if node.modifiers: + self.put(' '.join(node.modifiers)) + self.put(' ') + if node.visibility != 'private': + self.put(node.visibility) + self.put(u' ') + if node.api: + self.put(u'api ') + self.visit(node.declarator) + + def visit_StatNode(self, node): + pass + + +class CodeWriter(StatementWriter, ExpressionWriter): + """ + A complete Cython code writer. + """ diff --git a/Cython/Compiler/AnalysedTreeTransforms.py b/Cython/Compiler/AnalysedTreeTransforms.py index 07bf31f3e..d4941606e 100644 --- a/Cython/Compiler/AnalysedTreeTransforms.py +++ b/Cython/Compiler/AnalysedTreeTransforms.py @@ -10,9 +10,9 @@ from . import Symtab class AutoTestDictTransform(ScopeTrackingTransform): # Handles autotestdict directive - blacklist = ['__cinit__', '__dealloc__', '__richcmp__', - '__nonzero__', '__bool__', - '__len__', '__contains__'] + excludelist = ['__cinit__', '__dealloc__', '__richcmp__', + '__nonzero__', '__bool__', + '__len__', '__contains__'] def visit_ModuleNode(self, node): if node.is_pxd: @@ -81,7 +81,7 @@ class AutoTestDictTransform(ScopeTrackingTransform): name = node.entry.name else: name = node.name - if self.scope_type == 'cclass' and name in self.blacklist: + if self.scope_type == 'cclass' and name in self.excludelist: return node if self.scope_type == 'pyclass': class_name = self.scope_node.name diff --git a/Cython/Compiler/Annotate.py b/Cython/Compiler/Annotate.py index 5feac02d8..8e8d2c4a8 100644 --- a/Cython/Compiler/Annotate.py +++ b/Cython/Compiler/Annotate.py @@ -23,8 +23,12 @@ from .. import Utils class AnnotationCCodeWriter(CCodeWriter): - def __init__(self, create_from=None, buffer=None, copy_formatting=True): + # also used as marker for detection of complete code emission in tests + COMPLETE_CODE_TITLE = "Complete cythonized code" + + def __init__(self, create_from=None, buffer=None, copy_formatting=True, show_entire_c_code=False, source_desc=None): CCodeWriter.__init__(self, create_from, buffer, copy_formatting=copy_formatting) + self.show_entire_c_code = show_entire_c_code if create_from is None: self.annotation_buffer = StringIO() self.last_annotated_pos = None @@ -45,8 +49,8 @@ class AnnotationCCodeWriter(CCodeWriter): def create_new(self, create_from, buffer, copy_formatting): return AnnotationCCodeWriter(create_from, buffer, copy_formatting) - def write(self, s): - CCodeWriter.write(self, s) + def _write_to_buffer(self, s): + self.buffer.write(s) self.annotation_buffer.write(s) def mark_pos(self, pos, trace=True): @@ -69,7 +73,7 @@ class AnnotationCCodeWriter(CCodeWriter): """css template will later allow to choose a colormap""" css = [self._css_template] for i in range(255): - color = u"FFFF%02x" % int(255/(1+i/10.0)) + color = u"FFFF%02x" % int(255.0 // (1.0 + i/10.0)) css.append('.cython.score-%d {background-color: #%s;}' % (i, color)) try: from pygments.formatters import HtmlFormatter @@ -83,7 +87,7 @@ class AnnotationCCodeWriter(CCodeWriter): body.cython { font-family: courier; font-size: 12; } .cython.tag { } - .cython.line { margin: 0em } + .cython.line { color: #000000; margin: 0em } .cython.code { font-size: 9; color: #444444; display: none; margin: 0px 0px 0px 8px; border-left: 8px none; } .cython.line .run { background-color: #B0FFB0; } @@ -198,17 +202,24 @@ class AnnotationCCodeWriter(CCodeWriter): for line in coverage_data.iterfind('lines/line') ) - def _htmlify_code(self, code): + def _htmlify_code(self, code, language): try: from pygments import highlight - from pygments.lexers import CythonLexer + from pygments.lexers import CythonLexer, CppLexer from pygments.formatters import HtmlFormatter except ImportError: # no Pygments, just escape the code return html_escape(code) + if language == "cython": + lexer = CythonLexer(stripnl=False, stripall=False) + elif language == "c/cpp": + lexer = CppLexer(stripnl=False, stripall=False) + else: + # unknown language, use fallback + return html_escape(code) html_code = highlight( - code, CythonLexer(stripnl=False, stripall=False), + code, lexer, HtmlFormatter(nowrap=True)) return html_code @@ -228,7 +239,7 @@ class AnnotationCCodeWriter(CCodeWriter): return u"<span class='%s'>%s</span>" % ( group_name, match.group(group_name)) - lines = self._htmlify_code(cython_code).splitlines() + lines = self._htmlify_code(cython_code, "cython").splitlines() lineno_width = len(str(len(lines))) if not covered_lines: covered_lines = None @@ -279,6 +290,19 @@ class AnnotationCCodeWriter(CCodeWriter): outlist.append(u"<pre class='cython code score-{score} {covered}'>{code}</pre>".format( score=score, covered=covered, code=c_code)) outlist.append(u"</div>") + + # now the whole c-code if needed: + if self.show_entire_c_code: + outlist.append(u'<p><div class="cython">') + onclick_title = u"<pre class='cython line'{onclick}>+ {title}</pre>\n" + outlist.append(onclick_title.format( + onclick=self._onclick_attr, + title=AnnotationCCodeWriter.COMPLETE_CODE_TITLE, + )) + complete_code_as_html = self._htmlify_code(self.buffer.getvalue(), "c/cpp") + outlist.append(u"<pre class='cython code'>{code}</pre>".format(code=complete_code_as_html)) + outlist.append(u"</div></p>") + return outlist diff --git a/Cython/Compiler/AutoDocTransforms.py b/Cython/Compiler/AutoDocTransforms.py index d3c0a1d0d..6c342f7ef 100644 --- a/Cython/Compiler/AutoDocTransforms.py +++ b/Cython/Compiler/AutoDocTransforms.py @@ -3,18 +3,48 @@ from __future__ import absolute_import, print_function from .Visitor import CythonTransform from .StringEncoding import EncodedString from . import Options -from . import PyrexTypes, ExprNodes +from . import PyrexTypes from ..CodeWriter import ExpressionWriter +from .Errors import warning class AnnotationWriter(ExpressionWriter): + """ + A Cython code writer for Python expressions in argument/variable annotations. + """ + def __init__(self, description=None): + """description is optional. If specified it is used in + warning messages for the nodes that don't convert to string properly. + If not specified then no messages are generated. + """ + ExpressionWriter.__init__(self) + self.description = description + self.incomplete = False def visit_Node(self, node): self.put(u"<???>") + self.incomplete = True + if self.description: + warning(node.pos, + "Failed to convert code to string representation in {0}".format( + self.description), level=1) def visit_LambdaNode(self, node): # XXX Should we do better? self.put("<lambda>") + self.incomplete = True + if self.description: + warning(node.pos, + "Failed to convert lambda to string representation in {0}".format( + self.description), level=1) + + def visit_UnicodeNode(self, node): + # Discard Unicode prefix in annotations. Any tool looking at them + # would probably expect Py3 string semantics. + self.emit_string(node, "") + + def visit_AnnotationNode(self, node): + self.put(node.string.unicode_value) class EmbedSignature(CythonTransform): @@ -25,6 +55,12 @@ class EmbedSignature(CythonTransform): self.class_node = None def _fmt_expr(self, node): + writer = ExpressionWriter() + result = writer.write(node) + # print(type(node).__name__, '-->', result) + return result + + def _fmt_annotation(self, node): writer = AnnotationWriter() result = writer.write(node) # print(type(node).__name__, '-->', result) @@ -37,7 +73,7 @@ class EmbedSignature(CythonTransform): doc = arg.type.declaration_code(arg.name, for_display=1) if arg.annotation: - annotation = self._fmt_expr(arg.annotation) + annotation = self._fmt_annotation(arg.annotation) doc = doc + (': %s' % annotation) if arg.default: default = self._fmt_expr(arg.default) @@ -50,12 +86,12 @@ class EmbedSignature(CythonTransform): def _fmt_star_arg(self, arg): arg_doc = arg.name if arg.annotation: - annotation = self._fmt_expr(arg.annotation) + annotation = self._fmt_annotation(arg.annotation) arg_doc = arg_doc + (': %s' % annotation) return arg_doc def _fmt_arglist(self, args, - npargs=0, pargs=None, + npoargs=0, npargs=0, pargs=None, nkargs=0, kargs=None, hide_self=False): arglist = [] @@ -65,9 +101,11 @@ class EmbedSignature(CythonTransform): arglist.append(arg_doc) if pargs: arg_doc = self._fmt_star_arg(pargs) - arglist.insert(npargs, '*%s' % arg_doc) + arglist.insert(npargs + npoargs, '*%s' % arg_doc) elif nkargs: - arglist.insert(npargs, '*') + arglist.insert(npargs + npoargs, '*') + if npoargs: + arglist.insert(npoargs, '/') if kargs: arg_doc = self._fmt_star_arg(kargs) arglist.append('**%s' % arg_doc) @@ -80,12 +118,12 @@ class EmbedSignature(CythonTransform): return ret.declaration_code("", for_display=1) def _fmt_signature(self, cls_name, func_name, args, - npargs=0, pargs=None, + npoargs=0, npargs=0, pargs=None, nkargs=0, kargs=None, return_expr=None, return_type=None, hide_self=False): arglist = self._fmt_arglist(args, - npargs, pargs, + npoargs, npargs, pargs, nkargs, kargs, hide_self=hide_self) arglist_doc = ', '.join(arglist) @@ -94,7 +132,7 @@ class EmbedSignature(CythonTransform): func_doc = '%s.%s' % (cls_name, func_doc) ret_doc = None if return_expr: - ret_doc = self._fmt_expr(return_expr) + ret_doc = self._fmt_annotation(return_expr) elif return_type: ret_doc = self._fmt_ret_type(return_type) if ret_doc: @@ -147,11 +185,12 @@ class EmbedSignature(CythonTransform): else: class_name, func_name = self.class_name, node.name + npoargs = getattr(node, 'num_posonly_args', 0) nkargs = getattr(node, 'num_kwonly_args', 0) - npargs = len(node.args) - nkargs + npargs = len(node.args) - nkargs - npoargs signature = self._fmt_signature( class_name, func_name, node.args, - npargs, node.star_arg, + npoargs, npargs, node.star_arg, nkargs, node.starstar_arg, return_expr=node.return_type_annotation, return_type=None, hide_self=hide_self) @@ -176,7 +215,7 @@ class EmbedSignature(CythonTransform): def visit_CFuncDefNode(self, node): if not self.current_directives['embedsignature']: return node - if not node.overridable: # not cpdef FOO(...): + if not node.overridable: # not cpdef FOO(...): return node signature = self._fmt_signature( @@ -192,8 +231,9 @@ class EmbedSignature(CythonTransform): old_doc = None new_doc = self._embed_signature(signature, old_doc) node.entry.doc = EncodedString(new_doc) - if hasattr(node, 'py_func') and node.py_func is not None: - node.py_func.entry.doc = EncodedString(new_doc) + py_func = getattr(node, 'py_func', None) + if py_func is not None: + py_func.entry.doc = EncodedString(new_doc) return node def visit_PropertyNode(self, node): diff --git a/Cython/Compiler/Buffer.py b/Cython/Compiler/Buffer.py index c62a24f56..e86e1e9c2 100644 --- a/Cython/Compiler/Buffer.py +++ b/Cython/Compiler/Buffer.py @@ -85,7 +85,7 @@ class IntroduceBufferAuxiliaryVars(CythonTransform): aux_var = scope.declare_var(name=None, cname=cname, type=type, pos=node.pos) if entry.is_arg: - aux_var.used = True # otherwise, NameNode will mark whether it is used + aux_var.used = True # otherwise, NameNode will mark whether it is used return aux_var @@ -111,9 +111,9 @@ class IntroduceBufferAuxiliaryVars(CythonTransform): # # Analysis # -buffer_options = ("dtype", "ndim", "mode", "negative_indices", "cast") # ordered! +buffer_options = ("dtype", "ndim", "mode", "negative_indices", "cast") # ordered! buffer_defaults = {"ndim": 1, "mode": "full", "negative_indices": True, "cast": False} -buffer_positional_options_count = 1 # anything beyond this needs keyword argument +buffer_positional_options_count = 1 # anything beyond this needs keyword argument ERR_BUF_OPTION_UNKNOWN = '"%s" is not a buffer option' ERR_BUF_TOO_MANY = 'Too many buffer options' @@ -146,12 +146,12 @@ def analyse_buffer_options(globalpos, env, posargs, dictargs, defaults=None, nee options = {} for name, (value, pos) in dictargs.items(): - if not name in buffer_options: + if name not in buffer_options: raise CompileError(pos, ERR_BUF_OPTION_UNKNOWN % name) options[name] = value for name, (value, pos) in zip(buffer_options, posargs): - if not name in buffer_options: + if name not in buffer_options: raise CompileError(pos, ERR_BUF_OPTION_UNKNOWN % name) if name in options: raise CompileError(pos, ERR_BUF_DUP % name) @@ -159,7 +159,7 @@ def analyse_buffer_options(globalpos, env, posargs, dictargs, defaults=None, nee # Check that they are all there and copy defaults for name in buffer_options: - if not name in options: + if name not in options: try: options[name] = defaults[name] except KeyError: @@ -298,9 +298,10 @@ def put_unpack_buffer_aux_into_scope(buf_entry, code): ln = [] for i in range(buf_entry.type.ndim): for fldname in fldnames: - ln.append("%s.diminfo[%d].%s = %s.rcbuffer->pybuffer.%s[%d];" % \ - (pybuffernd_struct, i, fldname, - pybuffernd_struct, fldname, i)) + ln.append("%s.diminfo[%d].%s = %s.rcbuffer->pybuffer.%s[%d];" % ( + pybuffernd_struct, i, fldname, + pybuffernd_struct, fldname, i, + )) code.putln(' '.join(ln)) def put_init_vars(entry, code): @@ -373,7 +374,7 @@ def put_assign_to_buffer(lhs_cname, rhs_cname, buf_entry, code.putln("{") # Set up necessary stack for getbuffer code.putln("__Pyx_BufFmt_StackElem __pyx_stack[%d];" % buffer_type.dtype.struct_nesting_depth()) - getbuffer = get_getbuffer_call(code, "%s", buffer_aux, buffer_type) # fill in object below + getbuffer = get_getbuffer_call(code, "%s", buffer_aux, buffer_type) # fill in object below if is_initialized: # Release any existing buffer @@ -419,7 +420,7 @@ def put_assign_to_buffer(lhs_cname, rhs_cname, buf_entry, put_unpack_buffer_aux_into_scope(buf_entry, code) code.putln('}') - code.putln("}") # Release stack + code.putln("}") # Release stack def put_buffer_lookup_code(entry, index_signeds, index_cnames, directives, @@ -669,17 +670,25 @@ def get_type_information_cname(code, dtype, maxdepth=None): structinfo_name = "NULL" elif dtype.is_struct: struct_scope = dtype.scope - if dtype.is_const: - struct_scope = struct_scope.const_base_type_scope + if dtype.is_cv_qualified: + struct_scope = struct_scope.base_type_scope # Must pre-call all used types in order not to recurse during utility code writing. fields = struct_scope.var_entries assert len(fields) > 0 types = [get_type_information_cname(code, f.type, maxdepth - 1) for f in fields] typecode.putln("static __Pyx_StructField %s[] = {" % structinfo_name, safe=True) + + if dtype.is_cv_qualified: + # roughly speaking, remove "const" from struct_type + struct_type = dtype.cv_base_type.empty_declaration_code() + else: + struct_type = dtype.empty_declaration_code() + for f, typeinfo in zip(fields, types): typecode.putln(' {&%s, "%s", offsetof(%s, %s)},' % - (typeinfo, f.name, dtype.empty_declaration_code(), f.cname), safe=True) + (typeinfo, f.name, struct_type, f.cname), safe=True) + typecode.putln(' {NULL, NULL, 0}', safe=True) typecode.putln("};", safe=True) else: @@ -690,10 +699,10 @@ def get_type_information_cname(code, dtype, maxdepth=None): flags = "0" is_unsigned = "0" if dtype is PyrexTypes.c_char_type: - is_unsigned = "IS_UNSIGNED(%s)" % declcode + is_unsigned = "__PYX_IS_UNSIGNED(%s)" % declcode typegroup = "'H'" elif dtype.is_int: - is_unsigned = "IS_UNSIGNED(%s)" % declcode + is_unsigned = "__PYX_IS_UNSIGNED(%s)" % declcode typegroup = "%s ? 'U' : 'I'" % is_unsigned elif complex_possible or dtype.is_complex: typegroup = "'C'" diff --git a/Cython/Compiler/Builtin.py b/Cython/Compiler/Builtin.py index e0d203ae0..78b2638d5 100644 --- a/Cython/Compiler/Builtin.py +++ b/Cython/Compiler/Builtin.py @@ -4,11 +4,11 @@ from __future__ import absolute_import -from .Symtab import BuiltinScope, StructOrUnionScope -from .Code import UtilityCode +from .StringEncoding import EncodedString +from .Symtab import BuiltinScope, StructOrUnionScope, ModuleScope, Entry +from .Code import UtilityCode, TempitaUtilityCode from .TypeSlots import Signature from . import PyrexTypes -from . import Options # C-level implementations of builtin types, functions and methods @@ -30,17 +30,19 @@ builtin_utility_code = { class _BuiltinOverride(object): def __init__(self, py_name, args, ret_type, cname, py_equiv="*", utility_code=None, sig=None, func_type=None, - is_strict_signature=False, builtin_return_type=None): + is_strict_signature=False, builtin_return_type=None, + nogil=None): self.py_name, self.cname, self.py_equiv = py_name, cname, py_equiv self.args, self.ret_type = args, ret_type self.func_type, self.sig = func_type, sig self.builtin_return_type = builtin_return_type self.is_strict_signature = is_strict_signature self.utility_code = utility_code + self.nogil = nogil def build_func_type(self, sig=None, self_arg=None): if sig is None: - sig = Signature(self.args, self.ret_type) + sig = Signature(self.args, self.ret_type, nogil=self.nogil) sig.exception_check = False # not needed for the current builtins func_type = sig.function_type(self_arg) if self.is_strict_signature: @@ -54,7 +56,7 @@ class BuiltinAttribute(object): def __init__(self, py_name, cname=None, field_type=None, field_type_name=None): self.py_name = py_name self.cname = cname or py_name - self.field_type_name = field_type_name # can't do the lookup before the type is declared! + self.field_type_name = field_type_name # can't do the lookup before the type is declared! self.field_type = field_type def declare_in_type(self, self_type): @@ -89,16 +91,38 @@ class BuiltinMethod(_BuiltinOverride): self.py_name, method_type, self.cname, utility_code=self.utility_code) +class BuiltinProperty(object): + # read only for now + def __init__(self, py_name, property_type, call_cname, + exception_value=None, exception_check=None, utility_code=None): + self.py_name = py_name + self.property_type = property_type + self.call_cname = call_cname + self.utility_code = utility_code + self.exception_value = exception_value + self.exception_check = exception_check + + def declare_in_type(self, self_type): + self_type.scope.declare_cproperty( + self.py_name, + self.property_type, + self.call_cname, + exception_value=self.exception_value, + exception_check=self.exception_check, + utility_code=self.utility_code + ) + + builtin_function_table = [ # name, args, return, C API func, py equiv = "*" BuiltinFunction('abs', "d", "d", "fabs", - is_strict_signature = True), + is_strict_signature=True, nogil=True), BuiltinFunction('abs', "f", "f", "fabsf", - is_strict_signature = True), + is_strict_signature=True, nogil=True), BuiltinFunction('abs', "i", "i", "abs", - is_strict_signature = True), + is_strict_signature=True, nogil=True), BuiltinFunction('abs', "l", "l", "labs", - is_strict_signature = True), + is_strict_signature=True, nogil=True), BuiltinFunction('abs', None, None, "__Pyx_abs_longlong", utility_code = UtilityCode.load("abs_longlong", "Builtins.c"), func_type = PyrexTypes.CFuncType( @@ -209,7 +233,7 @@ builtin_function_table = [ #('sum', "", "", ""), #('sorted', "", "", ""), #('type', "O", "O", "PyObject_Type"), - #('unichr', "", "", ""), + BuiltinFunction('unichr', "i", "O", "PyUnicode_FromOrdinal", builtin_return_type='unicode'), #('unicode', "", "", ""), #('vars', "", "", ""), #('zip', "", "", ""), @@ -268,21 +292,33 @@ builtin_types_table = [ ("basestring", "PyBaseString_Type", [ BuiltinMethod("join", "TO", "T", "__Pyx_PyBaseString_Join", utility_code=UtilityCode.load("StringJoin", "StringTools.c")), + BuiltinMethod("__mul__", "Tz", "T", "__Pyx_PySequence_Multiply", + utility_code=UtilityCode.load("PySequenceMultiply", "ObjectHandling.c")), ]), ("bytearray", "PyByteArray_Type", [ + BuiltinMethod("__mul__", "Tz", "T", "__Pyx_PySequence_Multiply", + utility_code=UtilityCode.load("PySequenceMultiply", "ObjectHandling.c")), ]), ("bytes", "PyBytes_Type", [BuiltinMethod("join", "TO", "O", "__Pyx_PyBytes_Join", utility_code=UtilityCode.load("StringJoin", "StringTools.c")), + BuiltinMethod("__mul__", "Tz", "T", "__Pyx_PySequence_Multiply", + utility_code=UtilityCode.load("PySequenceMultiply", "ObjectHandling.c")), ]), ("str", "PyString_Type", [BuiltinMethod("join", "TO", "O", "__Pyx_PyString_Join", builtin_return_type='basestring', utility_code=UtilityCode.load("StringJoin", "StringTools.c")), + BuiltinMethod("__mul__", "Tz", "T", "__Pyx_PySequence_Multiply", + utility_code=UtilityCode.load("PySequenceMultiply", "ObjectHandling.c")), ]), ("unicode", "PyUnicode_Type", [BuiltinMethod("__contains__", "TO", "b", "PyUnicode_Contains"), BuiltinMethod("join", "TO", "T", "PyUnicode_Join"), + BuiltinMethod("__mul__", "Tz", "T", "__Pyx_PySequence_Multiply", + utility_code=UtilityCode.load("PySequenceMultiply", "ObjectHandling.c")), ]), - ("tuple", "PyTuple_Type", []), + ("tuple", "PyTuple_Type", [BuiltinMethod("__mul__", "Tz", "T", "__Pyx_PySequence_Multiply", + utility_code=UtilityCode.load("PySequenceMultiply", "ObjectHandling.c")), + ]), ("list", "PyList_Type", [BuiltinMethod("insert", "TzO", "r", "PyList_Insert"), BuiltinMethod("reverse", "T", "r", "PyList_Reverse"), @@ -290,6 +326,8 @@ builtin_types_table = [ utility_code=UtilityCode.load("ListAppend", "Optimize.c")), BuiltinMethod("extend", "TO", "r", "__Pyx_PyList_Extend", utility_code=UtilityCode.load("ListExtend", "Optimize.c")), + BuiltinMethod("__mul__", "Tz", "T", "__Pyx_PySequence_Multiply", + utility_code=UtilityCode.load("PySequenceMultiply", "ObjectHandling.c")), ]), ("dict", "PyDict_Type", [BuiltinMethod("__contains__", "TO", "b", "PyDict_Contains"), @@ -336,18 +374,45 @@ builtin_types_table = [ ("frozenset", "PyFrozenSet_Type", []), ("Exception", "((PyTypeObject*)PyExc_Exception)[0]", []), ("StopAsyncIteration", "((PyTypeObject*)__Pyx_PyExc_StopAsyncIteration)[0]", []), + ("memoryview", "PyMemoryView_Type", [ + # TODO - format would be nice, but hard to get + # __len__ can be accessed through a direct lookup of the buffer (but probably in Optimize.c) + # error checking would ideally be limited api only + BuiltinProperty("ndim", PyrexTypes.c_int_type, '__Pyx_PyMemoryView_Get_ndim', + exception_value="-1", exception_check=True, + utility_code=TempitaUtilityCode.load_cached( + "memoryview_get_from_buffer", "Builtins.c", + context=dict(name="ndim") + ) + ), + BuiltinProperty("readonly", PyrexTypes.c_bint_type, '__Pyx_PyMemoryView_Get_readonly', + exception_value="-1", exception_check=True, + utility_code=TempitaUtilityCode.load_cached( + "memoryview_get_from_buffer", "Builtins.c", + context=dict(name="readonly") + ) + ), + BuiltinProperty("itemsize", PyrexTypes.c_py_ssize_t_type, '__Pyx_PyMemoryView_Get_itemsize', + exception_value="-1", exception_check=True, + utility_code=TempitaUtilityCode.load_cached( + "memoryview_get_from_buffer", "Builtins.c", + context=dict(name="itemsize") + ) + )] + ) ] -types_that_construct_their_instance = set([ +types_that_construct_their_instance = frozenset({ # some builtin types do not always return an instance of # themselves - these do: 'type', 'bool', 'long', 'float', 'complex', 'bytes', 'unicode', 'bytearray', - 'tuple', 'list', 'dict', 'set', 'frozenset' + 'tuple', 'list', 'dict', 'set', 'frozenset', # 'str', # only in Py3.x # 'file', # only in Py2.x -]) + 'memoryview' +}) builtin_structs_table = [ @@ -397,7 +462,13 @@ def init_builtin_types(): objstruct_cname = "PyBaseExceptionObject" else: objstruct_cname = 'Py%sObject' % name.capitalize() - the_type = builtin_scope.declare_builtin_type(name, cname, utility, objstruct_cname) + type_class = PyrexTypes.BuiltinObjectType + if name in ['dict', 'list', 'set', 'frozenset']: + type_class = PyrexTypes.BuiltinTypeConstructorObjectType + elif name == 'tuple': + type_class = PyrexTypes.PythonTupleTypeConstructor + the_type = builtin_scope.declare_builtin_type(name, cname, utility, objstruct_cname, + type_class=type_class) builtin_types[name] = the_type for method in methods: method.declare_in_type(the_type) @@ -413,17 +484,20 @@ def init_builtin_structs(): def init_builtins(): + #Errors.init_thread() # hopefully not needed - we should not emit warnings ourselves init_builtin_structs() init_builtin_types() init_builtin_funcs() builtin_scope.declare_var( '__debug__', PyrexTypes.c_const_type(PyrexTypes.c_bint_type), - pos=None, cname='(!Py_OptimizeFlag)', is_cdef=True) + pos=None, cname='__pyx_assertions_enabled()', is_cdef=True) - global list_type, tuple_type, dict_type, set_type, frozenset_type - global bytes_type, str_type, unicode_type, basestring_type, slice_type - global float_type, bool_type, type_type, complex_type, bytearray_type + global type_type, list_type, tuple_type, dict_type, set_type, frozenset_type, slice_type + global bytes_type, str_type, unicode_type, basestring_type, bytearray_type + global float_type, int_type, long_type, bool_type, complex_type + global memoryview_type, py_buffer_type + global sequence_types type_type = builtin_scope.lookup('type').type list_type = builtin_scope.lookup('list').type tuple_type = builtin_scope.lookup('tuple').type @@ -431,14 +505,107 @@ def init_builtins(): set_type = builtin_scope.lookup('set').type frozenset_type = builtin_scope.lookup('frozenset').type slice_type = builtin_scope.lookup('slice').type + bytes_type = builtin_scope.lookup('bytes').type str_type = builtin_scope.lookup('str').type unicode_type = builtin_scope.lookup('unicode').type basestring_type = builtin_scope.lookup('basestring').type bytearray_type = builtin_scope.lookup('bytearray').type + memoryview_type = builtin_scope.lookup('memoryview').type + float_type = builtin_scope.lookup('float').type + int_type = builtin_scope.lookup('int').type + long_type = builtin_scope.lookup('long').type bool_type = builtin_scope.lookup('bool').type complex_type = builtin_scope.lookup('complex').type + sequence_types = ( + list_type, + tuple_type, + bytes_type, + str_type, + unicode_type, + basestring_type, + bytearray_type, + memoryview_type, + ) + + # Set up type inference links between equivalent Python/C types + bool_type.equivalent_type = PyrexTypes.c_bint_type + PyrexTypes.c_bint_type.equivalent_type = bool_type + + float_type.equivalent_type = PyrexTypes.c_double_type + PyrexTypes.c_double_type.equivalent_type = float_type + + complex_type.equivalent_type = PyrexTypes.c_double_complex_type + PyrexTypes.c_double_complex_type.equivalent_type = complex_type + + py_buffer_type = builtin_scope.lookup('Py_buffer').type + init_builtins() + +############################## +# Support for a few standard library modules that Cython understands (currently typing and dataclasses) +############################## +_known_module_scopes = {} + +def get_known_standard_library_module_scope(module_name): + mod = _known_module_scopes.get(module_name) + if mod: + return mod + + if module_name == "typing": + mod = ModuleScope(module_name, None, None) + for name, tp in [ + ('Dict', dict_type), + ('List', list_type), + ('Tuple', tuple_type), + ('Set', set_type), + ('FrozenSet', frozenset_type), + ]: + name = EncodedString(name) + entry = mod.declare_type(name, tp, pos = None) + var_entry = Entry(name, None, PyrexTypes.py_object_type) + var_entry.is_pyglobal = True + var_entry.is_variable = True + var_entry.scope = mod + entry.as_variable = var_entry + + for name in ['ClassVar', 'Optional']: + name = EncodedString(name) + indexed_type = PyrexTypes.SpecialPythonTypeConstructor(EncodedString("typing."+name)) + entry = mod.declare_type(name, indexed_type, pos = None) + var_entry = Entry(name, None, PyrexTypes.py_object_type) + var_entry.is_pyglobal = True + var_entry.is_variable = True + var_entry.scope = mod + entry.as_variable = var_entry + _known_module_scopes[module_name] = mod + elif module_name == "dataclasses": + mod = ModuleScope(module_name, None, None) + indexed_type = PyrexTypes.SpecialPythonTypeConstructor(EncodedString("dataclasses.InitVar")) + initvar_string = EncodedString("InitVar") + entry = mod.declare_type(initvar_string, indexed_type, pos = None) + var_entry = Entry(initvar_string, None, PyrexTypes.py_object_type) + var_entry.is_pyglobal = True + var_entry.scope = mod + entry.as_variable = var_entry + _known_module_scopes[module_name] = mod + return mod + + +def get_known_standard_library_entry(qualified_name): + name_parts = qualified_name.split(".") + module_name = EncodedString(name_parts[0]) + rest = name_parts[1:] + + if len(rest) > 1: # for now, we don't know how to deal with any nested modules + return None + + mod = get_known_standard_library_module_scope(module_name) + + # eventually handle more sophisticated multiple lookups if needed + if mod and rest: + return mod.lookup_here(rest[0]) + return None diff --git a/Cython/Compiler/CmdLine.py b/Cython/Compiler/CmdLine.py index 470fe6bd4..776636c32 100644 --- a/Cython/Compiler/CmdLine.py +++ b/Cython/Compiler/CmdLine.py @@ -4,237 +4,248 @@ from __future__ import absolute_import -import os import sys +import os +from argparse import ArgumentParser, Action, SUPPRESS from . import Options -usage = """\ -Cython (http://cython.org) is a compiler for code written in the -Cython language. Cython is based on Pyrex by Greg Ewing. - -Usage: cython [options] sourcefile.{pyx,py} ... - -Options: - -V, --version Display version number of cython compiler - -l, --create-listing Write error messages to a listing file - -I, --include-dir <directory> Search for include files in named directory - (multiple include directories are allowed). - -o, --output-file <filename> Specify name of generated C file - -t, --timestamps Only compile newer source files - -f, --force Compile all source files (overrides implied -t) - -v, --verbose Be verbose, print file names on multiple compilation - -p, --embed-positions If specified, the positions in Cython files of each - function definition is embedded in its docstring. - --cleanup <level> Release interned objects on python exit, for memory debugging. - Level indicates aggressiveness, default 0 releases nothing. - -w, --working <directory> Sets the working directory for Cython (the directory modules - are searched from) - --gdb Output debug information for cygdb - --gdb-outdir <directory> Specify gdb debug information output directory. Implies --gdb. - - -D, --no-docstrings Strip docstrings from the compiled module. - -a, --annotate Produce a colorized HTML version of the source. - --annotate-coverage <cov.xml> Annotate and include coverage information from cov.xml. - --line-directives Produce #line directives pointing to the .pyx source - --cplus Output a C++ rather than C file. - --embed[=<method_name>] Generate a main() function that embeds the Python interpreter. - -2 Compile based on Python-2 syntax and code semantics. - -3 Compile based on Python-3 syntax and code semantics. - --3str Compile based on Python-3 syntax and code semantics without - assuming unicode by default for string literals under Python 2. - --lenient Change some compile time errors to runtime errors to - improve Python compatibility - --capi-reexport-cincludes Add cincluded headers to any auto-generated header files. - --fast-fail Abort the compilation on the first error - --warning-errors, -Werror Make all warnings into errors - --warning-extra, -Wextra Enable extra warnings - -X, --directive <name>=<value>[,<name=value,...] Overrides a compiler directive - -E, --compile-time-env name=value[,<name=value,...] Provides compile time env like DEF would do. - --module-name Fully qualified module name. If not given, it is deduced from the - import path if source file is in a package, or equals the - filename otherwise. - -M, --depfile Produce depfiles for the sources -""" - - -# The following experimental options are supported only on MacOSX: -# -C, --compile Compile generated .c file to .o file -# --link Link .o file to produce extension module (implies -C) -# -+, --cplus Use C++ compiler for compiling and linking -# Additional .o files to link may be supplied when using -X.""" - -def bad_usage(): - sys.stderr.write(usage) - sys.exit(1) -def parse_command_line(args): - from .Main import CompilationOptions, default_options - - pending_arg = [] - - def pop_arg(): - if not args or pending_arg: - bad_usage() - if '=' in args[0] and args[0].startswith('--'): # allow "--long-option=xyz" - name, value = args.pop(0).split('=', 1) - pending_arg.append(value) - return name - return args.pop(0) - - def pop_value(default=None): - if pending_arg: - return pending_arg.pop() - elif default is not None: - return default - elif not args: - bad_usage() - return args.pop(0) - - def get_param(option): - tail = option[2:] - if tail: - return tail - else: - return pop_arg() - - options = CompilationOptions(default_options) - sources = [] - while args: - if args[0].startswith("-"): - option = pop_arg() - if option in ("-V", "--version"): - options.show_version = 1 - elif option in ("-l", "--create-listing"): - options.use_listing_file = 1 - elif option in ("-+", "--cplus"): - options.cplus = 1 - elif option == "--embed": - Options.embed = pop_value("main") - elif option.startswith("-I"): - options.include_path.append(get_param(option)) - elif option == "--include-dir": - options.include_path.append(pop_value()) - elif option in ("-w", "--working"): - options.working_path = pop_value() - elif option in ("-o", "--output-file"): - options.output_file = pop_value() - elif option in ("-t", "--timestamps"): - options.timestamps = 1 - elif option in ("-f", "--force"): - options.timestamps = 0 - elif option in ("-v", "--verbose"): - options.verbose += 1 - elif option in ("-p", "--embed-positions"): - Options.embed_pos_in_docstring = 1 - elif option in ("-z", "--pre-import"): - Options.pre_import = pop_value() - elif option == "--cleanup": - Options.generate_cleanup_code = int(pop_value()) - elif option in ("-D", "--no-docstrings"): - Options.docstrings = False - elif option in ("-a", "--annotate"): - Options.annotate = True - elif option == "--annotate-coverage": - Options.annotate = True - Options.annotate_coverage_xml = pop_value() - elif option == "--convert-range": - Options.convert_range = True - elif option == "--line-directives": - options.emit_linenums = True - elif option == "--no-c-in-traceback": - options.c_line_in_traceback = False - elif option == "--gdb": - options.gdb_debug = True - options.output_dir = os.curdir - elif option == "--gdb-outdir": - options.gdb_debug = True - options.output_dir = pop_value() - elif option == "--lenient": - Options.error_on_unknown_names = False - Options.error_on_uninitialized = False - elif option == '-2': - options.language_level = 2 - elif option == '-3': - options.language_level = 3 - elif option == '--3str': - options.language_level = '3str' - elif option == "--capi-reexport-cincludes": - options.capi_reexport_cincludes = True - elif option == "--fast-fail": - Options.fast_fail = True - elif option == "--cimport-from-pyx": - Options.cimport_from_pyx = True - elif option in ('-Werror', '--warning-errors'): - Options.warning_errors = True - elif option in ('-Wextra', '--warning-extra'): - options.compiler_directives.update(Options.extra_warnings) - elif option == "--old-style-globals": - Options.old_style_globals = True - elif option == "--directive" or option.startswith('-X'): - if option.startswith('-X') and option[2:].strip(): - x_args = option[2:] - else: - x_args = pop_value() - try: - options.compiler_directives = Options.parse_directive_list( - x_args, relaxed_bool=True, - current_settings=options.compiler_directives) - except ValueError as e: - sys.stderr.write("Error in compiler directive: %s\n" % e.args[0]) - sys.exit(1) - elif option == "--compile-time-env" or option.startswith('-E'): - if option.startswith('-E') and option[2:].strip(): - x_args = option[2:] - else: - x_args = pop_value() - try: - options.compile_time_env = Options.parse_compile_time_env( - x_args, current_settings=options.compile_time_env) - except ValueError as e: - sys.stderr.write("Error in compile-time-env: %s\n" % e.args[0]) - sys.exit(1) - elif option == "--module-name": - options.module_name = pop_value() - elif option in ('-M', '--depfile'): - options.depfile = True - elif option.startswith('--debug'): - option = option[2:].replace('-', '_') - from . import DebugFlags - if option in dir(DebugFlags): - setattr(DebugFlags, option, True) - else: - sys.stderr.write("Unknown debug flag: %s\n" % option) - bad_usage() - elif option in ('-h', '--help'): - sys.stdout.write(usage) - sys.exit(0) +if sys.version_info < (3, 3): + # TODO: This workaround can be removed in Cython 3.1 + FileNotFoundError = IOError + + +class ParseDirectivesAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + old_directives = dict(getattr(namespace, self.dest, + Options.get_directive_defaults())) + directives = Options.parse_directive_list( + values, relaxed_bool=True, current_settings=old_directives) + setattr(namespace, self.dest, directives) + + +class ParseOptionsAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + options = dict(getattr(namespace, self.dest, {})) + for opt in values.split(','): + if '=' in opt: + n, v = opt.split('=', 1) + v = v.lower() not in ('false', 'f', '0', 'no') else: - sys.stderr.write(usage) - sys.stderr.write("Unknown compiler flag: %s\n" % option) - sys.exit(1) + n, v = opt, True + options[n] = v + setattr(namespace, self.dest, options) + + +class ParseCompileTimeEnvAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + old_env = dict(getattr(namespace, self.dest, {})) + new_env = Options.parse_compile_time_env(values, current_settings=old_env) + setattr(namespace, self.dest, new_env) + + +class ActivateAllWarningsAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + directives = getattr(namespace, 'compiler_directives', {}) + directives.update(Options.extra_warnings) + namespace.compiler_directives = directives + + +class SetLenientAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + namespace.error_on_unknown_names = False + namespace.error_on_uninitialized = False + + +class SetGDBDebugAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + namespace.gdb_debug = True + namespace.output_dir = os.curdir + + +class SetGDBDebugOutputAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + namespace.gdb_debug = True + namespace.output_dir = values + + +class SetAnnotateCoverageAction(Action): + def __call__(self, parser, namespace, values, option_string=None): + namespace.annotate = True + namespace.annotate_coverage_xml = values + + +def create_cython_argparser(): + description = "Cython (https://cython.org/) is a compiler for code written in the "\ + "Cython language. Cython is based on Pyrex by Greg Ewing." + + parser = ArgumentParser(description=description, argument_default=SUPPRESS) + + parser.add_argument("-V", "--version", dest='show_version', action='store_const', const=1, + help='Display version number of cython compiler') + parser.add_argument("-l", "--create-listing", dest='use_listing_file', action='store_const', const=1, + help='Write error messages to a listing file') + parser.add_argument("-I", "--include-dir", dest='include_path', action='append', + help='Search for include files in named directory ' + '(multiple include directories are allowed).') + parser.add_argument("-o", "--output-file", dest='output_file', action='store', type=str, + help='Specify name of generated C file') + parser.add_argument("-t", "--timestamps", dest='timestamps', action='store_const', const=1, + help='Only compile newer source files') + parser.add_argument("-f", "--force", dest='timestamps', action='store_const', const=0, + help='Compile all source files (overrides implied -t)') + parser.add_argument("-v", "--verbose", dest='verbose', action='count', + help='Be verbose, print file names on multiple compilation') + parser.add_argument("-p", "--embed-positions", dest='embed_pos_in_docstring', action='store_const', const=1, + help='If specified, the positions in Cython files of each ' + 'function definition is embedded in its docstring.') + parser.add_argument("--cleanup", dest='generate_cleanup_code', action='store', type=int, + help='Release interned objects on python exit, for memory debugging. ' + 'Level indicates aggressiveness, default 0 releases nothing.') + parser.add_argument("-w", "--working", dest='working_path', action='store', type=str, + help='Sets the working directory for Cython (the directory modules are searched from)') + parser.add_argument("--gdb", action=SetGDBDebugAction, nargs=0, + help='Output debug information for cygdb') + parser.add_argument("--gdb-outdir", action=SetGDBDebugOutputAction, type=str, + help='Specify gdb debug information output directory. Implies --gdb.') + parser.add_argument("-D", "--no-docstrings", dest='docstrings', action='store_false', + help='Strip docstrings from the compiled module.') + parser.add_argument('-a', '--annotate', action='store_const', const='default', dest='annotate', + help='Produce a colorized HTML version of the source.') + parser.add_argument('--annotate-fullc', action='store_const', const='fullc', dest='annotate', + help='Produce a colorized HTML version of the source ' + 'which includes entire generated C/C++-code.') + parser.add_argument("--annotate-coverage", dest='annotate_coverage_xml', action=SetAnnotateCoverageAction, type=str, + help='Annotate and include coverage information from cov.xml.') + parser.add_argument("--line-directives", dest='emit_linenums', action='store_true', + help='Produce #line directives pointing to the .pyx source') + parser.add_argument("-+", "--cplus", dest='cplus', action='store_const', const=1, + help='Output a C++ rather than C file.') + parser.add_argument('--embed', action='store_const', const='main', + help='Generate a main() function that embeds the Python interpreter. ' + 'Pass --embed=<method_name> for a name other than main().') + parser.add_argument('-2', dest='language_level', action='store_const', const=2, + help='Compile based on Python-2 syntax and code semantics.') + parser.add_argument('-3', dest='language_level', action='store_const', const=3, + help='Compile based on Python-3 syntax and code semantics.') + parser.add_argument('--3str', dest='language_level', action='store_const', const='3str', + help='Compile based on Python-3 syntax and code semantics without ' + 'assuming unicode by default for string literals under Python 2.') + parser.add_argument("--lenient", action=SetLenientAction, nargs=0, + help='Change some compile time errors to runtime errors to ' + 'improve Python compatibility') + parser.add_argument("--capi-reexport-cincludes", dest='capi_reexport_cincludes', action='store_true', + help='Add cincluded headers to any auto-generated header files.') + parser.add_argument("--fast-fail", dest='fast_fail', action='store_true', + help='Abort the compilation on the first error') + parser.add_argument("-Werror", "--warning-errors", dest='warning_errors', action='store_true', + help='Make all warnings into errors') + parser.add_argument("-Wextra", "--warning-extra", action=ActivateAllWarningsAction, nargs=0, + help='Enable extra warnings') + + parser.add_argument('-X', '--directive', metavar='NAME=VALUE,...', + dest='compiler_directives', type=str, + action=ParseDirectivesAction, + help='Overrides a compiler directive') + parser.add_argument('-E', '--compile-time-env', metavar='NAME=VALUE,...', + dest='compile_time_env', type=str, + action=ParseCompileTimeEnvAction, + help='Provides compile time env like DEF would do.') + parser.add_argument("--module-name", + dest='module_name', type=str, action='store', + help='Fully qualified module name. If not given, is ' + 'deduced from the import path if source file is in ' + 'a package, or equals the filename otherwise.') + parser.add_argument('-M', '--depfile', action='store_true', help='produce depfiles for the sources') + parser.add_argument('sources', nargs='*', default=[]) + + # TODO: add help + parser.add_argument("-z", "--pre-import", dest='pre_import', action='store', type=str, help=SUPPRESS) + parser.add_argument("--convert-range", dest='convert_range', action='store_true', help=SUPPRESS) + parser.add_argument("--no-c-in-traceback", dest='c_line_in_traceback', action='store_false', help=SUPPRESS) + parser.add_argument("--cimport-from-pyx", dest='cimport_from_pyx', action='store_true', help=SUPPRESS) + parser.add_argument("--old-style-globals", dest='old_style_globals', action='store_true', help=SUPPRESS) + + # debug stuff: + from . import DebugFlags + for name in vars(DebugFlags): + if name.startswith("debug"): + option_name = name.replace('_', '-') + parser.add_argument("--" + option_name, action='store_true', help=SUPPRESS) + + return parser + + +def parse_command_line_raw(parser, args): + # special handling for --embed and --embed=xxxx as they aren't correctly parsed + def filter_out_embed_options(args): + with_embed, without_embed = [], [] + for x in args: + if x == '--embed' or x.startswith('--embed='): + with_embed.append(x) + else: + without_embed.append(x) + return with_embed, without_embed + + with_embed, args_without_embed = filter_out_embed_options(args) + + arguments, unknown = parser.parse_known_args(args_without_embed) + + sources = arguments.sources + del arguments.sources + + # unknown can be either debug, embed or input files or really unknown + for option in unknown: + if option.startswith('-'): + parser.error("unknown option " + option) + else: + sources.append(option) + + # embed-stuff must be handled extra: + for x in with_embed: + if x == '--embed': + name = 'main' # default value else: - sources.append(pop_arg()) + name = x[len('--embed='):] + setattr(arguments, 'embed', name) - if pending_arg: - bad_usage() + return arguments, sources + + +def parse_command_line(args): + parser = create_cython_argparser() + arguments, sources = parse_command_line_raw(parser, args) + + work_dir = getattr(arguments, 'working_path', '') + for source in sources: + if work_dir and not os.path.isabs(source): + source = os.path.join(work_dir, source) + if not os.path.exists(source): + import errno + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), source) + + options = Options.CompilationOptions(Options.default_options) + for name, value in vars(arguments).items(): + if name.startswith('debug'): + from . import DebugFlags + if name in dir(DebugFlags): + setattr(DebugFlags, name, value) + else: + parser.error("Unknown debug flag: %s\n" % name) + elif hasattr(Options, name): + setattr(Options, name, value) + else: + setattr(options, name, value) if options.use_listing_file and len(sources) > 1: - sys.stderr.write( - "cython: Only one source file allowed when using -o\n") - sys.exit(1) + parser.error("cython: Only one source file allowed when using -o\n") if len(sources) == 0 and not options.show_version: - bad_usage() + parser.error("cython: Need at least one source file\n") if Options.embed and len(sources) > 1: - sys.stderr.write( - "cython: Only one source file allowed when using --embed\n") - sys.exit(1) + parser.error("cython: Only one source file allowed when using --embed\n") if options.module_name: if options.timestamps: - sys.stderr.write( - "cython: Cannot use --module-name with --timestamps\n") - sys.exit(1) + parser.error("cython: Cannot use --module-name with --timestamps\n") if len(sources) > 1: - sys.stderr.write( - "cython: Only one source file allowed when using --module-name\n") - sys.exit(1) + parser.error("cython: Only one source file allowed when using --module-name\n") return options, sources diff --git a/Cython/Compiler/Code.pxd b/Cython/Compiler/Code.pxd index acad0c1cf..4601474b2 100644 --- a/Cython/Compiler/Code.pxd +++ b/Cython/Compiler/Code.pxd @@ -1,5 +1,4 @@ - -from __future__ import absolute_import +# cython: language_level=3 cimport cython from ..StringIOTree cimport StringIOTree @@ -55,6 +54,7 @@ cdef class FunctionState: cdef public object closure_temps cdef public bint should_declare_error_indicator cdef public bint uses_error_indicator + cdef public bint error_without_exception @cython.locals(n=size_t) cpdef new_label(self, name=*) @@ -110,6 +110,9 @@ cdef class CCodeWriter(object): cdef bint bol cpdef write(self, s) + @cython.final + cdef _write_lines(self, s) + cpdef _write_to_buffer(self, s) cpdef put(self, code) cpdef put_safe(self, code) cpdef putln(self, code=*, bint safe=*) @@ -117,6 +120,8 @@ cdef class CCodeWriter(object): cdef increase_indent(self) @cython.final cdef decrease_indent(self) + @cython.final + cdef indent(self) cdef class PyrexCodeWriter: diff --git a/Cython/Compiler/Code.py b/Cython/Compiler/Code.py index d0b4756e5..089685496 100644 --- a/Cython/Compiler/Code.py +++ b/Cython/Compiler/Code.py @@ -1,4 +1,4 @@ -# cython: language_level = 2 +# cython: language_level=3str # cython: auto_pickle=False # # Code output module @@ -13,27 +13,21 @@ cython.declare(os=object, re=object, operator=object, textwrap=object, DebugFlags=object, basestring=object, defaultdict=object, closing=object, partial=object) +import hashlib +import operator import os import re import shutil -import sys -import operator import textwrap from string import Template from functools import partial -from contextlib import closing +from contextlib import closing, contextmanager from collections import defaultdict -try: - import hashlib -except ImportError: - import md5 as hashlib - from . import Naming from . import Options from . import DebugFlags from . import StringEncoding -from . import Version from .. import Utils from .Scanning import SourceDescriptor from ..StringIOTree import StringIOTree @@ -43,8 +37,6 @@ try: except ImportError: from builtins import str as basestring -KEYWORDS_MUST_BE_BYTES = sys.version_info < (2, 7) - non_portable_builtins_map = { # builtins that have different names in different Python versions @@ -101,20 +93,18 @@ uncachable_builtins = [ '__build_class__', 'ascii', # might deserve an implementation in Cython #'exec', # implemented in Cython - ## - Py2.7+ - 'memoryview', ## - platform specific 'WindowsError', ## - others '_', # e.g. used by gettext ] -special_py_methods = set([ +special_py_methods = cython.declare(frozenset, frozenset(( '__cinit__', '__dealloc__', '__richcmp__', '__next__', '__await__', '__aiter__', '__anext__', '__getreadbuffer__', '__getwritebuffer__', '__getsegcount__', - '__getcharbuffer__', '__getbuffer__', '__releasebuffer__' -]) + '__getcharbuffer__', '__getbuffer__', '__releasebuffer__', +))) modifier_output_mapper = { 'inline': 'CYTHON_INLINE' @@ -203,6 +193,28 @@ def get_utility_dir(): Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) return os.path.join(Cython_dir, "Utility") +read_utilities_hook = None +""" +Override the hook for reading a utilities file that contains code fragments used +by the codegen. + +The hook functions takes the path of the utilities file, and returns a list +of strings, one per line. + +The default behavior is to open a file relative to get_utility_dir(). +""" + +def read_utilities_from_utility_dir(path): + """ + Read all lines of the file at the provided path from a path relative + to get_utility_dir(). + """ + filename = os.path.join(get_utility_dir(), path) + with closing(Utils.open_source_file(filename, encoding='UTF-8')) as f: + return f.readlines() + +# by default, read utilities from the utility directory. +read_utilities_hook = read_utilities_from_utility_dir class UtilityCodeBase(object): """ @@ -224,6 +236,15 @@ class UtilityCodeBase(object): [definitions] + ##### MyUtility ##### + #@subsitute: tempita + + [requires tempita substitution + - context can't be specified here though so only + tempita utility that requires no external context + will benefit from this tag + - only necessary when @required from non-tempita code] + for prototypes and implementation respectively. For non-python or -cython files backslashes should be used instead. 5 to 30 comment characters may be used on either side. @@ -242,8 +263,7 @@ class UtilityCodeBase(object): return code = '\n'.join(lines) - if tags and 'substitute' in tags and tags['substitute'] == set(['naming']): - del tags['substitute'] + if tags and 'substitute' in tags and 'naming' in tags['substitute']: try: code = Template(code).substitute(vars(Naming)) except (KeyError, ValueError) as e: @@ -259,15 +279,11 @@ class UtilityCodeBase(object): utility[1] = code else: all_tags = utility[2] - if KEYWORDS_MUST_BE_BYTES: - type = type.encode('ASCII') all_tags[type] = code if tags: all_tags = utility[2] for name, values in tags.items(): - if KEYWORDS_MUST_BE_BYTES: - name = name.encode('ASCII') all_tags.setdefault(name, set()).update(values) @classmethod @@ -276,7 +292,6 @@ class UtilityCodeBase(object): if utilities: return utilities - filename = os.path.join(get_utility_dir(), path) _, ext = os.path.splitext(path) if ext in ('.pyx', '.py', '.pxd', '.pxi'): comment = '#' @@ -292,8 +307,7 @@ class UtilityCodeBase(object): {'C': comment}).match match_type = re.compile(r'(.+)[.](proto(?:[.]\S+)?|impl|init|cleanup)$').match - with closing(Utils.open_source_file(filename, encoding='UTF-8')) as f: - all_lines = f.readlines() + all_lines = read_utilities_hook(path) utilities = defaultdict(lambda: [None, None, {}]) lines = [] @@ -335,43 +349,22 @@ class UtilityCodeBase(object): return utilities @classmethod - def load(cls, util_code_name, from_file=None, **kwargs): + def load(cls, util_code_name, from_file, **kwargs): """ Load utility code from a file specified by from_file (relative to - Cython/Utility) and name util_code_name. If from_file is not given, - load it from the file util_code_name.*. There should be only one - file matched by this pattern. + Cython/Utility) and name util_code_name. """ + if '::' in util_code_name: from_file, util_code_name = util_code_name.rsplit('::', 1) - if not from_file: - utility_dir = get_utility_dir() - prefix = util_code_name + '.' - try: - listing = os.listdir(utility_dir) - except OSError: - # XXX the code below assumes as 'zipimport.zipimporter' instance - # XXX should be easy to generalize, but too lazy right now to write it - import zipfile - global __loader__ - loader = __loader__ - archive = loader.archive - with closing(zipfile.ZipFile(archive)) as fileobj: - listing = [os.path.basename(name) - for name in fileobj.namelist() - if os.path.join(archive, name).startswith(utility_dir)] - files = [filename for filename in listing - if filename.startswith(prefix)] - if not files: - raise ValueError("No match found for utility code " + util_code_name) - if len(files) > 1: - raise ValueError("More than one filename match found for utility code " + util_code_name) - from_file = files[0] - + assert from_file utilities = cls.load_utilities_from_file(from_file) proto, impl, tags = utilities[util_code_name] if tags: + if "substitute" in tags and "tempita" in tags["substitute"]: + if not issubclass(cls, TempitaUtilityCode): + return TempitaUtilityCode.load(util_code_name, from_file, **kwargs) orig_kwargs = kwargs.copy() for name, values in tags.items(): if name in kwargs: @@ -385,6 +378,12 @@ class UtilityCodeBase(object): # dependencies are rarely unique, so use load_cached() when we can values = [cls.load_cached(dep, from_file) for dep in sorted(values)] + elif name == 'substitute': + # don't want to pass "naming" or "tempita" to the constructor + # since these will have been handled + values = values - {'naming', 'tempita'} + if not values: + continue elif not values: values = None elif len(values) == 1: @@ -404,11 +403,11 @@ class UtilityCodeBase(object): return cls(**kwargs) @classmethod - def load_cached(cls, utility_code_name, from_file=None, __cache={}): + def load_cached(cls, utility_code_name, from_file, __cache={}): """ Calls .load(), but using a per-type cache based on utility name and file name. """ - key = (cls, from_file, utility_code_name) + key = (utility_code_name, from_file, cls) try: return __cache[key] except KeyError: @@ -417,7 +416,7 @@ class UtilityCodeBase(object): return code @classmethod - def load_as_string(cls, util_code_name, from_file=None, **kwargs): + def load_as_string(cls, util_code_name, from_file, **kwargs): """ Load a utility code as a string. Returns (proto, implementation) """ @@ -437,7 +436,7 @@ class UtilityCodeBase(object): return "<%s(%s)>" % (type(self).__name__, self.name) def get_tree(self, **kwargs): - pass + return None def __deepcopy__(self, memodict=None): # No need to deep-copy utility code since it's essentially immutable. @@ -566,7 +565,7 @@ class UtilityCode(UtilityCodeBase): r'([a-zA-Z_]+),' # type cname r'\s*"([^"]+)",' # method name r'\s*([^),]+)' # object cname - r'((?:,\s*[^),]+)*)' # args* + r'((?:,[^),]+)*)' # args* r'\)', externalise, impl) assert 'CALL_UNBOUND_METHOD(' not in impl @@ -692,6 +691,7 @@ class LazyUtilityCode(UtilityCodeBase): class FunctionState(object): # return_label string function return point label # error_label string error catch point label + # error_without_exception boolean Can go to the error label without an exception (e.g. __next__ can return NULL) # continue_label string loop continue point label # break_label string loop break point label # return_from_error_cleanup_label string @@ -740,6 +740,8 @@ class FunctionState(object): self.should_declare_error_indicator = False self.uses_error_indicator = False + self.error_without_exception = False + # safety checks def validate_exit(self): @@ -770,9 +772,9 @@ class FunctionState(object): self.yield_labels.append(num_and_label) return num_and_label - def new_error_label(self): + def new_error_label(self, prefix=""): old_err_lbl = self.error_label - self.error_label = self.new_label('error') + self.error_label = self.new_label(prefix + 'error') return old_err_lbl def get_loop_labels(self): @@ -784,11 +786,11 @@ class FunctionState(object): (self.continue_label, self.break_label) = labels - def new_loop_labels(self): + def new_loop_labels(self, prefix=""): old_labels = self.get_loop_labels() self.set_loop_labels( - (self.new_label("continue"), - self.new_label("break"))) + (self.new_label(prefix + "continue"), + self.new_label(prefix + "break"))) return old_labels def get_all_labels(self): @@ -829,14 +831,14 @@ class FunctionState(object): allocated and released one of the same type). Type is simply registered and handed back, but will usually be a PyrexType. - If type.is_pyobject, manage_ref comes into play. If manage_ref is set to + If type.needs_refcounting, manage_ref comes into play. If manage_ref is set to True, the temp will be decref-ed on return statements and in exception handling clauses. Otherwise the caller has to deal with any reference counting of the variable. - If not type.is_pyobject, then manage_ref will be ignored, but it + If not type.needs_refcounting, then manage_ref will be ignored, but it still has to be passed. It is recommended to pass False by convention - if it is known that type will never be a Python object. + if it is known that type will never be a reference counted type. static=True marks the temporary declaration with "static". This is only used when allocating backing store for a module-level @@ -846,14 +848,16 @@ class FunctionState(object): A C string referring to the variable is returned. """ - if type.is_const and not type.is_reference: - type = type.const_base_type + if type.is_cv_qualified and not type.is_reference: + type = type.cv_base_type elif type.is_reference and not type.is_fake_reference: type = type.ref_base_type elif type.is_cfunction: from . import PyrexTypes type = PyrexTypes.c_ptr_type(type) # A function itself isn't an l-value - if not type.is_pyobject and not type.is_memoryviewslice: + elif type.is_cpp_class and not type.is_fake_reference and self.scope.directives['cpp_locals']: + self.scope.use_utility_code(UtilityCode.load_cached("OptionalLocals", "CppSupport.cpp")) + if not type.needs_refcounting: # Make manage_ref canonical, so that manage_ref will always mean # a decref is needed. manage_ref = False @@ -906,17 +910,17 @@ class FunctionState(object): for name, type, manage_ref, static in self.temps_allocated: freelist = self.temps_free.get((type, manage_ref)) if freelist is None or name not in freelist[1]: - used.append((name, type, manage_ref and type.is_pyobject)) + used.append((name, type, manage_ref and type.needs_refcounting)) return used def temps_holding_reference(self): """Return a list of (cname,type) tuples of temp names and their type - that are currently in use. This includes only temps of a - Python object type which owns its reference. + that are currently in use. This includes only temps + with a reference counted type which owns its reference. """ return [(name, type) for name, type, manage_ref in self.temps_in_use() - if manage_ref and type.is_pyobject] + if manage_ref and type.needs_refcounting] def all_managed_temps(self): """Return a list of (cname, type) tuples of refcount-managed Python objects. @@ -1121,10 +1125,10 @@ class GlobalState(object): 'h_code', 'filename_table', 'utility_code_proto_before_types', - 'numeric_typedefs', # Let these detailed individual parts stay!, - 'complex_type_declarations', # as the proper solution is to make a full DAG... - 'type_declarations', # More coarse-grained blocks would simply hide - 'utility_code_proto', # the ugliness, not fix it + 'numeric_typedefs', # Let these detailed individual parts stay!, + 'complex_type_declarations', # as the proper solution is to make a full DAG... + 'type_declarations', # More coarse-grained blocks would simply hide + 'utility_code_proto', # the ugliness, not fix it 'module_declarations', 'typeinfo', 'before_global_var', @@ -1132,19 +1136,34 @@ class GlobalState(object): 'string_decls', 'decls', 'late_includes', - 'all_the_rest', + 'module_state', + 'module_state_clear', + 'module_state_traverse', + 'module_state_defines', # redefines names used in module_state/_clear/_traverse + 'module_code', # user code goes here 'pystring_table', 'cached_builtins', 'cached_constants', - 'init_globals', + 'init_constants', + 'init_globals', # (utility code called at init-time) 'init_module', 'cleanup_globals', 'cleanup_module', 'main_method', + 'utility_code_pragmas', # silence some irrelevant warnings in utility code 'utility_code_def', + 'utility_code_pragmas_end', # clean-up the utility_code_pragmas 'end' ] + # h files can only have a much smaller list of sections + h_code_layout = [ + 'h_code', + 'utility_code_proto_before_types', + 'type_declarations', + 'utility_code_proto', + 'end' + ] def __init__(self, writer, module_node, code_config, common_utility_include_dir=None): self.filename_table = {} @@ -1156,8 +1175,8 @@ class GlobalState(object): self.code_config = code_config self.common_utility_include_dir = common_utility_include_dir self.parts = {} - self.module_node = module_node # because some utility code generation needs it - # (generating backwards-compatible Get/ReleaseBuffer + self.module_node = module_node # because some utility code generation needs it + # (generating backwards-compatible Get/ReleaseBuffer self.const_cnames_used = {} self.string_const_index = {} @@ -1173,8 +1192,10 @@ class GlobalState(object): def initialize_main_c_code(self): rootwriter = self.rootwriter - for part in self.code_layout: - self.parts[part] = rootwriter.insertion_point() + for i, part in enumerate(self.code_layout): + w = self.parts[part] = rootwriter.insertion_point() + if i > 0: + w.putln("/* #### Code section: %s ### */" % part) if not Options.cache_builtins: del self.parts['cached_builtins'] @@ -1188,13 +1209,18 @@ class GlobalState(object): w.putln("") w.putln("static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) {") w.put_declare_refcount_context() - w.put_setup_refcount_context("__Pyx_InitCachedConstants") + w.put_setup_refcount_context(StringEncoding.EncodedString("__Pyx_InitCachedConstants")) w = self.parts['init_globals'] w.enter_cfunc_scope() w.putln("") w.putln("static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void) {") + w = self.parts['init_constants'] + w.enter_cfunc_scope() + w.putln("") + w.putln("static CYTHON_SMALL_CODE int __Pyx_InitConstants(void) {") + if not Options.generate_cleanup_code: del self.parts['cleanup_globals'] else: @@ -1213,6 +1239,11 @@ class GlobalState(object): code.putln("") code.putln("/* --- Runtime support code --- */") + def initialize_main_h_code(self): + rootwriter = self.rootwriter + for part in self.h_code_layout: + self.parts[part] = rootwriter.insertion_point() + def finalize_main_c_code(self): self.close_global_decls() @@ -1224,6 +1255,18 @@ class GlobalState(object): code.put(util.format_code(util.impl)) code.putln("") + # + # utility code pragmas + # + code = self.parts['utility_code_pragmas'] + util = UtilityCode.load_cached("UtilityCodePragmas", "ModuleSetupCode.c") + code.putln(util.format_code(util.impl)) + code.putln("") + code = self.parts['utility_code_pragmas_end'] + util = UtilityCode.load_cached("UtilityCodePragmasEnd", "ModuleSetupCode.c") + code.putln(util.format_code(util.impl)) + code.putln("") + def __getitem__(self, key): return self.parts[key] @@ -1253,13 +1296,14 @@ class GlobalState(object): w.putln("}") w.exit_cfunc_scope() - w = self.parts['init_globals'] - w.putln("return 0;") - if w.label_used(w.error_label): - w.put_label(w.error_label) - w.putln("return -1;") - w.putln("}") - w.exit_cfunc_scope() + for part in ['init_globals', 'init_constants']: + w = self.parts[part] + w.putln("return 0;") + if w.label_used(w.error_label): + w.put_label(w.error_label) + w.putln("return -1;") + w.putln("}") + w.exit_cfunc_scope() if Options.generate_cleanup_code: w = self.parts['cleanup_globals'] @@ -1306,8 +1350,11 @@ class GlobalState(object): return const # create a new Python object constant const = self.new_py_const(type, prefix) - if cleanup_level is not None \ - and cleanup_level <= Options.generate_cleanup_code: + if (cleanup_level is not None + and cleanup_level <= Options.generate_cleanup_code + # Note that this function is used for all argument defaults + # which aren't just Python objects + and type.needs_refcounting): cleanup_writer = self.parts['cleanup_globals'] cleanup_writer.putln('Py_CLEAR(%s);' % const.cname) if dedup_key is not None: @@ -1376,23 +1423,33 @@ class GlobalState(object): value = bytes_value.decode('ASCII', 'ignore') return self.new_const_cname(value=value) - def new_num_const_cname(self, value, py_type): + def unique_const_cname(self, format_str): # type: (str) -> str + used = self.const_cnames_used + cname = value = format_str.format(sep='', counter='') + while cname in used: + counter = used[value] = used[value] + 1 + cname = format_str.format(sep='_', counter=counter) + used[cname] = 1 + return cname + + def new_num_const_cname(self, value, py_type): # type: (str, str) -> str if py_type == 'long': value += 'L' py_type = 'int' prefix = Naming.interned_prefixes[py_type] - cname = "%s%s" % (prefix, value) - cname = cname.replace('+', '_').replace('-', 'neg_').replace('.', '_') + + value = value.replace('.', '_').replace('+', '_').replace('-', 'neg_') + if len(value) > 42: + # update tests/run/large_integer_T5290.py in case the amount is changed + cname = self.unique_const_cname( + prefix + "large{counter}_" + value[:18] + "_xxx_" + value[-18:]) + else: + cname = "%s%s" % (prefix, value) return cname def new_const_cname(self, prefix='', value=''): value = replace_identifier('_', value)[:32].strip('_') - used = self.const_cnames_used - name_suffix = value - while name_suffix in used: - counter = used[value] = used[value] + 1 - name_suffix = '%s_%d' % (value, counter) - used[name_suffix] = 1 + name_suffix = self.unique_const_cname(value + "{sep}{counter}") if prefix: prefix = Naming.interned_prefixes[prefix] else: @@ -1460,26 +1517,38 @@ class GlobalState(object): consts = [(len(c.cname), c.cname, c) for c in self.py_constants] consts.sort() - decls_writer = self.parts['decls'] for _, cname, c in consts: - decls_writer.putln( - "static %s;" % c.type.declaration_code(cname)) + self.parts['module_state'].putln("%s;" % c.type.declaration_code(cname)) + self.parts['module_state_defines'].putln( + "#define %s %s->%s" % (cname, Naming.modulestateglobal_cname, cname)) + if not c.type.needs_refcounting: + # Note that py_constants is used for all argument defaults + # which aren't necessarily PyObjects, so aren't appropriate + # to clear. + continue + self.parts['module_state_clear'].putln( + "Py_CLEAR(clear_module_state->%s);" % cname) + self.parts['module_state_traverse'].putln( + "Py_VISIT(traverse_module_state->%s);" % cname) def generate_cached_methods_decls(self): if not self.cached_cmethods: return decl = self.parts['decls'] - init = self.parts['init_globals'] + init = self.parts['init_constants'] cnames = [] for (type_cname, method_name), cname in sorted(self.cached_cmethods.items()): cnames.append(cname) method_name_cname = self.get_interned_identifier(StringEncoding.EncodedString(method_name)).cname - decl.putln('static __Pyx_CachedCFunction %s = {0, &%s, 0, 0, 0};' % ( - cname, method_name_cname)) + decl.putln('static __Pyx_CachedCFunction %s = {0, 0, 0, 0, 0};' % ( + cname)) # split type reference storage as it might not be static init.putln('%s.type = (PyObject*)&%s;' % ( cname, type_cname)) + # method name string isn't static in limited api + init.putln('%s.method_name = &%s;' % ( + cname, method_name_cname)) if Options.generate_cleanup_code: cleanup = self.parts['cleanup_globals'] @@ -1517,13 +1586,18 @@ class GlobalState(object): decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf16_array)) decls_writer.putln("#endif") + init_constants = self.parts['init_constants'] if py_strings: self.use_utility_code(UtilityCode.load_cached("InitStrings", "StringTools.c")) py_strings.sort() w = self.parts['pystring_table'] w.putln("") - w.putln("static __Pyx_StringTabEntry %s[] = {" % Naming.stringtab_cname) - for c_cname, _, py_string in py_strings: + w.putln("static int __Pyx_CreateStringTabAndInitStrings(void) {") + # the stringtab is a function local rather than a global to + # ensure that it doesn't conflict with module state + w.putln("__Pyx_StringTabEntry %s[] = {" % Naming.stringtab_cname) + for py_string_args in py_strings: + c_cname, _, py_string = py_string_args if not py_string.is_str or not py_string.encoding or \ py_string.encoding in ('ASCII', 'USASCII', 'US-ASCII', 'UTF8', 'UTF-8'): @@ -1531,8 +1605,15 @@ class GlobalState(object): else: encoding = '"%s"' % py_string.encoding.lower() - decls_writer.putln( - "static PyObject *%s;" % py_string.cname) + self.parts['module_state'].putln("PyObject *%s;" % py_string.cname) + self.parts['module_state_defines'].putln("#define %s %s->%s" % ( + py_string.cname, + Naming.modulestateglobal_cname, + py_string.cname)) + self.parts['module_state_clear'].putln("Py_CLEAR(clear_module_state->%s);" % + py_string.cname) + self.parts['module_state_traverse'].putln("Py_VISIT(traverse_module_state->%s);" % + py_string.cname) if py_string.py3str_cstring: w.putln("#if PY_MAJOR_VERSION >= 3") w.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( @@ -1556,22 +1637,27 @@ class GlobalState(object): w.putln("#endif") w.putln("{0, 0, 0, 0, 0, 0, 0}") w.putln("};") + w.putln("return __Pyx_InitStrings(%s);" % Naming.stringtab_cname) + w.putln("}") - init_globals = self.parts['init_globals'] - init_globals.putln( - "if (__Pyx_InitStrings(%s) < 0) %s" % ( - Naming.stringtab_cname, - init_globals.error_goto(self.module_pos))) + init_constants.putln( + "if (__Pyx_CreateStringTabAndInitStrings() < 0) %s;" % + init_constants.error_goto(self.module_pos)) def generate_num_constants(self): consts = [(c.py_type, c.value[0] == '-', len(c.value), c.value, c.value_code, c) for c in self.num_const_index.values()] consts.sort() - decls_writer = self.parts['decls'] - init_globals = self.parts['init_globals'] + init_constants = self.parts['init_constants'] for py_type, _, _, value, value_code, c in consts: cname = c.cname - decls_writer.putln("static PyObject *%s;" % cname) + self.parts['module_state'].putln("PyObject *%s;" % cname) + self.parts['module_state_defines'].putln("#define %s %s->%s" % ( + cname, Naming.modulestateglobal_cname, cname)) + self.parts['module_state_clear'].putln( + "Py_CLEAR(clear_module_state->%s);" % cname) + self.parts['module_state_traverse'].putln( + "Py_VISIT(traverse_module_state->%s);" % cname) if py_type == 'float': function = 'PyFloat_FromDouble(%s)' elif py_type == 'long': @@ -1582,9 +1668,9 @@ class GlobalState(object): function = "PyInt_FromLong(%sL)" else: function = "PyInt_FromLong(%s)" - init_globals.putln('%s = %s; %s' % ( + init_constants.putln('%s = %s; %s' % ( cname, function % value_code, - init_globals.error_goto_if_null(cname, self.module_pos))) + init_constants.error_goto_if_null(cname, self.module_pos))) # The functions below are there in a transition phase only # and will be deprecated. They are called from Nodes.BlockNode. @@ -1759,10 +1845,21 @@ class CCodeWriter(object): return self.buffer.getvalue() def write(self, s): - # also put invalid markers (lineno 0), to indicate that those lines - # have no Cython source code correspondence - cython_lineno = self.last_marked_pos[1] if self.last_marked_pos else 0 - self.buffer.markers.extend([cython_lineno] * s.count('\n')) + if '\n' in s: + self._write_lines(s) + else: + self._write_to_buffer(s) + + def _write_lines(self, s): + # Cygdb needs to know which Cython source line corresponds to which C line. + # Therefore, we write this information into "self.buffer.markers" and then write it from there + # into cython_debug/cython_debug_info_* (see ModuleNode._serialize_lineno_map). + filename_line = self.last_marked_pos[:2] if self.last_marked_pos else (None, 0) + self.buffer.markers.extend([filename_line] * s.count('\n')) + + self._write_to_buffer(s) + + def _write_to_buffer(self, s): self.buffer.write(s) def insertion_point(self): @@ -1804,13 +1901,37 @@ class CCodeWriter(object): @funccontext_property def yield_labels(self): pass + def label_interceptor(self, new_labels, orig_labels, skip_to_label=None, pos=None, trace=True): + """ + Helper for generating multiple label interceptor code blocks. + + @param new_labels: the new labels that should be intercepted + @param orig_labels: the original labels that we should dispatch to after the interception + @param skip_to_label: a label to skip to before starting the code blocks + @param pos: the node position to mark for each interceptor block + @param trace: add a trace line for the pos marker or not + """ + for label, orig_label in zip(new_labels, orig_labels): + if not self.label_used(label): + continue + if skip_to_label: + # jump over the whole interception block + self.put_goto(skip_to_label) + skip_to_label = None + + if pos is not None: + self.mark_pos(pos, trace=trace) + self.put_label(label) + yield (label, orig_label) + self.put_goto(orig_label) + # Functions delegated to function scope def new_label(self, name=None): return self.funcstate.new_label(name) - def new_error_label(self): return self.funcstate.new_error_label() + def new_error_label(self, *args): return self.funcstate.new_error_label(*args) def new_yield_label(self, *args): return self.funcstate.new_yield_label(*args) def get_loop_labels(self): return self.funcstate.get_loop_labels() def set_loop_labels(self, labels): return self.funcstate.set_loop_labels(labels) - def new_loop_labels(self): return self.funcstate.new_loop_labels() + def new_loop_labels(self, *args): return self.funcstate.new_loop_labels(*args) def get_all_labels(self): return self.funcstate.get_all_labels() def set_all_labels(self, labels): return self.funcstate.set_all_labels(labels) def all_new_labels(self): return self.funcstate.all_new_labels() @@ -1822,6 +1943,7 @@ class CCodeWriter(object): self.funcstate = FunctionState(self, scope=scope) def exit_cfunc_scope(self): + self.funcstate.validate_exit() self.funcstate = None # constant handling @@ -1865,13 +1987,13 @@ class CCodeWriter(object): self.emit_marker() if self.code_config.emit_linenums and self.last_marked_pos: source_desc, line, _ = self.last_marked_pos - self.write('\n#line %s "%s"\n' % (line, source_desc.get_escaped_description())) + self._write_lines('\n#line %s "%s"\n' % (line, source_desc.get_escaped_description())) if code: if safe: self.put_safe(code) else: self.put(code) - self.write("\n") + self._write_lines("\n") self.bol = 1 def mark_pos(self, pos, trace=True): @@ -1885,13 +2007,13 @@ class CCodeWriter(object): pos, trace = self.last_pos self.last_marked_pos = pos self.last_pos = None - self.write("\n") + self._write_lines("\n") if self.code_config.emit_code_comments: self.indent() - self.write("/* %s */\n" % self._build_marker(pos)) + self._write_lines("/* %s */\n" % self._build_marker(pos)) if trace and self.funcstate and self.funcstate.can_trace and self.globalstate.directives['linetrace']: self.indent() - self.write('__Pyx_TraceLine(%d,%d,%s)\n' % ( + self._write_lines('__Pyx_TraceLine(%d,%d,%s)\n' % ( pos[1], not self.funcstate.gil_owned, self.error_goto(pos))) def _build_marker(self, pos): @@ -1912,7 +2034,7 @@ class CCodeWriter(object): include_dir = self.globalstate.common_utility_include_dir if include_dir and len(code) > 1024: include_file = "%s_%s.h" % ( - name, hashlib.md5(code.encode('utf8')).hexdigest()) + name, hashlib.sha1(code.encode('utf8')).hexdigest()) path = os.path.join(include_dir, include_file) if not os.path.exists(path): tmp_path = '%s.tmp%s' % (path, os.getpid()) @@ -1968,7 +2090,7 @@ class CCodeWriter(object): self.putln("}") def indent(self): - self.write(" " * self.level) + self._write_to_buffer(" " * self.level) def get_py_version_hex(self, pyversion): return "0x%02X%02X%02X%02X" % (tuple(pyversion) + (0,0,0,0))[:4] @@ -1990,26 +2112,33 @@ class CCodeWriter(object): if entry.visibility == "private" and not entry.used: #print "...private and not used, skipping", entry.cname ### return - if storage_class: - self.put("%s " % storage_class) if not entry.cf_used: self.put('CYTHON_UNUSED ') - self.put(entry.type.declaration_code( - entry.cname, dll_linkage=dll_linkage)) + if storage_class: + self.put("%s " % storage_class) + if entry.is_cpp_optional: + self.put(entry.type.cpp_optional_declaration_code( + entry.cname, dll_linkage=dll_linkage)) + else: + self.put(entry.type.declaration_code( + entry.cname, dll_linkage=dll_linkage)) if entry.init is not None: self.put_safe(" = %s" % entry.type.literal_code(entry.init)) elif entry.type.is_pyobject: self.put(" = NULL") self.putln(";") + self.funcstate.scope.use_entry_utility_code(entry) def put_temp_declarations(self, func_context): for name, type, manage_ref, static in func_context.temps_allocated: - decl = type.declaration_code(name) + if type.is_cpp_class and not type.is_fake_reference and func_context.scope.directives['cpp_locals']: + decl = type.cpp_optional_declaration_code(name) + else: + decl = type.declaration_code(name) if type.is_pyobject: self.putln("%s = NULL;" % decl) elif type.is_memoryviewslice: - from . import MemoryView - self.putln("%s = %s;" % (decl, MemoryView.memslice_entry_init)) + self.putln("%s = %s;" % (decl, type.literal_code(type.default_value))) else: self.putln("%s%s;" % (static and "static " or "", decl)) @@ -2024,7 +2153,7 @@ class CCodeWriter(object): self.putln("%sint %s = 0;" % (unused, Naming.clineno_cname)) def put_generated_by(self): - self.putln("/* Generated by Cython %s */" % Version.watermark) + self.putln(Utils.GENERATED_BY_MARKER) self.putln("") def put_h_guard(self, guard): @@ -2047,7 +2176,7 @@ class CCodeWriter(object): def entry_as_pyobject(self, entry): type = entry.type if (not entry.is_self_arg and not entry.type.is_complete() - or entry.type.is_extension_type): + or entry.type.is_extension_type): return "(PyObject *)" + entry.cname else: return entry.cname @@ -2056,123 +2185,89 @@ class CCodeWriter(object): from .PyrexTypes import py_object_type, typecast return typecast(py_object_type, type, cname) - def put_gotref(self, cname): - self.putln("__Pyx_GOTREF(%s);" % cname) + def put_gotref(self, cname, type): + type.generate_gotref(self, cname) - def put_giveref(self, cname): - self.putln("__Pyx_GIVEREF(%s);" % cname) + def put_giveref(self, cname, type): + type.generate_giveref(self, cname) - def put_xgiveref(self, cname): - self.putln("__Pyx_XGIVEREF(%s);" % cname) + def put_xgiveref(self, cname, type): + type.generate_xgiveref(self, cname) - def put_xgotref(self, cname): - self.putln("__Pyx_XGOTREF(%s);" % cname) + def put_xgotref(self, cname, type): + type.generate_xgotref(self, cname) def put_incref(self, cname, type, nanny=True): - if nanny: - self.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname, type)) - else: - self.putln("Py_INCREF(%s);" % self.as_pyobject(cname, type)) + # Note: original put_Memslice_Incref/Decref also added in some utility code + # this is unnecessary since the relevant utility code is loaded anyway if a memoryview is used + # and so has been removed. However, it's potentially a feature that might be useful here + type.generate_incref(self, cname, nanny=nanny) - def put_decref(self, cname, type, nanny=True): - self._put_decref(cname, type, nanny, null_check=False, clear=False) + def put_xincref(self, cname, type, nanny=True): + type.generate_xincref(self, cname, nanny=nanny) - def put_var_gotref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_GOTREF(%s);" % self.entry_as_pyobject(entry)) + def put_decref(self, cname, type, nanny=True, have_gil=True): + type.generate_decref(self, cname, nanny=nanny, have_gil=have_gil) - def put_var_giveref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_GIVEREF(%s);" % self.entry_as_pyobject(entry)) + def put_xdecref(self, cname, type, nanny=True, have_gil=True): + type.generate_xdecref(self, cname, nanny=nanny, have_gil=have_gil) - def put_var_xgotref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_XGOTREF(%s);" % self.entry_as_pyobject(entry)) + def put_decref_clear(self, cname, type, clear_before_decref=False, nanny=True, have_gil=True): + type.generate_decref_clear(self, cname, clear_before_decref=clear_before_decref, + nanny=nanny, have_gil=have_gil) - def put_var_xgiveref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_XGIVEREF(%s);" % self.entry_as_pyobject(entry)) + def put_xdecref_clear(self, cname, type, clear_before_decref=False, nanny=True, have_gil=True): + type.generate_xdecref_clear(self, cname, clear_before_decref=clear_before_decref, + nanny=nanny, have_gil=have_gil) - def put_var_incref(self, entry, nanny=True): - if entry.type.is_pyobject: - if nanny: - self.putln("__Pyx_INCREF(%s);" % self.entry_as_pyobject(entry)) - else: - self.putln("Py_INCREF(%s);" % self.entry_as_pyobject(entry)) + def put_decref_set(self, cname, type, rhs_cname): + type.generate_decref_set(self, cname, rhs_cname) - def put_var_xincref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_XINCREF(%s);" % self.entry_as_pyobject(entry)) + def put_xdecref_set(self, cname, type, rhs_cname): + type.generate_xdecref_set(self, cname, rhs_cname) - def put_decref_clear(self, cname, type, nanny=True, clear_before_decref=False): - self._put_decref(cname, type, nanny, null_check=False, - clear=True, clear_before_decref=clear_before_decref) + def put_incref_memoryviewslice(self, slice_cname, type, have_gil): + # TODO ideally this would just be merged into "put_incref" + type.generate_incref_memoryviewslice(self, slice_cname, have_gil=have_gil) - def put_xdecref(self, cname, type, nanny=True, have_gil=True): - self._put_decref(cname, type, nanny, null_check=True, - have_gil=have_gil, clear=False) + def put_var_incref_memoryviewslice(self, entry, have_gil): + self.put_incref_memoryviewslice(entry.cname, entry.type, have_gil=have_gil) - def put_xdecref_clear(self, cname, type, nanny=True, clear_before_decref=False): - self._put_decref(cname, type, nanny, null_check=True, - clear=True, clear_before_decref=clear_before_decref) + def put_var_gotref(self, entry): + self.put_gotref(entry.cname, entry.type) - def _put_decref(self, cname, type, nanny=True, null_check=False, - have_gil=True, clear=False, clear_before_decref=False): - if type.is_memoryviewslice: - self.put_xdecref_memoryviewslice(cname, have_gil=have_gil) - return + def put_var_giveref(self, entry): + self.put_giveref(entry.cname, entry.type) - prefix = '__Pyx' if nanny else 'Py' - X = 'X' if null_check else '' + def put_var_xgotref(self, entry): + self.put_xgotref(entry.cname, entry.type) - if clear: - if clear_before_decref: - if not nanny: - X = '' # CPython doesn't have a Py_XCLEAR() - self.putln("%s_%sCLEAR(%s);" % (prefix, X, cname)) - else: - self.putln("%s_%sDECREF(%s); %s = 0;" % ( - prefix, X, self.as_pyobject(cname, type), cname)) - else: - self.putln("%s_%sDECREF(%s);" % ( - prefix, X, self.as_pyobject(cname, type))) + def put_var_xgiveref(self, entry): + self.put_xgiveref(entry.cname, entry.type) - def put_decref_set(self, cname, rhs_cname): - self.putln("__Pyx_DECREF_SET(%s, %s);" % (cname, rhs_cname)) + def put_var_incref(self, entry, **kwds): + self.put_incref(entry.cname, entry.type, **kwds) - def put_xdecref_set(self, cname, rhs_cname): - self.putln("__Pyx_XDECREF_SET(%s, %s);" % (cname, rhs_cname)) + def put_var_xincref(self, entry, **kwds): + self.put_xincref(entry.cname, entry.type, **kwds) - def put_var_decref(self, entry): - if entry.type.is_pyobject: - self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry)) + def put_var_decref(self, entry, **kwds): + self.put_decref(entry.cname, entry.type, **kwds) - def put_var_xdecref(self, entry, nanny=True): - if entry.type.is_pyobject: - if nanny: - self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry)) - else: - self.putln("Py_XDECREF(%s);" % self.entry_as_pyobject(entry)) - - def put_var_decref_clear(self, entry): - self._put_var_decref_clear(entry, null_check=False) - - def put_var_xdecref_clear(self, entry): - self._put_var_decref_clear(entry, null_check=True) - - def _put_var_decref_clear(self, entry, null_check): - if entry.type.is_pyobject: - if entry.in_closure: - # reset before DECREF to make sure closure state is - # consistent during call to DECREF() - self.putln("__Pyx_%sCLEAR(%s);" % ( - null_check and 'X' or '', - entry.cname)) - else: - self.putln("__Pyx_%sDECREF(%s); %s = 0;" % ( - null_check and 'X' or '', - self.entry_as_pyobject(entry), - entry.cname)) + def put_var_xdecref(self, entry, **kwds): + self.put_xdecref(entry.cname, entry.type, **kwds) + + def put_var_decref_clear(self, entry, **kwds): + self.put_decref_clear(entry.cname, entry.type, clear_before_decref=entry.in_closure, **kwds) + + def put_var_decref_set(self, entry, rhs_cname, **kwds): + self.put_decref_set(entry.cname, entry.type, rhs_cname, **kwds) + + def put_var_xdecref_set(self, entry, rhs_cname, **kwds): + self.put_xdecref_set(entry.cname, entry.type, rhs_cname, **kwds) + + def put_var_xdecref_clear(self, entry, **kwds): + self.put_xdecref_clear(entry.cname, entry.type, clear_before_decref=entry.in_closure, **kwds) def put_var_decrefs(self, entries, used_only = 0): for entry in entries: @@ -2190,19 +2285,6 @@ class CCodeWriter(object): for entry in entries: self.put_var_xdecref_clear(entry) - def put_incref_memoryviewslice(self, slice_cname, have_gil=False): - from . import MemoryView - self.globalstate.use_utility_code(MemoryView.memviewslice_init_code) - self.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil))) - - def put_xdecref_memoryviewslice(self, slice_cname, have_gil=False): - from . import MemoryView - self.globalstate.use_utility_code(MemoryView.memviewslice_init_code) - self.putln("__PYX_XDEC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil))) - - def put_xgiveref_memoryviewslice(self, slice_cname): - self.put_xgiveref("%s.memview" % slice_cname) - def put_init_to_py_none(self, cname, type, nanny=True): from .PyrexTypes import py_object_type, typecast py_none = typecast(type, py_object_type, "Py_None") @@ -2220,8 +2302,11 @@ class CCodeWriter(object): self.put_giveref('Py_None') def put_pymethoddef(self, entry, term, allow_skip=True, wrapper_code_writer=None): + is_reverse_number_slot = False if entry.is_special or entry.name == '__getattribute__': - if entry.name not in special_py_methods: + from . import TypeSlots + is_reverse_number_slot = True + if entry.name not in special_py_methods and not TypeSlots.is_reverse_number_slot(entry.name): if entry.name == '__getattr__' and not self.globalstate.directives['fast_getattr']: pass # Python's typeobject.c will automatically fill in our slot @@ -2233,37 +2318,60 @@ class CCodeWriter(object): method_flags = entry.signature.method_flags() if not method_flags: return - from . import TypeSlots - if entry.is_special or TypeSlots.is_reverse_number_slot(entry.name): + if entry.is_special: method_flags += [TypeSlots.method_coexist] func_ptr = wrapper_code_writer.put_pymethoddef_wrapper(entry) if wrapper_code_writer else entry.func_cname # Add required casts, but try not to shadow real warnings. - cast = '__Pyx_PyCFunctionFast' if 'METH_FASTCALL' in method_flags else 'PyCFunction' - if 'METH_KEYWORDS' in method_flags: - cast += 'WithKeywords' + cast = entry.signature.method_function_type() if cast != 'PyCFunction': func_ptr = '(void*)(%s)%s' % (cast, func_ptr) + entry_name = entry.name.as_c_string_literal() + if is_reverse_number_slot: + # Unlike most special functions, reverse number operator slots are actually generated here + # (to ensure that they can be looked up). However, they're sometimes guarded by the preprocessor + # so a bit of extra logic is needed + slot = TypeSlots.get_slot_table(self.globalstate.directives).get_slot_by_method_name(entry.name) + preproc_guard = slot.preprocessor_guard_code() + if preproc_guard: + self.putln(preproc_guard) self.putln( - '{"%s", (PyCFunction)%s, %s, %s}%s' % ( - entry.name, + '{%s, (PyCFunction)%s, %s, %s}%s' % ( + entry_name, func_ptr, "|".join(method_flags), entry.doc_cname if entry.doc else '0', term)) + if is_reverse_number_slot and preproc_guard: + self.putln("#endif") def put_pymethoddef_wrapper(self, entry): func_cname = entry.func_cname if entry.is_special: - method_flags = entry.signature.method_flags() - if method_flags and 'METH_NOARGS' in method_flags: + method_flags = entry.signature.method_flags() or [] + from .TypeSlots import method_noargs + if method_noargs in method_flags: # Special NOARGS methods really take no arguments besides 'self', but PyCFunction expects one. func_cname = Naming.method_wrapper_prefix + func_cname - self.putln("static PyObject *%s(PyObject *self, CYTHON_UNUSED PyObject *arg) {return %s(self);}" % ( - func_cname, entry.func_cname)) + self.putln("static PyObject *%s(PyObject *self, CYTHON_UNUSED PyObject *arg) {" % func_cname) + func_call = "%s(self)" % entry.func_cname + if entry.name == "__next__": + self.putln("PyObject *res = %s;" % func_call) + # tp_iternext can return NULL without an exception + self.putln("if (!res && !PyErr_Occurred()) { PyErr_SetNone(PyExc_StopIteration); }") + self.putln("return res;") + else: + self.putln("return %s;" % func_call) + self.putln("}") return func_cname # GIL methods + def use_fast_gil_utility_code(self): + if self.globalstate.directives['fast_gil']: + self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) + else: + self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) + def put_ensure_gil(self, declare_gilstate=True, variable=None): """ Acquire the GIL. The generated code is safe even when no PyThreadState @@ -2273,10 +2381,7 @@ class CCodeWriter(object): """ self.globalstate.use_utility_code( UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c")) - if self.globalstate.directives['fast_gil']: - self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) - else: - self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) + self.use_fast_gil_utility_code() self.putln("#ifdef WITH_THREAD") if not variable: variable = '__pyx_gilstate_save' @@ -2289,41 +2394,43 @@ class CCodeWriter(object): """ Releases the GIL, corresponds to `put_ensure_gil`. """ - if self.globalstate.directives['fast_gil']: - self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) - else: - self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) + self.use_fast_gil_utility_code() if not variable: variable = '__pyx_gilstate_save' self.putln("#ifdef WITH_THREAD") self.putln("__Pyx_PyGILState_Release(%s);" % variable) self.putln("#endif") - def put_acquire_gil(self, variable=None): + def put_acquire_gil(self, variable=None, unknown_gil_state=True): """ Acquire the GIL. The thread's thread state must have been initialized by a previous `put_release_gil` """ - if self.globalstate.directives['fast_gil']: - self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) - else: - self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) + self.use_fast_gil_utility_code() self.putln("#ifdef WITH_THREAD") self.putln("__Pyx_FastGIL_Forget();") if variable: self.putln('_save = %s;' % variable) + if unknown_gil_state: + self.putln("if (_save) {") self.putln("Py_BLOCK_THREADS") + if unknown_gil_state: + self.putln("}") self.putln("#endif") - def put_release_gil(self, variable=None): + def put_release_gil(self, variable=None, unknown_gil_state=True): "Release the GIL, corresponds to `put_acquire_gil`." - if self.globalstate.directives['fast_gil']: - self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c")) - else: - self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c")) + self.use_fast_gil_utility_code() self.putln("#ifdef WITH_THREAD") self.putln("PyThreadState *_save;") + self.putln("_save = NULL;") + if unknown_gil_state: + # we don't *know* that we don't have the GIL (since we may be inside a nogil function, + # and Py_UNBLOCK_THREADS is unsafe without the GIL) + self.putln("if (PyGILState_Check()) {") self.putln("Py_UNBLOCK_THREADS") + if unknown_gil_state: + self.putln("}") if variable: self.putln('%s = _save;' % variable) self.putln("__Pyx_FastGIL_Remember();") @@ -2341,23 +2448,34 @@ class CCodeWriter(object): # return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos))) return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos))) - def put_error_if_unbound(self, pos, entry, in_nogil_context=False): - from . import ExprNodes + def put_error_if_unbound(self, pos, entry, in_nogil_context=False, unbound_check_code=None): if entry.from_closure: func = '__Pyx_RaiseClosureNameError' self.globalstate.use_utility_code( - ExprNodes.raise_closure_name_error_utility_code) + UtilityCode.load_cached("RaiseClosureNameError", "ObjectHandling.c")) elif entry.type.is_memoryviewslice and in_nogil_context: func = '__Pyx_RaiseUnboundMemoryviewSliceNogil' self.globalstate.use_utility_code( - ExprNodes.raise_unbound_memoryview_utility_code_nogil) + UtilityCode.load_cached("RaiseUnboundMemoryviewSliceNogil", "ObjectHandling.c")) + elif entry.type.is_cpp_class and entry.is_cglobal: + func = '__Pyx_RaiseCppGlobalNameError' + self.globalstate.use_utility_code( + UtilityCode.load_cached("RaiseCppGlobalNameError", "ObjectHandling.c")) + elif entry.type.is_cpp_class and entry.is_variable and not entry.is_member and entry.scope.is_c_class_scope: + # there doesn't seem to be a good way to detecting an instance-attribute of a C class + # (is_member is only set for class attributes) + func = '__Pyx_RaiseCppAttributeError' + self.globalstate.use_utility_code( + UtilityCode.load_cached("RaiseCppAttributeError", "ObjectHandling.c")) else: func = '__Pyx_RaiseUnboundLocalError' self.globalstate.use_utility_code( - ExprNodes.raise_unbound_local_error_utility_code) + UtilityCode.load_cached("RaiseUnboundLocalError", "ObjectHandling.c")) + if not unbound_check_code: + unbound_check_code = entry.type.check_for_null_code(entry.cname) self.putln('if (unlikely(!%s)) { %s("%s"); %s }' % ( - entry.type.check_for_null_code(entry.cname), + unbound_check_code, func, entry.name, self.error_goto(pos))) @@ -2390,7 +2508,8 @@ class CCodeWriter(object): return self.error_goto_if("!%s" % cname, pos) def error_goto_if_neg(self, cname, pos): - return self.error_goto_if("%s < 0" % cname, pos) + # Add extra parentheses to silence clang warnings about constant conditions. + return self.error_goto_if("(%s < 0)" % cname, pos) def error_goto_if_PyErr(self, pos): return self.error_goto_if("PyErr_Occurred()", pos) @@ -2402,13 +2521,14 @@ class CCodeWriter(object): self.putln('__Pyx_RefNannyDeclarations') def put_setup_refcount_context(self, name, acquire_gil=False): + name = name.as_c_string_literal() # handle unicode names if acquire_gil: self.globalstate.use_utility_code( UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c")) - self.putln('__Pyx_RefNannySetupContext("%s", %d);' % (name, acquire_gil and 1 or 0)) + self.putln('__Pyx_RefNannySetupContext(%s, %d);' % (name, acquire_gil and 1 or 0)) - def put_finish_refcount_context(self): - self.putln("__Pyx_RefNannyFinishContext();") + def put_finish_refcount_context(self, nogil=False): + self.putln("__Pyx_RefNannyFinishContextNogil()" if nogil else "__Pyx_RefNannyFinishContext();") def put_add_traceback(self, qualified_name, include_cline=True): """ @@ -2416,14 +2536,16 @@ class CCodeWriter(object): qualified_name should be the qualified name of the function. """ + qualified_name = qualified_name.as_c_string_literal() # handle unicode names format_tuple = ( qualified_name, Naming.clineno_cname if include_cline else 0, Naming.lineno_cname, Naming.filename_cname, ) + self.funcstate.uses_error_indicator = True - self.putln('__Pyx_AddTraceback("%s", %s, %s, %s);' % format_tuple) + self.putln('__Pyx_AddTraceback(%s, %s, %s, %s);' % format_tuple) def put_unraisable(self, qualified_name, nogil=False): """ @@ -2504,16 +2626,16 @@ class PyrexCodeWriter(object): def dedent(self): self.level -= 1 + class PyxCodeWriter(object): """ - Can be used for writing out some Cython code. To use the indenter - functionality, the Cython.Compiler.Importer module will have to be used - to load the code to support python 2.4 + Can be used for writing out some Cython code. """ def __init__(self, buffer=None, indent_level=0, context=None, encoding='ascii'): self.buffer = buffer or StringIOTree() self.level = indent_level + self.original_level = indent_level self.context = context self.encoding = encoding @@ -2524,22 +2646,19 @@ class PyxCodeWriter(object): def dedent(self, levels=1): self.level -= levels + @contextmanager def indenter(self, line): """ - Instead of - - with pyx_code.indenter("for i in range(10):"): - pyx_code.putln("print i") - - write - - if pyx_code.indenter("for i in range(10);"): - pyx_code.putln("print i") - pyx_code.dedent() + with pyx_code.indenter("for i in range(10):"): + pyx_code.putln("print i") """ self.putln(line) self.indent() - return True + yield + self.dedent() + + def empty(self): + return self.buffer.empty() def getvalue(self): result = self.buffer.getvalue() @@ -2554,7 +2673,7 @@ class PyxCodeWriter(object): self._putln(line) def _putln(self, line): - self.buffer.write("%s%s\n" % (self.level * " ", line)) + self.buffer.write(u"%s%s\n" % (self.level * u" ", line)) def put_chunk(self, chunk, context=None): context = context or self.context @@ -2566,8 +2685,13 @@ class PyxCodeWriter(object): self._putln(line) def insertion_point(self): - return PyxCodeWriter(self.buffer.insertion_point(), self.level, - self.context) + return type(self)(self.buffer.insertion_point(), self.level, self.context) + + def reset(self): + # resets the buffer so that nothing gets written. Most useful + # for abandoning all work in a specific insertion point + self.buffer.reset() + self.level = self.original_level def named_insertion_point(self, name): setattr(self, name, self.insertion_point()) diff --git a/Cython/Compiler/CythonScope.py b/Cython/Compiler/CythonScope.py index 1c25d1a6b..f73be0070 100644 --- a/Cython/Compiler/CythonScope.py +++ b/Cython/Compiler/CythonScope.py @@ -6,6 +6,7 @@ from .UtilityCode import CythonUtilityCode from .Errors import error from .Scanning import StringSourceDescriptor from . import MemoryView +from .StringEncoding import EncodedString class CythonScope(ModuleScope): @@ -50,7 +51,7 @@ class CythonScope(ModuleScope): def find_module(self, module_name, pos): error("cython.%s is not available" % module_name, pos) - def find_submodule(self, module_name): + def find_submodule(self, module_name, as_package=False): entry = self.entries.get(module_name, None) if not entry: self.load_cythonscope() @@ -125,10 +126,26 @@ class CythonScope(ModuleScope): view_utility_scope = MemoryView.view_utility_code.declare_in_scope( self.viewscope, cython_scope=self, - whitelist=MemoryView.view_utility_whitelist) + allowlist=MemoryView.view_utility_allowlist) + + # Marks the types as being cython_builtin_type so that they can be + # extended from without Cython attempting to import cython.view + ext_types = [ entry.type + for entry in view_utility_scope.entries.values() + if entry.type.is_extension_type ] + for ext_type in ext_types: + ext_type.is_cython_builtin_type = 1 # self.entries["array"] = view_utility_scope.entries.pop("array") + # dataclasses scope + dc_str = EncodedString(u'dataclasses') + dataclassesscope = ModuleScope(dc_str, self, context=None) + self.declare_module(dc_str, dataclassesscope, pos=None).as_module = dataclassesscope + dataclassesscope.is_cython_builtin = True + dataclassesscope.pxd_file_loaded = True + # doesn't actually have any contents + def create_cython_scope(context): # One could in fact probably make it a singleton, diff --git a/Cython/Compiler/Dataclass.py b/Cython/Compiler/Dataclass.py new file mode 100644 index 000000000..e775e9182 --- /dev/null +++ b/Cython/Compiler/Dataclass.py @@ -0,0 +1,840 @@ +# functions to transform a c class into a dataclass + +from collections import OrderedDict +from textwrap import dedent +import operator + +from . import ExprNodes +from . import Nodes +from . import PyrexTypes +from . import Builtin +from . import Naming +from .Errors import error, warning +from .Code import UtilityCode, TempitaUtilityCode, PyxCodeWriter +from .Visitor import VisitorTransform +from .StringEncoding import EncodedString +from .TreeFragment import TreeFragment +from .ParseTreeTransforms import NormalizeTree, SkipDeclarations +from .Options import copy_inherited_directives + +_dataclass_loader_utilitycode = None + +def make_dataclasses_module_callnode(pos): + global _dataclass_loader_utilitycode + if not _dataclass_loader_utilitycode: + python_utility_code = UtilityCode.load_cached("Dataclasses_fallback", "Dataclasses.py") + python_utility_code = EncodedString(python_utility_code.impl) + _dataclass_loader_utilitycode = TempitaUtilityCode.load( + "SpecificModuleLoader", "Dataclasses.c", + context={'cname': "dataclasses", 'py_code': python_utility_code.as_c_string_literal()}) + return ExprNodes.PythonCapiCallNode( + pos, "__Pyx_Load_dataclasses_Module", + PyrexTypes.CFuncType(PyrexTypes.py_object_type, []), + utility_code=_dataclass_loader_utilitycode, + args=[], + ) + +def make_dataclass_call_helper(pos, callable, kwds): + utility_code = UtilityCode.load_cached("DataclassesCallHelper", "Dataclasses.c") + func_type = PyrexTypes.CFuncType( + PyrexTypes.py_object_type, [ + PyrexTypes.CFuncTypeArg("callable", PyrexTypes.py_object_type, None), + PyrexTypes.CFuncTypeArg("kwds", PyrexTypes.py_object_type, None) + ], + ) + return ExprNodes.PythonCapiCallNode( + pos, + function_name="__Pyx_DataclassesCallHelper", + func_type=func_type, + utility_code=utility_code, + args=[callable, kwds], + ) + + +class RemoveAssignmentsToNames(VisitorTransform, SkipDeclarations): + """ + Cython (and Python) normally treats + + class A: + x = 1 + + as generating a class attribute. However for dataclasses the `= 1` should be interpreted as + a default value to initialize an instance attribute with. + This transform therefore removes the `x=1` assignment so that the class attribute isn't + generated, while recording what it has removed so that it can be used in the initialization. + """ + def __init__(self, names): + super(RemoveAssignmentsToNames, self).__init__() + self.names = names + self.removed_assignments = {} + + def visit_CClassNode(self, node): + self.visitchildren(node) + return node + + def visit_PyClassNode(self, node): + return node # go no further + + def visit_FuncDefNode(self, node): + return node # go no further + + def visit_SingleAssignmentNode(self, node): + if node.lhs.is_name and node.lhs.name in self.names: + if node.lhs.name in self.removed_assignments: + warning(node.pos, ("Multiple assignments for '%s' in dataclass; " + "using most recent") % node.lhs.name, 1) + self.removed_assignments[node.lhs.name] = node.rhs + return [] + return node + + # I believe cascaded assignment is always a syntax error with annotations + # so there's no need to define visit_CascadedAssignmentNode + + def visit_Node(self, node): + self.visitchildren(node) + return node + + +class TemplateCode(object): + """ + Adds the ability to keep track of placeholder argument names to PyxCodeWriter. + + Also adds extra_stats which are nodes bundled at the end when this + is converted to a tree. + """ + _placeholder_count = 0 + + def __init__(self, writer=None, placeholders=None, extra_stats=None): + self.writer = PyxCodeWriter() if writer is None else writer + self.placeholders = {} if placeholders is None else placeholders + self.extra_stats = [] if extra_stats is None else extra_stats + + def add_code_line(self, code_line): + self.writer.putln(code_line) + + def add_code_lines(self, code_lines): + for line in code_lines: + self.writer.putln(line) + + def reset(self): + # don't attempt to reset placeholders - it really doesn't matter if + # we have unused placeholders + self.writer.reset() + + def empty(self): + return self.writer.empty() + + def indenter(self): + return self.writer.indenter() + + def new_placeholder(self, field_names, value): + name = self._new_placeholder_name(field_names) + self.placeholders[name] = value + return name + + def add_extra_statements(self, statements): + if self.extra_stats is None: + assert False, "Can only use add_extra_statements on top-level writer" + self.extra_stats.extend(statements) + + def _new_placeholder_name(self, field_names): + while True: + name = "DATACLASS_PLACEHOLDER_%d" % self._placeholder_count + if (name not in self.placeholders + and name not in field_names): + # make sure name isn't already used and doesn't + # conflict with a variable name (which is unlikely but possible) + break + self._placeholder_count += 1 + return name + + def generate_tree(self, level='c_class'): + stat_list_node = TreeFragment( + self.writer.getvalue(), + level=level, + pipeline=[NormalizeTree(None)], + ).substitute(self.placeholders) + + stat_list_node.stats += self.extra_stats + return stat_list_node + + def insertion_point(self): + new_writer = self.writer.insertion_point() + return TemplateCode( + writer=new_writer, + placeholders=self.placeholders, + extra_stats=self.extra_stats + ) + + +class _MISSING_TYPE(object): + pass +MISSING = _MISSING_TYPE() + + +class Field(object): + """ + Field is based on the dataclasses.field class from the standard library module. + It is used internally during the generation of Cython dataclasses to keep track + of the settings for individual attributes. + + Attributes of this class are stored as nodes so they can be used in code construction + more readily (i.e. we store BoolNode rather than bool) + """ + default = MISSING + default_factory = MISSING + private = False + + literal_keys = ("repr", "hash", "init", "compare", "metadata") + + # default values are defined by the CPython dataclasses.field + def __init__(self, pos, default=MISSING, default_factory=MISSING, + repr=None, hash=None, init=None, + compare=None, metadata=None, + is_initvar=False, is_classvar=False, + **additional_kwds): + if default is not MISSING: + self.default = default + if default_factory is not MISSING: + self.default_factory = default_factory + self.repr = repr or ExprNodes.BoolNode(pos, value=True) + self.hash = hash or ExprNodes.NoneNode(pos) + self.init = init or ExprNodes.BoolNode(pos, value=True) + self.compare = compare or ExprNodes.BoolNode(pos, value=True) + self.metadata = metadata or ExprNodes.NoneNode(pos) + self.is_initvar = is_initvar + self.is_classvar = is_classvar + + for k, v in additional_kwds.items(): + # There should not be any additional keywords! + error(v.pos, "cython.dataclasses.field() got an unexpected keyword argument '%s'" % k) + + for field_name in self.literal_keys: + field_value = getattr(self, field_name) + if not field_value.is_literal: + error(field_value.pos, + "cython.dataclasses.field parameter '%s' must be a literal value" % field_name) + + def iterate_record_node_arguments(self): + for key in (self.literal_keys + ('default', 'default_factory')): + value = getattr(self, key) + if value is not MISSING: + yield key, value + + +def process_class_get_fields(node): + var_entries = node.scope.var_entries + # order of definition is used in the dataclass + var_entries = sorted(var_entries, key=operator.attrgetter('pos')) + var_names = [entry.name for entry in var_entries] + + # don't treat `x = 1` as an assignment of a class attribute within the dataclass + transform = RemoveAssignmentsToNames(var_names) + transform(node) + default_value_assignments = transform.removed_assignments + + base_type = node.base_type + fields = OrderedDict() + while base_type: + if base_type.is_external or not base_type.scope.implemented: + warning(node.pos, "Cannot reliably handle Cython dataclasses with base types " + "in external modules since it is not possible to tell what fields they have", 2) + if base_type.dataclass_fields: + fields = base_type.dataclass_fields.copy() + break + base_type = base_type.base_type + + for entry in var_entries: + name = entry.name + is_initvar = entry.declared_with_pytyping_modifier("dataclasses.InitVar") + # TODO - classvars aren't included in "var_entries" so are missed here + # and thus this code is never triggered + is_classvar = entry.declared_with_pytyping_modifier("typing.ClassVar") + if name in default_value_assignments: + assignment = default_value_assignments[name] + if (isinstance(assignment, ExprNodes.CallNode) + and assignment.function.as_cython_attribute() == "dataclasses.field"): + # I believe most of this is well-enforced when it's treated as a directive + # but it doesn't hurt to make sure + valid_general_call = (isinstance(assignment, ExprNodes.GeneralCallNode) + and isinstance(assignment.positional_args, ExprNodes.TupleNode) + and not assignment.positional_args.args + and (assignment.keyword_args is None or isinstance(assignment.keyword_args, ExprNodes.DictNode))) + valid_simple_call = (isinstance(assignment, ExprNodes.SimpleCallNode) and not assignment.args) + if not (valid_general_call or valid_simple_call): + error(assignment.pos, "Call to 'cython.dataclasses.field' must only consist " + "of compile-time keyword arguments") + continue + keyword_args = assignment.keyword_args.as_python_dict() if valid_general_call and assignment.keyword_args else {} + if 'default' in keyword_args and 'default_factory' in keyword_args: + error(assignment.pos, "cannot specify both default and default_factory") + continue + field = Field(node.pos, **keyword_args) + else: + if isinstance(assignment, ExprNodes.CallNode): + func = assignment.function + if ((func.is_name and func.name == "field") + or (func.is_attribute and func.attribute == "field")): + warning(assignment.pos, "Do you mean cython.dataclasses.field instead?", 1) + if assignment.type in [Builtin.list_type, Builtin.dict_type, Builtin.set_type]: + # The standard library module generates a TypeError at runtime + # in this situation. + # Error message is copied from CPython + error(assignment.pos, "mutable default <class '{0}'> for field {1} is not allowed: " + "use default_factory".format(assignment.type.name, name)) + + field = Field(node.pos, default=assignment) + else: + field = Field(node.pos) + field.is_initvar = is_initvar + field.is_classvar = is_classvar + if entry.visibility == "private": + field.private = True + fields[name] = field + node.entry.type.dataclass_fields = fields + return fields + + +def handle_cclass_dataclass(node, dataclass_args, analyse_decs_transform): + # default argument values from https://docs.python.org/3/library/dataclasses.html + kwargs = dict(init=True, repr=True, eq=True, + order=False, unsafe_hash=False, + frozen=False, kw_only=False) + if dataclass_args is not None: + if dataclass_args[0]: + error(node.pos, "cython.dataclasses.dataclass takes no positional arguments") + for k, v in dataclass_args[1].items(): + if k not in kwargs: + error(node.pos, + "cython.dataclasses.dataclass() got an unexpected keyword argument '%s'" % k) + if not isinstance(v, ExprNodes.BoolNode): + error(node.pos, + "Arguments passed to cython.dataclasses.dataclass must be True or False") + kwargs[k] = v.value + + kw_only = kwargs['kw_only'] + + fields = process_class_get_fields(node) + + dataclass_module = make_dataclasses_module_callnode(node.pos) + + # create __dataclass_params__ attribute. I try to use the exact + # `_DataclassParams` class defined in the standard library module if at all possible + # for maximum duck-typing compatibility. + dataclass_params_func = ExprNodes.AttributeNode(node.pos, obj=dataclass_module, + attribute=EncodedString("_DataclassParams")) + dataclass_params_keywords = ExprNodes.DictNode.from_pairs( + node.pos, + [ (ExprNodes.IdentifierStringNode(node.pos, value=EncodedString(k)), + ExprNodes.BoolNode(node.pos, value=v)) + for k, v in kwargs.items() ] + + [ (ExprNodes.IdentifierStringNode(node.pos, value=EncodedString(k)), + ExprNodes.BoolNode(node.pos, value=v)) + for k, v in [('kw_only', kw_only), ('match_args', False), + ('slots', False), ('weakref_slot', False)] + ]) + dataclass_params = make_dataclass_call_helper( + node.pos, dataclass_params_func, dataclass_params_keywords) + dataclass_params_assignment = Nodes.SingleAssignmentNode( + node.pos, + lhs = ExprNodes.NameNode(node.pos, name=EncodedString("__dataclass_params__")), + rhs = dataclass_params) + + dataclass_fields_stats = _set_up_dataclass_fields(node, fields, dataclass_module) + + stats = Nodes.StatListNode(node.pos, + stats=[dataclass_params_assignment] + dataclass_fields_stats) + + code = TemplateCode() + generate_init_code(code, kwargs['init'], node, fields, kw_only) + generate_repr_code(code, kwargs['repr'], node, fields) + generate_eq_code(code, kwargs['eq'], node, fields) + generate_order_code(code, kwargs['order'], node, fields) + generate_hash_code(code, kwargs['unsafe_hash'], kwargs['eq'], kwargs['frozen'], node, fields) + + stats.stats += code.generate_tree().stats + + # turn off annotation typing, so all arguments to __init__ are accepted as + # generic objects and thus can accept _HAS_DEFAULT_FACTORY. + # Type conversion comes later + comp_directives = Nodes.CompilerDirectivesNode(node.pos, + directives=copy_inherited_directives(node.scope.directives, annotation_typing=False), + body=stats) + + comp_directives.analyse_declarations(node.scope) + # probably already in this scope, but it doesn't hurt to make sure + analyse_decs_transform.enter_scope(node, node.scope) + analyse_decs_transform.visit(comp_directives) + analyse_decs_transform.exit_scope() + + node.body.stats.append(comp_directives) + + +def generate_init_code(code, init, node, fields, kw_only): + """ + Notes on CPython generated "__init__": + * Implemented in `_init_fn`. + * The use of the `dataclasses._HAS_DEFAULT_FACTORY` sentinel value as + the default argument for fields that need constructing with a factory + function is copied from the CPython implementation. (`None` isn't + suitable because it could also be a value for the user to pass.) + There's no real reason why it needs importing from the dataclasses module + though - it could equally be a value generated by Cython when the module loads. + * seen_default and the associated error message are copied directly from Python + * Call to user-defined __post_init__ function (if it exists) is copied from + CPython. + + Cython behaviour deviates a little here (to be decided if this is right...) + Because the class variable from the assignment does not exist Cython fields will + return None (or whatever their type default is) if not initialized while Python + dataclasses will fall back to looking up the class variable. + """ + if not init or node.scope.lookup_here("__init__"): + return + + # selfname behaviour copied from the cpython module + selfname = "__dataclass_self__" if "self" in fields else "self" + args = [selfname] + + if kw_only: + args.append("*") + + function_start_point = code.insertion_point() + code = code.insertion_point() + + # create a temp to get _HAS_DEFAULT_FACTORY + dataclass_module = make_dataclasses_module_callnode(node.pos) + has_default_factory = ExprNodes.AttributeNode( + node.pos, + obj=dataclass_module, + attribute=EncodedString("_HAS_DEFAULT_FACTORY") + ) + + default_factory_placeholder = code.new_placeholder(fields, has_default_factory) + + seen_default = False + for name, field in fields.items(): + entry = node.scope.lookup(name) + if entry.annotation: + annotation = u": %s" % entry.annotation.string.value + else: + annotation = u"" + assignment = u'' + if field.default is not MISSING or field.default_factory is not MISSING: + seen_default = True + if field.default_factory is not MISSING: + ph_name = default_factory_placeholder + else: + ph_name = code.new_placeholder(fields, field.default) # 'default' should be a node + assignment = u" = %s" % ph_name + elif seen_default and not kw_only and field.init.value: + error(entry.pos, ("non-default argument '%s' follows default argument " + "in dataclass __init__") % name) + code.reset() + return + + if field.init.value: + args.append(u"%s%s%s" % (name, annotation, assignment)) + + if field.is_initvar: + continue + elif field.default_factory is MISSING: + if field.init.value: + code.add_code_line(u" %s.%s = %s" % (selfname, name, name)) + elif assignment: + # not an argument to the function, but is still initialized + code.add_code_line(u" %s.%s%s" % (selfname, name, assignment)) + else: + ph_name = code.new_placeholder(fields, field.default_factory) + if field.init.value: + # close to: + # def __init__(self, name=_PLACEHOLDER_VALUE): + # self.name = name_default_factory() if name is _PLACEHOLDER_VALUE else name + code.add_code_line(u" %s.%s = %s() if %s is %s else %s" % ( + selfname, name, ph_name, name, default_factory_placeholder, name)) + else: + # still need to use the default factory to initialize + code.add_code_line(u" %s.%s = %s()" % ( + selfname, name, ph_name)) + + if node.scope.lookup("__post_init__"): + post_init_vars = ", ".join(name for name, field in fields.items() + if field.is_initvar) + code.add_code_line(" %s.__post_init__(%s)" % (selfname, post_init_vars)) + + if code.empty(): + code.add_code_line(" pass") + + args = u", ".join(args) + function_start_point.add_code_line(u"def __init__(%s):" % args) + + +def generate_repr_code(code, repr, node, fields): + """ + The core of the CPython implementation is just: + ['return self.__class__.__qualname__ + f"(' + + ', '.join([f"{f.name}={{self.{f.name}!r}}" + for f in fields]) + + ')"'], + + The only notable difference here is self.__class__.__qualname__ -> type(self).__name__ + which is because Cython currently supports Python 2. + + However, it also has some guards for recursive repr invokations. In the standard + library implementation they're done with a wrapper decorator that captures a set + (with the set keyed by id and thread). Here we create a set as a thread local + variable and key only by id. + """ + if not repr or node.scope.lookup("__repr__"): + return + + # The recursive guard is likely a little costly, so skip it if possible. + # is_gc_simple defines where it can contain recursive objects + needs_recursive_guard = False + for name in fields.keys(): + entry = node.scope.lookup(name) + type_ = entry.type + if type_.is_memoryviewslice: + type_ = type_.dtype + if not type_.is_pyobject: + continue # no GC + if not type_.is_gc_simple: + needs_recursive_guard = True + break + + if needs_recursive_guard: + code.add_code_line("__pyx_recursive_repr_guard = __import__('threading').local()") + code.add_code_line("__pyx_recursive_repr_guard.running = set()") + code.add_code_line("def __repr__(self):") + if needs_recursive_guard: + code.add_code_line(" key = id(self)") + code.add_code_line(" guard_set = self.__pyx_recursive_repr_guard.running") + code.add_code_line(" if key in guard_set: return '...'") + code.add_code_line(" guard_set.add(key)") + code.add_code_line(" try:") + strs = [u"%s={self.%s!r}" % (name, name) + for name, field in fields.items() + if field.repr.value and not field.is_initvar] + format_string = u", ".join(strs) + + code.add_code_line(u' name = getattr(type(self), "__qualname__", type(self).__name__)') + code.add_code_line(u" return f'{name}(%s)'" % format_string) + if needs_recursive_guard: + code.add_code_line(" finally:") + code.add_code_line(" guard_set.remove(key)") + + +def generate_cmp_code(code, op, funcname, node, fields): + if node.scope.lookup_here(funcname): + return + + names = [name for name, field in fields.items() if (field.compare.value and not field.is_initvar)] + + code.add_code_lines([ + "def %s(self, other):" % funcname, + " if not isinstance(other, %s):" % node.class_name, + " return NotImplemented", + # + " cdef %s other_cast" % node.class_name, + " other_cast = <%s>other" % node.class_name, + ]) + + # The Python implementation of dataclasses.py does a tuple comparison + # (roughly): + # return self._attributes_to_tuple() {op} other._attributes_to_tuple() + # + # For the Cython implementation a tuple comparison isn't an option because + # not all attributes can be converted to Python objects and stored in a tuple + # + # TODO - better diagnostics of whether the types support comparison before + # generating the code. Plus, do we want to convert C structs to dicts and + # compare them that way (I think not, but it might be in demand)? + checks = [] + for name in names: + checks.append("(self.%s %s other_cast.%s)" % ( + name, op, name)) + + if checks: + code.add_code_line(" return " + " and ".join(checks)) + else: + if "=" in op: + code.add_code_line(" return True") # "() == ()" is True + else: + code.add_code_line(" return False") + + +def generate_eq_code(code, eq, node, fields): + if not eq: + return + generate_cmp_code(code, "==", "__eq__", node, fields) + + +def generate_order_code(code, order, node, fields): + if not order: + return + + for op, name in [("<", "__lt__"), + ("<=", "__le__"), + (">", "__gt__"), + (">=", "__ge__")]: + generate_cmp_code(code, op, name, node, fields) + + +def generate_hash_code(code, unsafe_hash, eq, frozen, node, fields): + """ + Copied from CPython implementation - the intention is to follow this as far as + is possible: + # +------------------- unsafe_hash= parameter + # | +----------- eq= parameter + # | | +--- frozen= parameter + # | | | + # v v v | | | + # | no | yes | <--- class has explicitly defined __hash__ + # +=======+=======+=======+========+========+ + # | False | False | False | | | No __eq__, use the base class __hash__ + # +-------+-------+-------+--------+--------+ + # | False | False | True | | | No __eq__, use the base class __hash__ + # +-------+-------+-------+--------+--------+ + # | False | True | False | None | | <-- the default, not hashable + # +-------+-------+-------+--------+--------+ + # | False | True | True | add | | Frozen, so hashable, allows override + # +-------+-------+-------+--------+--------+ + # | True | False | False | add | raise | Has no __eq__, but hashable + # +-------+-------+-------+--------+--------+ + # | True | False | True | add | raise | Has no __eq__, but hashable + # +-------+-------+-------+--------+--------+ + # | True | True | False | add | raise | Not frozen, but hashable + # +-------+-------+-------+--------+--------+ + # | True | True | True | add | raise | Frozen, so hashable + # +=======+=======+=======+========+========+ + # For boxes that are blank, __hash__ is untouched and therefore + # inherited from the base class. If the base is object, then + # id-based hashing is used. + + The Python implementation creates a tuple of all the fields, then hashes them. + This implementation creates a tuple of all the hashes of all the fields and hashes that. + The reason for this slight difference is to avoid to-Python conversions for anything + that Cython knows how to hash directly (It doesn't look like this currently applies to + anything though...). + """ + + hash_entry = node.scope.lookup_here("__hash__") + if hash_entry: + # TODO ideally assignment of __hash__ to None shouldn't trigger this + # but difficult to get the right information here + if unsafe_hash: + # error message taken from CPython dataclasses module + error(node.pos, "Cannot overwrite attribute __hash__ in class %s" % node.class_name) + return + + if not unsafe_hash: + if not eq: + return + if not frozen: + code.add_extra_statements([ + Nodes.SingleAssignmentNode( + node.pos, + lhs=ExprNodes.NameNode(node.pos, name=EncodedString("__hash__")), + rhs=ExprNodes.NoneNode(node.pos), + ) + ]) + return + + names = [ + name for name, field in fields.items() + if not field.is_initvar and ( + field.compare.value if field.hash.value is None else field.hash.value) + ] + + # make a tuple of the hashes + hash_tuple_items = u", ".join(u"self.%s" % name for name in names) + if hash_tuple_items: + hash_tuple_items += u"," # ensure that one arg form is a tuple + + # if we're here we want to generate a hash + code.add_code_lines([ + "def __hash__(self):", + " return hash((%s))" % hash_tuple_items, + ]) + + +def get_field_type(pos, entry): + """ + sets the .type attribute for a field + + Returns the annotation if possible (since this is what the dataclasses + module does). If not (for example, attributes defined with cdef) then + it creates a string fallback. + """ + if entry.annotation: + # Right now it doesn't look like cdef classes generate an + # __annotations__ dict, therefore it's safe to just return + # entry.annotation + # (TODO: remove .string if we ditch PEP563) + return entry.annotation.string + # If they do in future then we may need to look up into that + # to duplicating the node. The code below should do this: + #class_name_node = ExprNodes.NameNode(pos, name=entry.scope.name) + #annotations = ExprNodes.AttributeNode( + # pos, obj=class_name_node, + # attribute=EncodedString("__annotations__") + #) + #return ExprNodes.IndexNode( + # pos, base=annotations, + # index=ExprNodes.StringNode(pos, value=entry.name) + #) + else: + # it's slightly unclear what the best option is here - we could + # try to return PyType_Type. This case should only happen with + # attributes defined with cdef so Cython is free to make it's own + # decision + s = EncodedString(entry.type.declaration_code("", for_display=1)) + return ExprNodes.StringNode(pos, value=s) + + +class FieldRecordNode(ExprNodes.ExprNode): + """ + __dataclass_fields__ contains a bunch of field objects recording how each field + of the dataclass was initialized (mainly corresponding to the arguments passed to + the "field" function). This node is used for the attributes of these field objects. + + If possible, coerces `arg` to a Python object. + Otherwise, generates a sensible backup string. + """ + subexprs = ['arg'] + + def __init__(self, pos, arg): + super(FieldRecordNode, self).__init__(pos, arg=arg) + + def analyse_types(self, env): + self.arg.analyse_types(env) + self.type = self.arg.type + return self + + def coerce_to_pyobject(self, env): + if self.arg.type.can_coerce_to_pyobject(env): + return self.arg.coerce_to_pyobject(env) + else: + # A string representation of the code that gave the field seems like a reasonable + # fallback. This'll mostly happen for "default" and "default_factory" where the + # type may be a C-type that can't be converted to Python. + return self._make_string() + + def _make_string(self): + from .AutoDocTransforms import AnnotationWriter + writer = AnnotationWriter(description="Dataclass field") + string = writer.write(self.arg) + return ExprNodes.StringNode(self.pos, value=EncodedString(string)) + + def generate_evaluation_code(self, code): + return self.arg.generate_evaluation_code(code) + + +def _set_up_dataclass_fields(node, fields, dataclass_module): + # For defaults and default_factories containing things like lambda, + # they're already declared in the class scope, and it creates a big + # problem if multiple copies are floating around in both the __init__ + # function, and in the __dataclass_fields__ structure. + # Therefore, create module-level constants holding these values and + # pass those around instead + # + # If possible we use the `Field` class defined in the standard library + # module so that the information stored here is as close to a regular + # dataclass as is possible. + variables_assignment_stats = [] + for name, field in fields.items(): + if field.private: + continue # doesn't appear in the public interface + for attrname in [ "default", "default_factory" ]: + field_default = getattr(field, attrname) + if field_default is MISSING or field_default.is_literal or field_default.is_name: + # some simple cases where we don't need to set up + # the variable as a module-level constant + continue + global_scope = node.scope.global_scope() + module_field_name = global_scope.mangle( + global_scope.mangle(Naming.dataclass_field_default_cname, node.class_name), + name) + # create an entry in the global scope for this variable to live + field_node = ExprNodes.NameNode(field_default.pos, name=EncodedString(module_field_name)) + field_node.entry = global_scope.declare_var( + field_node.name, type=field_default.type or PyrexTypes.unspecified_type, + pos=field_default.pos, cname=field_node.name, is_cdef=True, + # TODO: do we need to set 'pytyping_modifiers' here? + ) + # replace the field so that future users just receive the namenode + setattr(field, attrname, field_node) + + variables_assignment_stats.append( + Nodes.SingleAssignmentNode(field_default.pos, lhs=field_node, rhs=field_default)) + + placeholders = {} + field_func = ExprNodes.AttributeNode(node.pos, obj=dataclass_module, + attribute=EncodedString("field")) + dc_fields = ExprNodes.DictNode(node.pos, key_value_pairs=[]) + dc_fields_namevalue_assignments = [] + + for name, field in fields.items(): + if field.private: + continue # doesn't appear in the public interface + type_placeholder_name = "PLACEHOLDER_%s" % name + placeholders[type_placeholder_name] = get_field_type( + node.pos, node.scope.entries[name] + ) + + # defining these make the fields introspect more like a Python dataclass + field_type_placeholder_name = "PLACEHOLDER_FIELD_TYPE_%s" % name + if field.is_initvar: + placeholders[field_type_placeholder_name] = ExprNodes.AttributeNode( + node.pos, obj=dataclass_module, + attribute=EncodedString("_FIELD_INITVAR") + ) + elif field.is_classvar: + # TODO - currently this isn't triggered + placeholders[field_type_placeholder_name] = ExprNodes.AttributeNode( + node.pos, obj=dataclass_module, + attribute=EncodedString("_FIELD_CLASSVAR") + ) + else: + placeholders[field_type_placeholder_name] = ExprNodes.AttributeNode( + node.pos, obj=dataclass_module, + attribute=EncodedString("_FIELD") + ) + + dc_field_keywords = ExprNodes.DictNode.from_pairs( + node.pos, + [(ExprNodes.IdentifierStringNode(node.pos, value=EncodedString(k)), + FieldRecordNode(node.pos, arg=v)) + for k, v in field.iterate_record_node_arguments()] + + ) + dc_field_call = make_dataclass_call_helper( + node.pos, field_func, dc_field_keywords + ) + dc_fields.key_value_pairs.append( + ExprNodes.DictItemNode( + node.pos, + key=ExprNodes.IdentifierStringNode(node.pos, value=EncodedString(name)), + value=dc_field_call)) + dc_fields_namevalue_assignments.append( + dedent(u"""\ + __dataclass_fields__[{0!r}].name = {0!r} + __dataclass_fields__[{0!r}].type = {1} + __dataclass_fields__[{0!r}]._field_type = {2} + """).format(name, type_placeholder_name, field_type_placeholder_name)) + + dataclass_fields_assignment = \ + Nodes.SingleAssignmentNode(node.pos, + lhs = ExprNodes.NameNode(node.pos, + name=EncodedString("__dataclass_fields__")), + rhs = dc_fields) + + dc_fields_namevalue_assignments = u"\n".join(dc_fields_namevalue_assignments) + dc_fields_namevalue_assignments = TreeFragment(dc_fields_namevalue_assignments, + level="c_class", + pipeline=[NormalizeTree(None)]) + dc_fields_namevalue_assignments = dc_fields_namevalue_assignments.substitute(placeholders) + + return (variables_assignment_stats + + [dataclass_fields_assignment] + + dc_fields_namevalue_assignments.stats) diff --git a/Cython/Compiler/Errors.py b/Cython/Compiler/Errors.py index 9761b52c3..bde320732 100644 --- a/Cython/Compiler/Errors.py +++ b/Cython/Compiler/Errors.py @@ -12,6 +12,13 @@ except ImportError: import sys from contextlib import contextmanager +try: + from threading import local as _threadlocal +except ImportError: + class _threadlocal(object): pass + +threadlocal = _threadlocal() + from ..Utils import open_new_file from . import DebugFlags from . import Options @@ -24,6 +31,8 @@ class PyrexError(Exception): class PyrexWarning(Exception): pass +class CannotSpecialize(PyrexError): + pass def context(position): source = position[0] @@ -36,7 +45,7 @@ def context(position): s = u"[unprintable code]\n" else: s = u''.join(F[max(0, position[1]-6):position[1]]) - s = u'...\n%s%s^\n' % (s, u' '*(position[2]-1)) + s = u'...\n%s%s^\n' % (s, u' '*(position[2])) s = u'%s\n%s%s\n' % (u'-'*60, s, u'-'*60) return s @@ -60,11 +69,9 @@ class CompileError(PyrexError): self.message_only = message self.formatted_message = format_error(message, position) self.reported = False - # Deprecated and withdrawn in 2.6: - # self.message = message Exception.__init__(self, self.formatted_message) # Python Exception subclass pickling is broken, - # see http://bugs.python.org/issue1692335 + # see https://bugs.python.org/issue1692335 self.args = (position, message) def __str__(self): @@ -74,8 +81,6 @@ class CompileWarning(PyrexWarning): def __init__(self, position = None, message = ""): self.position = position - # Deprecated and withdrawn in 2.6: - # self.message = message Exception.__init__(self, format_position(position) + message) class InternalError(Exception): @@ -114,7 +119,7 @@ class CompilerCrash(CompileError): message += u'%s: %s' % (cause.__class__.__name__, cause) CompileError.__init__(self, pos, message) # Python Exception subclass pickling is broken, - # see http://bugs.python.org/issue1692335 + # see https://bugs.python.org/issue1692335 self.args = (pos, context, message, cause, stacktrace) class NoElementTreeInstalledException(PyrexError): @@ -122,35 +127,29 @@ class NoElementTreeInstalledException(PyrexError): implementation was found """ -listing_file = None -num_errors = 0 -echo_file = None - -def open_listing_file(path, echo_to_stderr = 1): +def open_listing_file(path, echo_to_stderr=True): # Begin a new error listing. If path is None, no file # is opened, the error counter is just reset. - global listing_file, num_errors, echo_file if path is not None: - listing_file = open_new_file(path) + threadlocal.cython_errors_listing_file = open_new_file(path) else: - listing_file = None + threadlocal.cython_errors_listing_file = None if echo_to_stderr: - echo_file = sys.stderr + threadlocal.cython_errors_echo_file = sys.stderr else: - echo_file = None - num_errors = 0 + threadlocal.cython_errors_echo_file = None + threadlocal.cython_errors_count = 0 def close_listing_file(): - global listing_file - if listing_file: - listing_file.close() - listing_file = None + if threadlocal.cython_errors_listing_file: + threadlocal.cython_errors_listing_file.close() + threadlocal.cython_errors_listing_file = None def report_error(err, use_stack=True): + error_stack = threadlocal.cython_errors_stack if error_stack and use_stack: error_stack[-1].append(err) else: - global num_errors # See Main.py for why dual reporting occurs. Quick fix for now. if err.reported: return err.reported = True @@ -159,41 +158,50 @@ def report_error(err, use_stack=True): # Python <= 2.5 does this for non-ASCII Unicode exceptions line = format_error(getattr(err, 'message_only', "[unprintable exception message]"), getattr(err, 'position', None)) + u'\n' + listing_file = threadlocal.cython_errors_listing_file if listing_file: try: listing_file.write(line) except UnicodeEncodeError: listing_file.write(line.encode('ASCII', 'replace')) + echo_file = threadlocal.cython_errors_echo_file if echo_file: try: echo_file.write(line) except UnicodeEncodeError: echo_file.write(line.encode('ASCII', 'replace')) - num_errors += 1 + threadlocal.cython_errors_count += 1 if Options.fast_fail: raise AbortError("fatal errors") - def error(position, message): #print("Errors.error:", repr(position), repr(message)) ### if position is None: raise InternalError(message) err = CompileError(position, message) - if DebugFlags.debug_exception_on_error: raise Exception(err) # debug + if DebugFlags.debug_exception_on_error: raise Exception(err) # debug report_error(err) return err -LEVEL = 1 # warn about all errors level 1 or higher +LEVEL = 1 # warn about all errors level 1 or higher + +def _write_file_encode(file, line): + try: + file.write(line) + except UnicodeEncodeError: + file.write(line.encode('ascii', 'replace')) def message(position, message, level=1): if level < LEVEL: return warn = CompileWarning(position, message) - line = "note: %s\n" % warn + line = u"note: %s\n" % warn + listing_file = threadlocal.cython_errors_listing_file if listing_file: - listing_file.write(line) + _write_file_encode(listing_file, line) + echo_file = threadlocal.cython_errors_echo_file if echo_file: - echo_file.write(line) + _write_file_encode(echo_file, line) return warn @@ -203,63 +211,76 @@ def warning(position, message, level=0): if Options.warning_errors and position: return error(position, message) warn = CompileWarning(position, message) - line = "warning: %s\n" % warn + line = u"warning: %s\n" % warn + listing_file = threadlocal.cython_errors_listing_file if listing_file: - listing_file.write(line) + _write_file_encode(listing_file, line) + echo_file = threadlocal.cython_errors_echo_file if echo_file: - echo_file.write(line) + _write_file_encode(echo_file, line) return warn -_warn_once_seen = {} def warn_once(position, message, level=0): - if level < LEVEL or message in _warn_once_seen: + if level < LEVEL: + return + warn_once_seen = threadlocal.cython_errors_warn_once_seen + if message in warn_once_seen: return warn = CompileWarning(position, message) - line = "warning: %s\n" % warn + line = u"warning: %s\n" % warn + listing_file = threadlocal.cython_errors_listing_file if listing_file: - listing_file.write(line) + _write_file_encode(listing_file, line) + echo_file = threadlocal.cython_errors_echo_file if echo_file: - echo_file.write(line) - _warn_once_seen[message] = True + _write_file_encode(echo_file, line) + warn_once_seen[message] = True return warn # These functions can be used to momentarily suppress errors. -error_stack = [] - - def hold_errors(): - error_stack.append([]) + errors = [] + threadlocal.cython_errors_stack.append(errors) + return errors def release_errors(ignore=False): - held_errors = error_stack.pop() + held_errors = threadlocal.cython_errors_stack.pop() if not ignore: for err in held_errors: report_error(err) def held_errors(): - return error_stack[-1] + return threadlocal.cython_errors_stack[-1] # same as context manager: @contextmanager def local_errors(ignore=False): - errors = [] - error_stack.append(errors) + errors = hold_errors() try: yield errors finally: release_errors(ignore=ignore) -# this module needs a redesign to support parallel cythonisation, but -# for now, the following works at least in sequential compiler runs +# Keep all global state in thread local storage to support parallel cythonisation in distutils. + +def init_thread(): + threadlocal.cython_errors_count = 0 + threadlocal.cython_errors_listing_file = None + threadlocal.cython_errors_echo_file = None + threadlocal.cython_errors_warn_once_seen = set() + threadlocal.cython_errors_stack = [] def reset(): - _warn_once_seen.clear() - del error_stack[:] + threadlocal.cython_errors_warn_once_seen.clear() + del threadlocal.cython_errors_stack[:] + +def get_errors_count(): + return threadlocal.cython_errors_count diff --git a/Cython/Compiler/ExprNodes.py b/Cython/Compiler/ExprNodes.py index 305eae9eb..242a97d6a 100644 --- a/Cython/Compiler/ExprNodes.py +++ b/Cython/Compiler/ExprNodes.py @@ -13,7 +13,8 @@ cython.declare(error=object, warning=object, warn_once=object, InternalError=obj unicode_type=object, str_type=object, bytes_type=object, type_type=object, Builtin=object, Symtab=object, Utils=object, find_coercion_error=object, debug_disposal_code=object, debug_temp_alloc=object, debug_coercion=object, - bytearray_type=object, slice_type=object, _py_int_types=object, + bytearray_type=object, slice_type=object, memoryview_type=object, + builtin_sequence_types=object, _py_int_types=object, IS_PYTHON3=cython.bint) import re @@ -23,26 +24,30 @@ import os.path import operator from .Errors import ( - error, warning, InternalError, CompileError, report_error, local_errors) + error, warning, InternalError, CompileError, report_error, local_errors, + CannotSpecialize) from .Code import UtilityCode, TempitaUtilityCode from . import StringEncoding from . import Naming from . import Nodes -from .Nodes import Node, utility_code_for_imports, analyse_type_annotation +from .Nodes import Node, utility_code_for_imports, SingleAssignmentNode from . import PyrexTypes -from .PyrexTypes import py_object_type, c_long_type, typecast, error_type, \ +from .PyrexTypes import py_object_type, typecast, error_type, \ unspecified_type from . import TypeSlots -from .Builtin import list_type, tuple_type, set_type, dict_type, type_type, \ - unicode_type, str_type, bytes_type, bytearray_type, basestring_type, slice_type +from .Builtin import ( + list_type, tuple_type, set_type, dict_type, type_type, + unicode_type, str_type, bytes_type, bytearray_type, basestring_type, + slice_type, long_type, sequence_types as builtin_sequence_types, memoryview_type, +) from . import Builtin from . import Symtab from .. import Utils from .Annotate import AnnotationItem from . import Future from ..Debugging import print_call_chain -from .DebugFlags import debug_disposal_code, debug_temp_alloc, \ - debug_coercion +from .DebugFlags import debug_disposal_code, debug_coercion + from .Pythran import (to_pythran, is_pythran_supported_type, is_pythran_supported_operation_type, is_pythran_expr, pythran_func_type, pythran_binop_type, pythran_unaryop_type, has_np_pythran, pythran_indexing_code, pythran_indexing_type, is_pythran_supported_node_or_none, pythran_type, @@ -182,7 +187,7 @@ def infer_sequence_item_type(env, seq_node, index_node=None, seq_type=None): else: return item.infer_type(env) # if we're lucky, all items have the same type - item_types = set([item.infer_type(env) for item in seq_node.args]) + item_types = {item.infer_type(env) for item in seq_node.args} if len(item_types) == 1: return item_types.pop() return None @@ -205,6 +210,11 @@ def make_dedup_key(outer_type, item_nodes): # For constants, look at the Python value type if we don't know the concrete Cython type. else (node.type, node.constant_result, type(node.constant_result) if node.type is py_object_type else None) if node.has_constant_result() + # IdentifierStringNode doesn't usually have a "constant_result" set because: + # 1. it doesn't usually have unicode_value + # 2. it's often created later in the compilation process after ConstantFolding + # but should be cacheable + else (node.type, node.value, node.unicode_value, "IdentifierStringNode") if isinstance(node, IdentifierStringNode) else None # something we cannot handle => short-circuit below for node in item_nodes ] @@ -237,19 +247,23 @@ def get_exception_handler(exception_value): exception_value.entry.cname), False) + def maybe_check_py_error(code, check_py_exception, pos, nogil): if check_py_exception: if nogil: + code.globalstate.use_utility_code( + UtilityCode.load_cached("ErrOccurredWithGIL", "Exceptions.c")) code.putln(code.error_goto_if("__Pyx_ErrOccurredWithGIL()", pos)) else: code.putln(code.error_goto_if("PyErr_Occurred()", pos)) + def translate_cpp_exception(code, pos, inside, py_result, exception_value, nogil): raise_py_exception, check_py_exception = get_exception_handler(exception_value) code.putln("try {") code.putln("%s" % inside) if py_result: - code.putln(code.error_goto_if_null(py_result, pos)) + code.putln(code.error_goto_if_null(py_result, pos)) maybe_check_py_error(code, check_py_exception, pos, nogil) code.putln("} catch(...) {") if nogil: @@ -260,10 +274,24 @@ def translate_cpp_exception(code, pos, inside, py_result, exception_value, nogil code.putln(code.error_goto(pos)) code.putln("}") +def needs_cpp_exception_conversion(node): + assert node.exception_check == "+" + if node.exception_value is None: + return True + # exception_value can be a NameNode + # (in which case it's used as a handler function and no conversion is needed) + if node.exception_value.is_name: + return False + # or a CharNode with a value of "*" + if isinstance(node.exception_value, CharNode) and node.exception_value.value == "*": + return True + # Most other const-nodes are disallowed after "+" by the parser + return False + + # Used to handle the case where an lvalue expression and an overloaded assignment # both have an exception declaration. -def translate_double_cpp_exception(code, pos, lhs_type, lhs_code, rhs_code, - lhs_exc_val, assign_exc_val, nogil): +def translate_double_cpp_exception(code, pos, lhs_type, lhs_code, rhs_code, lhs_exc_val, assign_exc_val, nogil): handle_lhs_exc, lhc_check_py_exc = get_exception_handler(lhs_exc_val) handle_assignment_exc, assignment_check_py_exc = get_exception_handler(assign_exc_val) code.putln("try {") @@ -301,23 +329,23 @@ class ExprNode(Node): # is_sequence_constructor # boolean Is a list or tuple constructor expression # is_starred boolean Is a starred expression (e.g. '*a') - # saved_subexpr_nodes - # [ExprNode or [ExprNode or None] or None] - # Cached result of subexpr_nodes() # use_managed_ref boolean use ref-counted temps/assignments/etc. # result_is_used boolean indicates that the result will be dropped and the # result_code/temp_result can safely be set to None # is_numpy_attribute boolean Is a Numpy module attribute # annotation ExprNode or None PEP526 annotation for names or expressions + # generator_arg_tag None or Node A tag to mark ExprNodes that potentially need to + # be changed to a generator argument result_ctype = None type = None annotation = None temp_code = None - old_temp = None # error checker for multiple frees etc. - use_managed_ref = True # can be set by optimisation transforms + old_temp = None # error checker for multiple frees etc. + use_managed_ref = True # can be set by optimisation transforms result_is_used = True is_numpy_attribute = False + generator_arg_tag = None # The Analyse Expressions phase for expressions is split # into two sub-phases: @@ -446,8 +474,8 @@ class ExprNode(Node): is_memview_broadcast = False is_memview_copy_assignment = False - saved_subexpr_nodes = None is_temp = False + has_temp_moved = False # if True then attempting to do anything but free the temp is invalid is_target = False is_starred = False @@ -455,11 +483,13 @@ class ExprNode(Node): child_attrs = property(fget=operator.attrgetter('subexprs')) + def analyse_annotations(self, env): + pass + def not_implemented(self, method_name): - print_call_chain(method_name, "not implemented") ### + print_call_chain(method_name, "not implemented") raise InternalError( - "%s.%s not implemented" % - (self.__class__.__name__, method_name)) + "%s.%s not implemented" % (self.__class__.__name__, method_name)) def is_lvalue(self): return 0 @@ -498,11 +528,27 @@ class ExprNode(Node): else: return self.calculate_result_code() + def _make_move_result_rhs(self, result, optional=False): + if optional and not (self.is_temp and self.type.is_cpp_class and not self.type.is_reference): + return result + self.has_temp_moved = True + return "{}({})".format("__PYX_STD_MOVE_IF_SUPPORTED" if optional else "std::move", result) + + def move_result_rhs(self): + return self._make_move_result_rhs(self.result(), optional=True) + + def move_result_rhs_as(self, type): + result = self.result_as(type) + if not (type.is_reference or type.needs_refcounting): + requires_move = type.is_rvalue_reference and self.is_temp + result = self._make_move_result_rhs(result, optional=not requires_move) + return result + def pythran_result(self, type_=None): if is_pythran_supported_node_or_none(self): return to_pythran(self) - assert(type_ is not None) + assert type_ is not None return to_pythran(self, type_) def is_c_result_required(self): @@ -569,6 +615,9 @@ class ExprNode(Node): def analyse_target_declaration(self, env): error(self.pos, "Cannot assign to or delete this") + def analyse_assignment_expression_target_declaration(self, env): + error(self.pos, "Cannot use anything except a name in an assignment expression") + # ------------- Expression Analysis ---------------- def analyse_const_expression(self, env): @@ -614,7 +663,7 @@ class ExprNode(Node): def type_dependencies(self, env): # Returns the list of entries whose types must be determined # before the type of self can be inferred. - if hasattr(self, 'type') and self.type is not None: + if getattr(self, 'type', None) is not None: return () return sum([node.type_dependencies(env) for node in self.subexpr_nodes()], ()) @@ -623,12 +672,13 @@ class ExprNode(Node): # Differs from analyse_types as it avoids unnecessary # analysis of subexpressions, but can assume everything # in self.type_dependencies() has been resolved. - if hasattr(self, 'type') and self.type is not None: - return self.type - elif hasattr(self, 'entry') and self.entry is not None: - return self.entry.type - else: - self.not_implemented("infer_type") + type = getattr(self, 'type', None) + if type is not None: + return type + entry = getattr(self, 'entry', None) + if entry is not None: + return entry.type + self.not_implemented("infer_type") def nonlocally_immutable(self): # Returns whether this variable is a safe reference, i.e. @@ -655,6 +705,19 @@ class ExprNode(Node): # type, return that type, else None. return None + def analyse_as_specialized_type(self, env): + type = self.analyse_as_type(env) + if type and type.is_fused and env.fused_to_specific: + # while it would be nice to test "if entry.type in env.fused_to_specific" + # rather than try/catch this doesn't work reliably (mainly for nested fused types) + try: + return type.specialize(env.fused_to_specific) + except KeyError: + pass + if type and type.is_fused: + error(self.pos, "Type is not specific") + return type + def analyse_as_extension_type(self, env): # If this node can be interpreted as a reference to an # extension type or builtin type, return its type, else None. @@ -746,19 +809,20 @@ class ExprNode(Node): def make_owned_reference(self, code): """ - If result is a pyobject, make sure we own a reference to it. + Make sure we own a reference to result. If the result is in a temp, it is already a new reference. """ - if self.type.is_pyobject and not self.result_in_temp(): + if not self.result_in_temp(): code.put_incref(self.result(), self.ctype()) def make_owned_memoryviewslice(self, code): """ Make sure we own the reference to this memoryview slice. """ + # TODO ideally this would be shared with "make_owned_reference" if not self.result_in_temp(): - code.put_incref_memoryviewslice(self.result(), - have_gil=self.in_nogil_context) + code.put_incref_memoryviewslice(self.result(), self.type, + have_gil=not self.in_nogil_context) def generate_evaluation_code(self, code): # Generate code to evaluate this node and @@ -785,19 +849,17 @@ class ExprNode(Node): self.not_implemented("generate_result_code") def generate_disposal_code(self, code): + if self.has_temp_moved: + code.globalstate.use_utility_code( + UtilityCode.load_cached("MoveIfSupported", "CppSupport.cpp")) if self.is_temp: if self.type.is_string or self.type.is_pyunicode_ptr: # postponed from self.generate_evaluation_code() self.generate_subexpr_disposal_code(code) self.free_subexpr_temps(code) if self.result(): - if self.type.is_pyobject: - code.put_decref_clear(self.result(), self.ctype()) - elif self.type.is_memoryviewslice: - code.put_xdecref_memoryviewslice( - self.result(), have_gil=not self.in_nogil_context) - code.putln("%s.memview = NULL;" % self.result()) - code.putln("%s.data = NULL;" % self.result()) + code.put_decref_clear(self.result(), self.ctype(), + have_gil=not self.in_nogil_context) else: # Already done if self.is_temp self.generate_subexpr_disposal_code(code) @@ -819,11 +881,15 @@ class ExprNode(Node): elif self.type.is_memoryviewslice: code.putln("%s.memview = NULL;" % self.result()) code.putln("%s.data = NULL;" % self.result()) + + if self.has_temp_moved: + code.globalstate.use_utility_code( + UtilityCode.load_cached("MoveIfSupported", "CppSupport.cpp")) else: self.generate_subexpr_disposal_code(code) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): # Stub method for nodes which are not legal as # the LHS of an assignment. An error will have # been reported earlier. @@ -849,6 +915,32 @@ class ExprNode(Node): def generate_function_definitions(self, env, code): pass + # ----Generation of small bits of reference counting -- + + def generate_decref_set(self, code, rhs): + code.put_decref_set(self.result(), self.ctype(), rhs) + + def generate_xdecref_set(self, code, rhs): + code.put_xdecref_set(self.result(), self.ctype(), rhs) + + def generate_gotref(self, code, handle_null=False, + maybe_null_extra_check=True): + if not (handle_null and self.cf_is_null): + if (handle_null and self.cf_maybe_null + and maybe_null_extra_check): + self.generate_xgotref(code) + else: + code.put_gotref(self.result(), self.ctype()) + + def generate_xgotref(self, code): + code.put_xgotref(self.result(), self.ctype()) + + def generate_giveref(self, code): + code.put_giveref(self.result(), self.ctype()) + + def generate_xgiveref(self, code): + code.put_xgiveref(self.result(), self.ctype()) + # ---------------- Annotation --------------------- def annotate(self, code): @@ -883,8 +975,8 @@ class ExprNode(Node): if used_as_reference and not src_type.is_reference: dst_type = dst_type.ref_base_type - if src_type.is_const: - src_type = src_type.const_base_type + if src_type.is_cv_qualified: + src_type = src_type.cv_base_type if src_type.is_fused or dst_type.is_fused: # See if we are coercing a fused function to a pointer to a @@ -970,7 +1062,12 @@ class ExprNode(Node): and src_type != dst_type and dst_type.assignable_from(src_type)): src = CoerceToComplexNode(src, dst_type, env) - else: # neither src nor dst are py types + elif (src_type is PyrexTypes.soft_complex_type + and src_type != dst_type + and not dst_type.assignable_from(src_type)): + src = coerce_from_soft_complex(src, dst_type, env) + else: + # neither src nor dst are py types # Added the string comparison, since for c types that # is enough, but Cython gets confused when the types are # in different pxi files. @@ -1010,6 +1107,8 @@ class ExprNode(Node): type = self.type if type.is_enum or type.is_error: return self + elif type is PyrexTypes.c_bint_type: + return self elif type.is_pyobject or type.is_int or type.is_ptr or type.is_float: return CoerceToBooleanNode(self, env) elif type.is_cpp_class and type.scope and type.scope.lookup("operator bool"): @@ -1090,6 +1189,15 @@ class ExprNode(Node): kwargs[attr_name] = value return cls(node.pos, **kwargs) + def get_known_standard_library_import(self): + """ + Gets the module.path that this node was imported from. + + Many nodes do not have one, or it is ambiguous, in which case + this function returns a false value. + """ + return None + class AtomicExprNode(ExprNode): # Abstract base class for expression nodes which have @@ -1108,6 +1216,7 @@ class PyConstNode(AtomicExprNode): is_literal = 1 type = py_object_type + nogil_check = None def is_simple(self): return 1 @@ -1133,8 +1242,6 @@ class NoneNode(PyConstNode): constant_result = None - nogil_check = None - def compile_time_value(self, denv): return None @@ -1204,7 +1311,7 @@ class BoolNode(ConstNode): def calculate_result_code(self): if self.type.is_pyobject: - return self.value and 'Py_True' or 'Py_False' + return 'Py_True' if self.value else 'Py_False' else: return str(int(self.value)) @@ -1256,7 +1363,7 @@ class IntNode(ConstNode): unsigned = "" longness = "" - is_c_literal = None # unknown + is_c_literal = None # unknown def __init__(self, pos, **kwds): ExprNode.__init__(self, pos, **kwds) @@ -1429,18 +1536,26 @@ class FloatNode(ConstNode): def _analyse_name_as_type(name, pos, env): - type = PyrexTypes.parse_basic_type(name) - if type is not None: - return type - - global_entry = env.global_scope().lookup(name) - if global_entry and global_entry.type and ( - global_entry.type.is_extension_type - or global_entry.type.is_struct_or_union - or global_entry.type.is_builtin_type - or global_entry.type.is_cpp_class): - return global_entry.type + ctype = PyrexTypes.parse_basic_type(name) + if ctype is not None and env.in_c_type_context: + return ctype + + global_scope = env.global_scope() + global_entry = global_scope.lookup(name) + if global_entry and global_entry.is_type: + type = global_entry.type + if (not env.in_c_type_context + and type is Builtin.int_type + and global_scope.context.language_level == 2): + # While we still support Python2 this needs to be downgraded + # to a generic Python object to include both int and long. + # With language_level > 3, we keep the type but also accept 'long' in Py2. + type = py_object_type + if type and (type.is_pyobject or env.in_c_type_context): + return type + ctype = ctype or type + # This is fairly heavy, so it's worth trying some easier things above. from .TreeFragment import TreeFragment with local_errors(ignore=True): pos = (pos[0], pos[1], pos[2]-7) @@ -1453,8 +1568,11 @@ def _analyse_name_as_type(name, pos, env): if isinstance(sizeof_node, SizeofTypeNode): sizeof_node = sizeof_node.analyse_types(env) if isinstance(sizeof_node, SizeofTypeNode): - return sizeof_node.arg_type - return None + type = sizeof_node.arg_type + if type and (type.is_pyobject or env.in_c_type_context): + return type + ctype = ctype or type + return ctype class BytesNode(ConstNode): @@ -1539,7 +1657,7 @@ class BytesNode(ConstNode): self.result_code = result def get_constant_c_result_code(self): - return None # FIXME + return None # FIXME def calculate_result_code(self): return self.result_code @@ -1594,12 +1712,9 @@ class UnicodeNode(ConstNode): if dst_type.is_string and self.bytes_value is not None: # special case: '-3' enforced unicode literal used in a # C char* context - return BytesNode(self.pos, value=self.bytes_value - ).coerce_to(dst_type, env) + return BytesNode(self.pos, value=self.bytes_value).coerce_to(dst_type, env) if dst_type.is_pyunicode_ptr: - node = UnicodeNode(self.pos, value=self.value) - node.type = dst_type - return node + return UnicodeNode(self.pos, value=self.value, type=dst_type) error(self.pos, "Unicode literals do not support coercion to C types other " "than Py_UNICODE/Py_UCS4 (for characters) or Py_UNICODE* " @@ -1784,7 +1899,7 @@ class ImagNode(AtomicExprNode): self.result(), float(self.value), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class NewExprNode(AtomicExprNode): @@ -1837,7 +1952,7 @@ class NameNode(AtomicExprNode): is_name = True is_cython_module = False cython_attribute = None - lhs_of_first_assignment = False # TODO: remove me + lhs_of_first_assignment = False # TODO: remove me is_used_as_rvalue = 0 entry = None type_entry = None @@ -1919,30 +2034,66 @@ class NameNode(AtomicExprNode): def declare_from_annotation(self, env, as_target=False): """Implements PEP 526 annotation typing in a fairly relaxed way. - Annotations are ignored for global variables, Python class attributes and already declared variables. - String literals are allowed and ignored. - The ambiguous Python types 'int' and 'long' are ignored and the 'cython.int' form must be used instead. + Annotations are ignored for global variables. + All other annotations are stored on the entry in the symbol table. + String literals are allowed and not evaluated. + The ambiguous Python types 'int' and 'long' are not evaluated - the 'cython.int' form must be used instead. """ - if not env.directives['annotation_typing']: - return - if env.is_module_scope or env.is_py_class_scope: - # annotations never create global cdef names and Python classes don't support them anyway - return name = self.name - if self.entry or env.lookup_here(name) is not None: - # already declared => ignore annotation - return - annotation = self.annotation - if annotation.is_string_literal: - # name: "description" => not a type, but still a declared variable or attribute - atype = None - else: - _, atype = analyse_type_annotation(annotation, env) - if atype is None: - atype = unspecified_type if as_target and env.directives['infer_types'] != False else py_object_type - self.entry = env.declare_var(name, atype, self.pos, is_cdef=not as_target) - self.entry.annotation = annotation + entry = self.entry or env.lookup_here(name) + if not entry: + # annotations never create global cdef names + if env.is_module_scope: + return + + modifiers = () + if ( + # name: "description" => not a type, but still a declared variable or attribute + annotation.expr.is_string_literal + # don't do type analysis from annotations if not asked to, but still collect the annotation + or not env.directives['annotation_typing'] + ): + atype = None + elif env.is_py_class_scope: + # For Python class scopes every attribute is a Python object + atype = py_object_type + else: + modifiers, atype = annotation.analyse_type_annotation(env) + + if atype is None: + atype = unspecified_type if as_target and env.directives['infer_types'] != False else py_object_type + elif atype.is_fused and env.fused_to_specific: + try: + atype = atype.specialize(env.fused_to_specific) + except CannotSpecialize: + error(self.pos, + "'%s' cannot be specialized since its type is not a fused argument to this function" % + self.name) + atype = error_type + + visibility = 'private' + if env.is_c_dataclass_scope: + # handle "frozen" directive - full inspection of the dataclass directives happens + # in Dataclass.py + is_frozen = env.is_c_dataclass_scope == "frozen" + if atype.is_pyobject or atype.can_coerce_to_pyobject(env): + visibility = 'readonly' if is_frozen else 'public' + # If the object can't be coerced that's fine - we just don't create a property + + if as_target and env.is_c_class_scope and not (atype.is_pyobject or atype.is_error): + # TODO: this will need revising slightly if annotated cdef attributes are implemented + atype = py_object_type + warning(annotation.pos, "Annotation ignored since class-level attributes must be Python objects. " + "Were you trying to set up an instance attribute?", 2) + + entry = self.entry = env.declare_var( + name, atype, self.pos, is_cdef=not as_target, visibility=visibility, + pytyping_modifiers=modifiers) + + # Even if the entry already exists, make sure we're supplying an annotation if we can. + if annotation and not entry.annotation: + entry.annotation = annotation def analyse_as_module(self, env): # Try to interpret this as a reference to a cimported module. @@ -1952,22 +2103,50 @@ class NameNode(AtomicExprNode): entry = env.lookup(self.name) if entry and entry.as_module: return entry.as_module + if entry and entry.known_standard_library_import: + scope = Builtin.get_known_standard_library_module_scope(entry.known_standard_library_import) + if scope and scope.is_module_scope: + return scope return None def analyse_as_type(self, env): + type = None if self.cython_attribute: type = PyrexTypes.parse_basic_type(self.cython_attribute) - else: + elif env.in_c_type_context: type = PyrexTypes.parse_basic_type(self.name) if type: return type + entry = self.entry if not entry: entry = env.lookup(self.name) + if entry and not entry.is_type and entry.known_standard_library_import: + entry = Builtin.get_known_standard_library_entry(entry.known_standard_library_import) if entry and entry.is_type: - return entry.type - else: - return None + # Infer equivalent C types instead of Python types when possible. + type = entry.type + if not env.in_c_type_context and type is Builtin.long_type: + # Try to give a helpful warning when users write plain C type names. + warning(self.pos, "Found Python 2.x type 'long' in a Python annotation. Did you mean to use 'cython.long'?") + type = py_object_type + elif type.is_pyobject and type.equivalent_type: + type = type.equivalent_type + elif type is Builtin.int_type and env.global_scope().context.language_level == 2: + # While we still support Python 2 this must be a plain object + # so that it can be either int or long. With language_level=3(str), + # we pick up the type but accept both int and long in Py2. + type = py_object_type + return type + if self.name == 'object': + # This is normally parsed as "simple C type", but not if we don't parse C types. + return py_object_type + + # Try to give a helpful warning when users write plain C type names. + if not env.in_c_type_context and PyrexTypes.parse_basic_type(self.name): + warning(self.pos, "Found C type '%s' in a Python annotation. Did you mean to use 'cython.%s'?" % (self.name, self.name)) + + return None def analyse_as_extension_type(self, env): # Try to interpret this as a reference to an extension type. @@ -1981,11 +2160,29 @@ class NameNode(AtomicExprNode): return None def analyse_target_declaration(self, env): + return self._analyse_target_declaration(env, is_assignment_expression=False) + + def analyse_assignment_expression_target_declaration(self, env): + return self._analyse_target_declaration(env, is_assignment_expression=True) + + def _analyse_target_declaration(self, env, is_assignment_expression): + self.is_target = True if not self.entry: - self.entry = env.lookup_here(self.name) + if is_assignment_expression: + self.entry = env.lookup_assignment_expression_target(self.name) + else: + self.entry = env.lookup_here(self.name) + if self.entry: + self.entry.known_standard_library_import = "" # already exists somewhere and so is now ambiguous if not self.entry and self.annotation is not None: # name : type = ... - self.declare_from_annotation(env, as_target=True) + is_dataclass = env.is_c_dataclass_scope + # In a dataclass, an assignment should not prevent a name from becoming an instance attribute. + # Hence, "as_target = not is_dataclass". + self.declare_from_annotation(env, as_target=not is_dataclass) + elif (self.entry and self.entry.is_inherited and + self.annotation and env.is_c_dataclass_scope): + error(self.pos, "Cannot redeclare inherited fields in Cython dataclasses") if not self.entry: if env.directives['warn.undeclared']: warning(self.pos, "implicit declaration of '%s'" % self.name, 1) @@ -1993,7 +2190,10 @@ class NameNode(AtomicExprNode): type = unspecified_type else: type = py_object_type - self.entry = env.declare_var(self.name, type, self.pos) + if is_assignment_expression: + self.entry = env.declare_assignment_expression_target(self.name, type, self.pos) + else: + self.entry = env.declare_var(self.name, type, self.pos) if self.entry.is_declared_generic: self.result_ctype = py_object_type if self.entry.as_module: @@ -2033,8 +2233,6 @@ class NameNode(AtomicExprNode): if self.type.is_const: error(self.pos, "Assignment to const '%s'" % self.name) - if self.type.is_reference: - error(self.pos, "Assignment to reference '%s'" % self.name) if not self.is_lvalue(): error(self.pos, "Assignment to non-lvalue '%s'" % self.name) self.type = PyrexTypes.error_type @@ -2071,7 +2269,7 @@ class NameNode(AtomicExprNode): if self.is_used_as_rvalue: entry = self.entry if entry.is_builtin: - if not entry.is_const: # cached builtins are ok + if not entry.is_const: # cached builtins are ok self.gil_error() elif entry.is_pyglobal: self.gil_error() @@ -2096,7 +2294,7 @@ class NameNode(AtomicExprNode): entry = self.entry if entry.is_type and entry.type.is_extension_type: self.type_entry = entry - if entry.is_type and entry.type.is_enum: + if entry.is_type and (entry.type.is_enum or entry.type.is_cpp_enum): py_entry = Symtab.Entry(self.name, None, py_object_type) py_entry.is_pyglobal = True py_entry.scope = self.entry.scope @@ -2122,7 +2320,7 @@ class NameNode(AtomicExprNode): def may_be_none(self): if self.cf_state and self.type and (self.type.is_pyobject or self.type.is_memoryviewslice): - # gard against infinite recursion on self-dependencies + # guard against infinite recursion on self-dependencies if getattr(self, '_none_checking', False): # self-dependency - either this node receives a None # value from *another* node, or it can not reference @@ -2189,24 +2387,25 @@ class NameNode(AtomicExprNode): def calculate_result_code(self): entry = self.entry if not entry: - return "<error>" # There was an error earlier + return "<error>" # There was an error earlier + if self.entry.is_cpp_optional and not self.is_target: + return "(*%s)" % entry.cname return entry.cname def generate_result_code(self, code): - assert hasattr(self, 'entry') entry = self.entry if entry is None: - return # There was an error earlier + return # There was an error earlier if entry.utility_code: code.globalstate.use_utility_code(entry.utility_code) if entry.is_builtin and entry.is_const: - return # Lookup already cached + return # Lookup already cached elif entry.is_pyclass_attr: assert entry.type.is_pyobject, "Python global or builtin not a Python object" interned_cname = code.intern_identifier(self.entry.name) if entry.is_builtin: namespace = Naming.builtins_cname - else: # entry.is_pyglobal + else: # entry.is_pyglobal namespace = entry.scope.namespace_cname if not self.cf_is_null: code.putln( @@ -2225,7 +2424,7 @@ class NameNode(AtomicExprNode): if not self.cf_is_null: code.putln("}") code.putln(code.error_goto_if_null(self.result(), self.pos)) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif entry.is_builtin and not entry.scope.is_module_scope: # known builtin @@ -2238,7 +2437,7 @@ class NameNode(AtomicExprNode): self.result(), interned_cname, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif entry.is_pyglobal or (entry.is_builtin and entry.scope.is_module_scope): # name in class body, global name or unknown builtin @@ -2262,25 +2461,34 @@ class NameNode(AtomicExprNode): entry.scope.namespace_cname, interned_cname, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif entry.is_local or entry.in_closure or entry.from_closure or entry.type.is_memoryviewslice: # Raise UnboundLocalError for objects and memoryviewslices raise_unbound = ( (self.cf_maybe_null or self.cf_is_null) and not self.allow_null) - null_code = entry.type.check_for_null_code(entry.cname) memslice_check = entry.type.is_memoryviewslice and self.initialized_check + optional_cpp_check = entry.is_cpp_optional and self.initialized_check + + if optional_cpp_check: + unbound_check_code = entry.type.cpp_optional_check_for_null_code(entry.cname) + else: + unbound_check_code = entry.type.check_for_null_code(entry.cname) + + if unbound_check_code and raise_unbound and (entry.type.is_pyobject or memslice_check or optional_cpp_check): + code.put_error_if_unbound(self.pos, entry, self.in_nogil_context, unbound_check_code=unbound_check_code) - if null_code and raise_unbound and (entry.type.is_pyobject or memslice_check): - code.put_error_if_unbound(self.pos, entry, self.in_nogil_context) + elif entry.is_cglobal and entry.is_cpp_optional and self.initialized_check: + unbound_check_code = entry.type.cpp_optional_check_for_null_code(entry.cname) + code.put_error_if_unbound(self.pos, entry, unbound_check_code=unbound_check_code) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): #print "NameNode.generate_assignment_code:", self.name ### entry = self.entry if entry is None: - return # There was an error earlier + return # There was an error earlier if (self.entry.type.is_ptr and isinstance(rhs, ListNode) and not self.lhs_of_first_assignment and not rhs.in_module_scope): @@ -2301,8 +2509,10 @@ class NameNode(AtomicExprNode): setter = 'PyDict_SetItem' namespace = Naming.moddict_cname elif entry.is_pyclass_attr: - code.globalstate.use_utility_code(UtilityCode.load_cached("SetNameInClass", "ObjectHandling.c")) - setter = '__Pyx_SetNameInClass' + # Special-case setting __new__ + n = "SetNewInClass" if self.name == "__new__" else "SetNameInClass" + code.globalstate.use_utility_code(UtilityCode.load_cached(n, "ObjectHandling.c")) + setter = '__Pyx_' + n else: assert False, repr(entry) code.put_error_if_neg( @@ -2344,31 +2554,24 @@ class NameNode(AtomicExprNode): rhs.make_owned_reference(code) is_external_ref = entry.is_cglobal or self.entry.in_closure or self.entry.from_closure if is_external_ref: - if not self.cf_is_null: - if self.cf_maybe_null: - code.put_xgotref(self.py_result()) - else: - code.put_gotref(self.py_result()) + self.generate_gotref(code, handle_null=True) assigned = True if entry.is_cglobal: - code.put_decref_set( - self.result(), rhs.result_as(self.ctype())) + self.generate_decref_set(code, rhs.result_as(self.ctype())) else: if not self.cf_is_null: if self.cf_maybe_null: - code.put_xdecref_set( - self.result(), rhs.result_as(self.ctype())) + self.generate_xdecref_set(code, rhs.result_as(self.ctype())) else: - code.put_decref_set( - self.result(), rhs.result_as(self.ctype())) + self.generate_decref_set(code, rhs.result_as(self.ctype())) else: assigned = False if is_external_ref: - code.put_giveref(rhs.py_result()) + rhs.generate_giveref(code) if not self.type.is_memoryviewslice: if not assigned: if overloaded_assignment: - result = rhs.result() + result = rhs.move_result_rhs() if exception_check == '+': translate_cpp_exception( code, self.pos, @@ -2378,7 +2581,7 @@ class NameNode(AtomicExprNode): else: code.putln('%s = %s;' % (self.result(), result)) else: - result = rhs.result_as(self.ctype()) + result = rhs.move_result_rhs_as(self.ctype()) if is_pythran_expr(self.type): code.putln('new (&%s) decltype(%s){%s};' % (self.result(), self.result(), result)) @@ -2431,7 +2634,7 @@ class NameNode(AtomicExprNode): def generate_deletion_code(self, code, ignore_nonexisting=False): if self.entry is None: - return # There was an error earlier + return # There was an error earlier elif self.entry.is_pyclass_attr: namespace = self.entry.scope.namespace_cname interned_cname = code.intern_identifier(self.entry.name) @@ -2467,26 +2670,20 @@ class NameNode(AtomicExprNode): if self.cf_maybe_null and not ignore_nonexisting: code.put_error_if_unbound(self.pos, self.entry) - if self.entry.type.is_pyobject: - if self.entry.in_closure: - # generator - if ignore_nonexisting and self.cf_maybe_null: - code.put_xgotref(self.result()) - else: - code.put_gotref(self.result()) - if ignore_nonexisting and self.cf_maybe_null: - code.put_xdecref(self.result(), self.ctype()) - else: - code.put_decref(self.result(), self.ctype()) - code.putln('%s = NULL;' % self.result()) + if self.entry.in_closure: + # generator + self.generate_gotref(code, handle_null=True, maybe_null_extra_check=ignore_nonexisting) + if ignore_nonexisting and self.cf_maybe_null: + code.put_xdecref_clear(self.result(), self.ctype(), + have_gil=not self.nogil) else: - code.put_xdecref_memoryviewslice(self.entry.cname, - have_gil=not self.nogil) + code.put_decref_clear(self.result(), self.ctype(), + have_gil=not self.nogil) else: error(self.pos, "Deletion of C names not supported") def annotate(self, code): - if hasattr(self, 'is_called') and self.is_called: + if getattr(self, 'is_called', False): pos = (self.pos[0], self.pos[1], self.pos[2] - len(self.name) - 1) if self.type.is_pyobject: style, text = 'py_call', 'python function (%s)' @@ -2494,6 +2691,11 @@ class NameNode(AtomicExprNode): style, text = 'c_call', 'c function (%s)' code.annotate(pos, AnnotationItem(style, text % self.type, size=len(self.name))) + def get_known_standard_library_import(self): + if self.entry: + return self.entry.known_standard_library_import + return None + class BackquoteNode(ExprNode): # `expr` # @@ -2520,7 +2722,7 @@ class BackquoteNode(ExprNode): self.result(), self.arg.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class ImportNode(ExprNode): @@ -2539,44 +2741,68 @@ class ImportNode(ExprNode): # relative to the current module. # None: decide the level according to language level and # directives + # get_top_level_module int true: return top-level module, false: return imported module + # module_names TupleNode the separate names of the module and submodules, or None type = py_object_type + module_names = None + get_top_level_module = False + is_temp = True - subexprs = ['module_name', 'name_list'] + subexprs = ['module_name', 'name_list', 'module_names'] def analyse_types(self, env): if self.level is None: - if (env.directives['py2_import'] or - Future.absolute_import not in env.global_scope().context.future_directives): + # For modules in packages, and without 'absolute_import' enabled, try relative (Py2) import first. + if env.global_scope().parent_module and ( + env.directives['py2_import'] or + Future.absolute_import not in env.global_scope().context.future_directives): self.level = -1 else: self.level = 0 module_name = self.module_name.analyse_types(env) self.module_name = module_name.coerce_to_pyobject(env) + assert self.module_name.is_string_literal if self.name_list: name_list = self.name_list.analyse_types(env) self.name_list = name_list.coerce_to_pyobject(env) - self.is_temp = 1 + elif '.' in self.module_name.value: + self.module_names = TupleNode(self.module_name.pos, args=[ + IdentifierStringNode(self.module_name.pos, value=part, constant_result=part) + for part in map(StringEncoding.EncodedString, self.module_name.value.split('.')) + ]).analyse_types(env) return self gil_message = "Python import" def generate_result_code(self, code): - if self.name_list: - name_list_code = self.name_list.py_result() + assert self.module_name.is_string_literal + module_name = self.module_name.value + + if self.level <= 0 and not self.name_list and not self.get_top_level_module: + if self.module_names: + assert self.module_names.is_literal # make sure we create the tuple only once + if self.level == 0: + utility_code = UtilityCode.load_cached("ImportDottedModule", "ImportExport.c") + helper_func = "__Pyx_ImportDottedModule" + else: + utility_code = UtilityCode.load_cached("ImportDottedModuleRelFirst", "ImportExport.c") + helper_func = "__Pyx_ImportDottedModuleRelFirst" + code.globalstate.use_utility_code(utility_code) + import_code = "%s(%s, %s)" % ( + helper_func, + self.module_name.py_result(), + self.module_names.py_result() if self.module_names else 'NULL', + ) else: - name_list_code = "0" - - code.globalstate.use_utility_code(UtilityCode.load_cached("Import", "ImportExport.c")) - import_code = "__Pyx_Import(%s, %s, %d)" % ( - self.module_name.py_result(), - name_list_code, - self.level) + code.globalstate.use_utility_code(UtilityCode.load_cached("Import", "ImportExport.c")) + import_code = "__Pyx_Import(%s, %s, %d)" % ( + self.module_name.py_result(), + self.name_list.py_result() if self.name_list else '0', + self.level) - if (self.level <= 0 and - self.module_name.is_string_literal and - self.module_name.value in utility_code_for_imports): - helper_func, code_name, code_file = utility_code_for_imports[self.module_name.value] + if self.level <= 0 and module_name in utility_code_for_imports: + helper_func, code_name, code_file = utility_code_for_imports[module_name] code.globalstate.use_utility_code(UtilityCode.load_cached(code_name, code_file)) import_code = '%s(%s)' % (helper_func, import_code) @@ -2584,10 +2810,104 @@ class ImportNode(ExprNode): self.result(), import_code, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) + + def get_known_standard_library_import(self): + return self.module_name.value + + +class ScopedExprNode(ExprNode): + # Abstract base class for ExprNodes that have their own local + # scope, such as generator expressions. + # + # expr_scope Scope the inner scope of the expression + subexprs = [] + expr_scope = None + + # does this node really have a local scope, e.g. does it leak loop + # variables or not? non-leaking Py3 behaviour is default, except + # for list comprehensions where the behaviour differs in Py2 and + # Py3 (set in Parsing.py based on parser context) + has_local_scope = True + + def init_scope(self, outer_scope, expr_scope=None): + if expr_scope is not None: + self.expr_scope = expr_scope + elif self.has_local_scope: + self.expr_scope = Symtab.ComprehensionScope(outer_scope) + elif not self.expr_scope: # don't unset if it's already been set + self.expr_scope = None + + def analyse_declarations(self, env): + self.init_scope(env) + + def analyse_scoped_declarations(self, env): + # this is called with the expr_scope as env + pass -class IteratorNode(ExprNode): + def analyse_types(self, env): + # no recursion here, the children will be analysed separately below + return self + + def analyse_scoped_expressions(self, env): + # this is called with the expr_scope as env + return self + + def generate_evaluation_code(self, code): + # set up local variables and free their references on exit + generate_inner_evaluation_code = super(ScopedExprNode, self).generate_evaluation_code + if not self.has_local_scope or not self.expr_scope.var_entries: + # no local variables => delegate, done + generate_inner_evaluation_code(code) + return + + code.putln('{ /* enter inner scope */') + py_entries = [] + for _, entry in sorted(item for item in self.expr_scope.entries.items() if item[0]): + if not entry.in_closure: + if entry.type.is_pyobject and entry.used: + py_entries.append(entry) + if not py_entries: + # no local Python references => no cleanup required + generate_inner_evaluation_code(code) + code.putln('} /* exit inner scope */') + return + + # must free all local Python references at each exit point + old_loop_labels = code.new_loop_labels() + old_error_label = code.new_error_label() + + generate_inner_evaluation_code(code) + + # normal (non-error) exit + self._generate_vars_cleanup(code, py_entries) + + # error/loop body exit points + exit_scope = code.new_label('exit_scope') + code.put_goto(exit_scope) + for label, old_label in ([(code.error_label, old_error_label)] + + list(zip(code.get_loop_labels(), old_loop_labels))): + if code.label_used(label): + code.put_label(label) + self._generate_vars_cleanup(code, py_entries) + code.put_goto(old_label) + code.put_label(exit_scope) + code.putln('} /* exit inner scope */') + + code.set_loop_labels(old_loop_labels) + code.error_label = old_error_label + + def _generate_vars_cleanup(self, code, py_entries): + for entry in py_entries: + if entry.is_cglobal: + code.put_var_gotref(entry) + code.put_var_decref_set(entry, "Py_None") + else: + code.put_var_xdecref_clear(entry) + + +class IteratorNode(ScopedExprNode): # Used as part of for statement implementation. # # Implements result = iter(sequence) @@ -2597,20 +2917,25 @@ class IteratorNode(ExprNode): type = py_object_type iter_func_ptr = None counter_cname = None - cpp_iterator_cname = None reversed = False # currently only used for list/tuple types (see Optimize.py) is_async = False + has_local_scope = False subexprs = ['sequence'] def analyse_types(self, env): + if self.expr_scope: + env = self.expr_scope # actually evaluate sequence in this scope instead self.sequence = self.sequence.analyse_types(env) if (self.sequence.type.is_array or self.sequence.type.is_ptr) and \ not self.sequence.type.is_string: # C array iteration will be transformed later on self.type = self.sequence.type elif self.sequence.type.is_cpp_class: - self.analyse_cpp_types(env) + return CppIteratorNode(self.pos, sequence=self.sequence).analyse_types(env) + elif self.is_reversed_cpp_iteration(): + sequence = self.sequence.arg_tuple.args[0].arg + return CppIteratorNode(self.pos, sequence=sequence, reversed=True).analyse_types(env) else: self.sequence = self.sequence.coerce_to_pyobject(env) if self.sequence.type in (list_type, tuple_type): @@ -2625,8 +2950,27 @@ class IteratorNode(ExprNode): PyrexTypes.CFuncTypeArg("it", PyrexTypes.py_object_type, None), ])) + def is_reversed_cpp_iteration(self): + """ + Returns True if the 'reversed' function is applied to a C++ iterable. + + This supports C++ classes with reverse_iterator implemented. + """ + if not (isinstance(self.sequence, SimpleCallNode) and + self.sequence.arg_tuple and len(self.sequence.arg_tuple.args) == 1): + return False + func = self.sequence.function + if func.is_name and func.name == "reversed": + if not func.entry.is_builtin: + return False + arg = self.sequence.arg_tuple.args[0] + if isinstance(arg, CoercionNode) and arg.arg.is_name: + arg = arg.arg.entry + return arg.type.is_cpp_class + return False + def type_dependencies(self, env): - return self.sequence.type_dependencies(env) + return self.sequence.type_dependencies(self.expr_scope or env) def infer_type(self, env): sequence_type = self.sequence.infer_type(env) @@ -2640,65 +2984,10 @@ class IteratorNode(ExprNode): return sequence_type return py_object_type - def analyse_cpp_types(self, env): - sequence_type = self.sequence.type - if sequence_type.is_ptr: - sequence_type = sequence_type.base_type - begin = sequence_type.scope.lookup("begin") - end = sequence_type.scope.lookup("end") - if (begin is None - or not begin.type.is_cfunction - or begin.type.args): - error(self.pos, "missing begin() on %s" % self.sequence.type) - self.type = error_type - return - if (end is None - or not end.type.is_cfunction - or end.type.args): - error(self.pos, "missing end() on %s" % self.sequence.type) - self.type = error_type - return - iter_type = begin.type.return_type - if iter_type.is_cpp_class: - if env.lookup_operator_for_types( - self.pos, - "!=", - [iter_type, end.type.return_type]) is None: - error(self.pos, "missing operator!= on result of begin() on %s" % self.sequence.type) - self.type = error_type - return - if env.lookup_operator_for_types(self.pos, '++', [iter_type]) is None: - error(self.pos, "missing operator++ on result of begin() on %s" % self.sequence.type) - self.type = error_type - return - if env.lookup_operator_for_types(self.pos, '*', [iter_type]) is None: - error(self.pos, "missing operator* on result of begin() on %s" % self.sequence.type) - self.type = error_type - return - self.type = iter_type - elif iter_type.is_ptr: - if not (iter_type == end.type.return_type): - error(self.pos, "incompatible types for begin() and end()") - self.type = iter_type - else: - error(self.pos, "result type of begin() on %s must be a C++ class or pointer" % self.sequence.type) - self.type = error_type - return - def generate_result_code(self, code): sequence_type = self.sequence.type if sequence_type.is_cpp_class: - if self.sequence.is_name: - # safe: C++ won't allow you to reassign to class references - begin_func = "%s.begin" % self.sequence.result() - else: - sequence_type = PyrexTypes.c_ptr_type(sequence_type) - self.cpp_iterator_cname = code.funcstate.allocate_temp(sequence_type, manage_ref=False) - code.putln("%s = &%s;" % (self.cpp_iterator_cname, self.sequence.result())) - begin_func = "%s->begin" % self.cpp_iterator_cname - # TODO: Limit scope. - code.putln("%s = %s();" % (self.result(), begin_func)) - return + assert False, "Should have been changed to CppIteratorNode" if sequence_type.is_array or sequence_type.is_ptr: raise InternalError("for in carray slice not transformed") @@ -2740,12 +3029,12 @@ class IteratorNode(ExprNode): self.result(), self.sequence.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) # PyObject_GetIter() fails if "tp_iternext" is not set, but the check below # makes it visible to the C compiler that the pointer really isn't NULL, so that # it can distinguish between the special cases and the generic case - code.putln("%s = Py_TYPE(%s)->tp_iternext; %s" % ( + code.putln("%s = __Pyx_PyObject_GetIterNextFunc(%s); %s" % ( self.iter_func_ptr, self.py_result(), code.error_goto_if_null(self.iter_func_ptr, self.pos))) if self.may_be_a_sequence: @@ -2787,28 +3076,14 @@ class IteratorNode(ExprNode): self.counter_cname, inc_dec, code.error_goto_if_null(result_name, self.pos))) - code.put_gotref(result_name) + code.put_gotref(result_name, py_object_type) code.putln("#endif") def generate_iter_next_result_code(self, result_name, code): sequence_type = self.sequence.type if self.reversed: code.putln("if (%s < 0) break;" % self.counter_cname) - if sequence_type.is_cpp_class: - if self.cpp_iterator_cname: - end_func = "%s->end" % self.cpp_iterator_cname - else: - end_func = "%s.end" % self.sequence.result() - # TODO: Cache end() call? - code.putln("if (!(%s != %s())) break;" % ( - self.result(), - end_func)) - code.putln("%s = *%s;" % ( - result_name, - self.result())) - code.putln("++%s;" % self.result()) - return - elif sequence_type is list_type: + if sequence_type is list_type: self.generate_next_sequence_item('List', result_name, code) return elif sequence_type is tuple_type: @@ -2838,7 +3113,7 @@ class IteratorNode(ExprNode): code.putln("}") code.putln("break;") code.putln("}") - code.put_gotref(result_name) + code.put_gotref(result_name, py_object_type) code.putln("}") def free_temps(self, code): @@ -2847,11 +3122,178 @@ class IteratorNode(ExprNode): if self.iter_func_ptr: code.funcstate.release_temp(self.iter_func_ptr) self.iter_func_ptr = None - if self.cpp_iterator_cname: - code.funcstate.release_temp(self.cpp_iterator_cname) ExprNode.free_temps(self, code) +class CppIteratorNode(ExprNode): + # Iteration over a C++ container. + # Created at the analyse_types stage by IteratorNode + cpp_sequence_cname = None + cpp_attribute_op = "." + extra_dereference = "" + is_temp = True + reversed = False + + subexprs = ['sequence'] + + def get_iterator_func_names(self): + return ("begin", "end") if not self.reversed else ("rbegin", "rend") + + def analyse_types(self, env): + sequence_type = self.sequence.type + if sequence_type.is_ptr: + sequence_type = sequence_type.base_type + begin_name, end_name = self.get_iterator_func_names() + begin = sequence_type.scope.lookup(begin_name) + end = sequence_type.scope.lookup(end_name) + if (begin is None + or not begin.type.is_cfunction + or begin.type.args): + error(self.pos, "missing %s() on %s" % (begin_name, self.sequence.type)) + self.type = error_type + return self + if (end is None + or not end.type.is_cfunction + or end.type.args): + error(self.pos, "missing %s() on %s" % (end_name, self.sequence.type)) + self.type = error_type + return self + iter_type = begin.type.return_type + if iter_type.is_cpp_class: + if env.directives['cpp_locals']: + self.extra_dereference = "*" + if env.lookup_operator_for_types( + self.pos, + "!=", + [iter_type, end.type.return_type]) is None: + error(self.pos, "missing operator!= on result of %s() on %s" % (begin_name, self.sequence.type)) + self.type = error_type + return self + if env.lookup_operator_for_types(self.pos, '++', [iter_type]) is None: + error(self.pos, "missing operator++ on result of %s() on %s" % (begin_name, self.sequence.type)) + self.type = error_type + return self + if env.lookup_operator_for_types(self.pos, '*', [iter_type]) is None: + error(self.pos, "missing operator* on result of %s() on %s" % (begin_name, self.sequence.type)) + self.type = error_type + return self + self.type = iter_type + elif iter_type.is_ptr: + if not (iter_type == end.type.return_type): + error(self.pos, "incompatible types for %s() and %s()" % (begin_name, end_name)) + self.type = iter_type + else: + error(self.pos, "result type of %s() on %s must be a C++ class or pointer" % (begin_name, self.sequence.type)) + self.type = error_type + return self + + def generate_result_code(self, code): + sequence_type = self.sequence.type + begin_name, _ = self.get_iterator_func_names() + # essentially 3 options: + if self.sequence.is_simple(): + # 1) Sequence can be accessed directly, like a name; + # assigning to it may break the container, but that's the responsibility + # of the user + code.putln("%s = %s%s%s();" % ( + self.result(), + self.sequence.result(), + self.cpp_attribute_op, + begin_name)) + else: + # (while it'd be nice to limit the scope of the loop temp, it's essentially + # impossible to do while supporting generators) + temp_type = sequence_type + if temp_type.is_reference: + # 2) Sequence is a reference (often obtained by dereferencing a pointer); + # make the temp a pointer so we are not sensitive to users reassigning + # the pointer than it came from + temp_type = PyrexTypes.CPtrType(sequence_type.ref_base_type) + if temp_type.is_ptr or code.globalstate.directives['cpp_locals']: + self.cpp_attribute_op = "->" + # 3) (otherwise) sequence comes from a function call or similar, so we must + # create a temp to store it in + self.cpp_sequence_cname = code.funcstate.allocate_temp(temp_type, manage_ref=False) + code.putln("%s = %s%s;" % (self.cpp_sequence_cname, + "&" if temp_type.is_ptr else "", + self.sequence.move_result_rhs())) + code.putln("%s = %s%s%s();" % ( + self.result(), + self.cpp_sequence_cname, + self.cpp_attribute_op, + begin_name)) + + def generate_iter_next_result_code(self, result_name, code): + # end call isn't cached to support containers that allow adding while iterating + # (much as this is usually a bad idea) + _, end_name = self.get_iterator_func_names() + code.putln("if (!(%s%s != %s%s%s())) break;" % ( + self.extra_dereference, + self.result(), + self.cpp_sequence_cname or self.sequence.result(), + self.cpp_attribute_op, + end_name)) + code.putln("%s = *%s%s;" % ( + result_name, + self.extra_dereference, + self.result())) + code.putln("++%s%s;" % (self.extra_dereference, self.result())) + + def generate_subexpr_disposal_code(self, code): + if not self.cpp_sequence_cname: + # the sequence is accessed directly so any temporary result in its + # subexpressions must remain available until the iterator is not needed + return + ExprNode.generate_subexpr_disposal_code(self, code) + + def free_subexpr_temps(self, code): + if not self.cpp_sequence_cname: + # the sequence is accessed directly so any temporary result in its + # subexpressions must remain available until the iterator is not needed + return + ExprNode.free_subexpr_temps(self, code) + + def generate_disposal_code(self, code): + if not self.cpp_sequence_cname: + # postponed from CppIteratorNode.generate_subexpr_disposal_code + # and CppIteratorNode.free_subexpr_temps + ExprNode.generate_subexpr_disposal_code(self, code) + ExprNode.free_subexpr_temps(self, code) + ExprNode.generate_disposal_code(self, code) + + def free_temps(self, code): + if self.cpp_sequence_cname: + code.funcstate.release_temp(self.cpp_sequence_cname) + # skip over IteratorNode since we don't use any of the temps it does + ExprNode.free_temps(self, code) + + +def remove_const(item_type): + """ + Removes the constness of a given type and its underlying templates + if any. + + This is to solve the compilation error when the temporary variable used to + store the result of an iterator cannot be changed due to its constness. + For example, the value_type of std::map, which will also be the type of + the temporarry variable, is std::pair<const Key, T>. This means the first + component of the variable cannot be reused to store the result of each + iteration, which leads to a compilation error. + """ + if item_type.is_const: + item_type = item_type.cv_base_type + if item_type.is_typedef: + item_type = remove_const(item_type.typedef_base_type) + if item_type.is_cpp_class and item_type.templates: + templates = [remove_const(t) if t.is_const else t for t in item_type.templates] + template_type = item_type.template_type + item_type = PyrexTypes.CppClassType( + template_type.name, template_type.scope, + template_type.cname, template_type.base_classes, + templates, template_type) + return item_type + + class NextNode(AtomicExprNode): # Used as part of for statement implementation. # Implements result = next(iterator) @@ -2876,16 +3318,12 @@ class NextNode(AtomicExprNode): iterator_type = self.iterator.infer_type(env) if iterator_type.is_ptr or iterator_type.is_array: return iterator_type.base_type - elif self.iterator.sequence.type is bytearray_type: - # This is a temporary work-around to fix bytearray iteration in 0.29.x - # It has been fixed properly in master, refer to ticket: 3473 - return py_object_type elif iterator_type.is_cpp_class: item_type = env.lookup_operator_for_types(self.pos, "*", [iterator_type]).type.return_type if item_type.is_reference: item_type = item_type.ref_base_type - if item_type.is_const: - item_type = item_type.const_base_type + if item_type.is_cv_qualified: + item_type = item_type.cv_base_type return item_type else: # Avoid duplication of complicated logic. @@ -2898,6 +3336,7 @@ class NextNode(AtomicExprNode): def analyse_types(self, env): self.type = self.infer_type(env, self.iterator.type) + self.type = remove_const(self.type) self.is_temp = 1 return self @@ -2905,7 +3344,7 @@ class NextNode(AtomicExprNode): self.iterator.generate_iter_next_result_code(self.result(), code) -class AsyncIteratorNode(ExprNode): +class AsyncIteratorNode(ScopedExprNode): # Used as part of 'async for' statement implementation. # # Implements result = sequence.__aiter__() @@ -2917,11 +3356,14 @@ class AsyncIteratorNode(ExprNode): is_async = True type = py_object_type is_temp = 1 + has_local_scope = False def infer_type(self, env): return py_object_type def analyse_types(self, env): + if self.expr_scope: + env = self.expr_scope self.sequence = self.sequence.analyse_types(env) if not self.sequence.type.is_pyobject: error(self.pos, "async for loops not allowed on C/C++ types") @@ -2934,7 +3376,7 @@ class AsyncIteratorNode(ExprNode): self.result(), self.sequence.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) class AsyncNextNode(AtomicExprNode): @@ -2964,7 +3406,7 @@ class AsyncNextNode(AtomicExprNode): self.result(), self.iterator.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) class WithExitCallNode(ExprNode): @@ -3007,7 +3449,7 @@ class WithExitCallNode(ExprNode): self.args.free_temps(code) code.putln(code.error_goto_if_null(result_var, self.pos)) - code.put_gotref(result_var) + code.put_gotref(result_var, py_object_type) if self.await_expr: # FIXME: result_var temp currently leaks into the closure @@ -3072,7 +3514,7 @@ class TempNode(ExprNode): return self def analyse_target_declaration(self, env): - pass + self.is_target = True def generate_result_code(self, code): pass @@ -3139,6 +3581,7 @@ class JoinedStrNode(ExprNode): # type = unicode_type is_temp = True + gil_message = "String concatenation" subexprs = ['values'] @@ -3161,7 +3604,7 @@ class JoinedStrNode(ExprNode): list_var, num_items, code.error_goto_if_null(list_var, self.pos))) - code.put_gotref(list_var) + code.put_gotref(list_var, py_object_type) code.putln("%s = 0;" % ulength_var) code.putln("%s = 127;" % max_char_var) # at least ASCII character range @@ -3205,7 +3648,7 @@ class JoinedStrNode(ExprNode): max_char_var, max_char_value, max_char_var, max_char_value, max_char_var)) code.putln("%s += %s;" % (ulength_var, ulength)) - code.put_giveref(node.py_result()) + node.generate_giveref(code) code.putln('PyTuple_SET_ITEM(%s, %s, %s);' % (list_var, i, node.py_result())) node.generate_post_assignment_code(code) node.free_temps(code) @@ -3220,7 +3663,7 @@ class JoinedStrNode(ExprNode): ulength_var, max_char_var, code.error_goto_if_null(self.py_result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) code.put_decref_clear(list_var, py_object_type) code.funcstate.release_temp(list_var) @@ -3232,7 +3675,7 @@ class FormattedValueNode(ExprNode): # {}-delimited portions of an f-string # # value ExprNode The expression itself - # conversion_char str or None Type conversion (!s, !r, !a, or none, or 'd' for integer conversion) + # conversion_char str or None Type conversion (!s, !r, !a, none, or 'd' for integer conversion) # format_spec JoinedStrNode or None Format string passed to __format__ # c_format_spec str or None If not None, formatting can be done at the C level @@ -3241,6 +3684,7 @@ class FormattedValueNode(ExprNode): type = unicode_type is_temp = True c_format_spec = None + gil_message = "String formatting" find_conversion_func = { 's': 'PyObject_Unicode', @@ -3278,7 +3722,7 @@ class FormattedValueNode(ExprNode): self.result(), convert_func_call, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) return value_result = self.value.py_result() @@ -3317,7 +3761,7 @@ class FormattedValueNode(ExprNode): value_result, format_spec, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) #------------------------------------------------------------------- @@ -3355,7 +3799,7 @@ class ParallelThreadsAvailableNode(AtomicExprNode): return self.temp_code -class ParallelThreadIdNode(AtomicExprNode): #, Nodes.ParallelNode): +class ParallelThreadIdNode(AtomicExprNode): #, Nodes.ParallelNode): """ Implements cython.parallel.threadid() """ @@ -3464,9 +3908,9 @@ class IndexNode(_IndexingBaseNode): def analyse_as_type(self, env): base_type = self.base.analyse_as_type(env) - if base_type and not base_type.is_pyobject: - if base_type.is_cpp_class: - if isinstance(self.index, TupleNode): + if base_type: + if base_type.is_cpp_class or base_type.python_type_constructor_name: + if self.index.is_sequence_constructor: template_values = self.index.args else: template_values = [self.index] @@ -3481,7 +3925,7 @@ class IndexNode(_IndexingBaseNode): env.use_utility_code(MemoryView.view_utility_code) axes = [self.index] if self.index.is_slice else list(self.index.args) return PyrexTypes.MemoryViewSliceType(base_type, MemoryView.get_axes_specs(env, axes)) - else: + elif not base_type.is_pyobject: # C array index = self.index.compile_time_value(env) if index is not None: @@ -3494,6 +3938,19 @@ class IndexNode(_IndexingBaseNode): error(self.pos, "Array size must be a compile time constant") return None + def analyse_pytyping_modifiers(self, env): + # Check for declaration modifiers, e.g. "typing.Optional[...]" or "dataclasses.InitVar[...]" + # TODO: somehow bring this together with TemplatedTypeNode.analyse_pytyping_modifiers() + modifiers = [] + modifier_node = self + while modifier_node.is_subscript: + modifier_type = modifier_node.base.analyse_as_type(env) + if (modifier_type and modifier_type.python_type_constructor_name + and modifier_type.modifier_name): + modifiers.append(modifier_type.modifier_name) + modifier_node = modifier_node.index + return modifiers + def type_dependencies(self, env): return self.base.type_dependencies(env) + self.index.type_dependencies(env) @@ -3511,6 +3968,8 @@ class IndexNode(_IndexingBaseNode): bytearray_type, list_type, tuple_type): # slicing these returns the same type return base_type + elif base_type.is_memoryviewslice: + return base_type else: # TODO: Handle buffers (hopefully without too much redundancy). return py_object_type @@ -3553,6 +4012,23 @@ class IndexNode(_IndexingBaseNode): index += base_type.size if 0 <= index < base_type.size: return base_type.components[index] + elif base_type.is_memoryviewslice: + if base_type.ndim == 0: + pass # probably an error, but definitely don't know what to do - return pyobject for now + if base_type.ndim == 1: + return base_type.dtype + else: + return PyrexTypes.MemoryViewSliceType(base_type.dtype, base_type.axes[1:]) + + if self.index.is_sequence_constructor and base_type.is_memoryviewslice: + inferred_type = base_type + for a in self.index.args: + if not inferred_type.is_memoryviewslice: + break # something's gone wrong + inferred_type = IndexNode(self.pos, base=ExprNode(self.base.pos, type=inferred_type), + index=a).infer_type(env) + else: + return inferred_type if base_type.is_cpp_class: class FakeOperand: @@ -3630,6 +4106,8 @@ class IndexNode(_IndexingBaseNode): if not base_type.is_cfunction: self.index = self.index.analyse_types(env) self.original_index_type = self.index.type + if self.original_index_type.is_reference: + self.original_index_type = self.original_index_type.ref_base_type if base_type.is_unicode_char: # we infer Py_UNICODE/Py_UCS4 for unicode strings in some @@ -3670,12 +4148,13 @@ class IndexNode(_IndexingBaseNode): self.is_temp = 1 elif self.index.type.is_int and base_type is not dict_type: if (getting + and not env.directives['boundscheck'] and (base_type in (list_type, tuple_type, bytearray_type)) and (not self.index.type.signed or not env.directives['wraparound'] or (isinstance(self.index, IntNode) and self.index.has_constant_result() and self.index.constant_result >= 0)) - and not env.directives['boundscheck']): + ): self.is_temp = 0 else: self.is_temp = 1 @@ -3702,12 +4181,16 @@ class IndexNode(_IndexingBaseNode): if base_type in (list_type, tuple_type) and self.index.type.is_int: item_type = infer_sequence_item_type( env, self.base, self.index, seq_type=base_type) - if item_type is None: - item_type = py_object_type - self.type = item_type if base_type in (list_type, tuple_type, dict_type): # do the None check explicitly (not in a helper) to allow optimising it away self.base = self.base.as_none_safe_node("'NoneType' object is not subscriptable") + if item_type is None or not item_type.is_pyobject: + # Even if we inferred a C type as result, we will read a Python object, so trigger coercion if needed. + # We could potentially use "item_type.equivalent_type" here, but that may trigger assumptions + # about the actual runtime item types, rather than just their ability to coerce to the C "item_type". + self.type = py_object_type + else: + self.type = item_type self.wrap_in_nonecheck_node(env, getting) return self @@ -3715,6 +4198,8 @@ class IndexNode(_IndexingBaseNode): def analyse_as_c_array(self, env, is_slice): base_type = self.base.type self.type = base_type.base_type + if self.type.is_cpp_class: + self.type = PyrexTypes.CReferenceType(self.type) if is_slice: self.type = base_type elif self.index.type.is_pyobject: @@ -3739,7 +4224,7 @@ class IndexNode(_IndexingBaseNode): if self.exception_check: if not setting: self.is_temp = True - if self.exception_value is None: + if needs_cpp_exception_conversion(self): env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp")) self.index = self.index.coerce_to(func_type.args[0].type, env) self.type = func_type.return_type @@ -4001,6 +4486,7 @@ class IndexNode(_IndexingBaseNode): return utility_code = None + error_value = None if self.type.is_pyobject: error_value = 'NULL' if self.index.type.is_int: @@ -4036,8 +4522,8 @@ class IndexNode(_IndexingBaseNode): error_value = '-1' utility_code = UtilityCode.load_cached("GetItemIntByteArray", "StringTools.c") elif not (self.base.type.is_cpp_class and self.exception_check): - assert False, "unexpected type %s and base type %s for indexing" % ( - self.type, self.base.type) + assert False, "unexpected type %s and base type %s for indexing (%s)" % ( + self.type, self.base.type, self.pos) if utility_code is not None: code.globalstate.use_utility_code(utility_code) @@ -4064,7 +4550,7 @@ class IndexNode(_IndexingBaseNode): self.extra_index_params(code), code.error_goto_if(error_check % self.result(), self.pos))) if self.type.is_pyobject: - code.put_gotref(self.py_result()) + self.generate_gotref(code) def generate_setitem_code(self, value_code, code): if self.index.type.is_int: @@ -4100,7 +4586,7 @@ class IndexNode(_IndexingBaseNode): self.pos)) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): self.generate_subexpr_evaluation_code(code) if self.type.is_pyobject: @@ -4109,8 +4595,7 @@ class IndexNode(_IndexingBaseNode): value_code = self._check_byte_value(code, rhs) self.generate_setitem_code(value_code, code) elif self.base.type.is_cpp_class and self.exception_check and self.exception_check == '+': - if overloaded_assignment and exception_check and \ - self.exception_value != exception_value: + if overloaded_assignment and exception_check and self.exception_value != exception_value: # Handle the case that both the index operator and the assignment # operator have a c++ exception handler and they are not the same. translate_double_cpp_exception(code, self.pos, self.type, @@ -4357,11 +4842,11 @@ class BufferIndexNode(_IndexingBaseNode): manage_ref=False) rhs_code = rhs.result() code.putln("%s = %s;" % (ptr, ptrexpr)) - code.put_xgotref("*%s" % ptr) + code.put_xgotref("*%s" % ptr, self.buffer_type.dtype) code.putln("__Pyx_INCREF(%s); __Pyx_XDECREF(*%s);" % ( rhs_code, ptr)) code.putln("*%s %s= %s;" % (ptr, op, rhs_code)) - code.put_xgiveref("*%s" % ptr) + code.put_xgiveref("*%s" % ptr, self.buffer_type.dtype) code.funcstate.release_temp(ptr) else: # Simple case @@ -4627,7 +5112,7 @@ class MemoryViewSliceNode(MemoryViewIndexNode): assert not list(it) buffer_entry.generate_buffer_slice_code( - code, self.original_indices, self.result(), + code, self.original_indices, self.result(), self.type, have_gil=have_gil, have_slices=have_slices, directives=code.globalstate.directives) @@ -4730,8 +5215,17 @@ class MemoryCopyScalar(MemoryCopyNode): code.putln("%s __pyx_temp_slice = %s;" % (slice_decl, self.dst.result())) dst_temp = "__pyx_temp_slice" + force_strided = False + indices = self.dst.original_indices + for idx in indices: + if isinstance(idx, SliceNode) and not (idx.start.is_none and + idx.stop.is_none and + idx.step.is_none): + force_strided = True + slice_iter_obj = MemoryView.slice_iter(self.dst.type, dst_temp, - self.dst.type.ndim, code) + self.dst.type.ndim, code, + force_strided=force_strided) p = slice_iter_obj.start_loops() if dtype.is_pyobject: @@ -4753,8 +5247,10 @@ class SliceIndexNode(ExprNode): # start ExprNode or None # stop ExprNode or None # slice ExprNode or None constant slice object + # nogil bool used internally subexprs = ['base', 'start', 'stop', 'slice'] + nogil = False slice = None @@ -4924,7 +5420,7 @@ class SliceIndexNode(ExprNode): def analyse_as_type(self, env): base_type = self.base.analyse_as_type(env) - if base_type and not base_type.is_pyobject: + if base_type: if not self.start and not self.stop: # memory view from . import MemoryView @@ -4940,7 +5436,10 @@ class SliceIndexNode(ExprNode): base_type, MemoryView.get_axes_specs(env, [slice_node])) return None - nogil_check = Node.gil_error + def nogil_check(self, env): + self.nogil = env.nogil + return super(SliceIndexNode, self).nogil_check(env) + gil_message = "Slicing Python object" get_slice_utility_code = TempitaUtilityCode.load( @@ -5064,15 +5563,14 @@ class SliceIndexNode(ExprNode): start_code, stop_code, code.error_goto_if_null(result, self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): self.generate_subexpr_evaluation_code(code) if self.type.is_pyobject: code.globalstate.use_utility_code(self.set_slice_utility_code) - (has_c_start, has_c_stop, c_start, c_stop, - py_start, py_stop, py_slice) = self.get_slice_config() + has_c_start, has_c_stop, c_start, c_stop, py_start, py_stop, py_slice = self.get_slice_config() code.put_error_if_neg(self.pos, "__Pyx_PyObject_SetSlice(%s, %s, %s, %s, %s, %s, %s, %d, %d, %d)" % ( self.base.py_result(), @@ -5209,11 +5707,15 @@ class SliceIndexNode(ExprNode): if runtime_check: code.putln("if (unlikely((%s) != (%s))) {" % (runtime_check, target_size)) + if self.nogil: + code.put_ensure_gil() code.putln( 'PyErr_Format(PyExc_ValueError, "Assignment to slice of wrong length,' ' expected %%" CYTHON_FORMAT_SSIZE_T "d, got %%" CYTHON_FORMAT_SSIZE_T "d",' ' (Py_ssize_t)(%s), (Py_ssize_t)(%s));' % ( target_size, runtime_check)) + if self.nogil: + code.put_release_ensured_gil() code.putln(code.error_goto(self.pos)) code.putln("}") @@ -5299,9 +5801,9 @@ class SliceNode(ExprNode): self.stop.py_result(), self.step.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) if self.is_literal: - code.put_giveref(self.py_result()) + self.generate_giveref(code) class SliceIntNode(SliceNode): # start:stop:step in subscript list @@ -5398,6 +5900,9 @@ class CallNode(ExprNode): return PyrexTypes.c_double_type elif function.entry.name in Builtin.types_that_construct_their_instance: return result_type + func_type = self.function.analyse_as_type(env) + if func_type and (func_type.is_struct_or_union or func_type.is_cpp_class): + return func_type return py_object_type def type_dependencies(self, env): @@ -5523,6 +6028,16 @@ class SimpleCallNode(CallNode): except Exception as e: self.compile_time_value_error(e) + @classmethod + def for_cproperty(cls, pos, obj, entry): + # Create a call node for C property access. + property_scope = entry.scope + getter_entry = property_scope.lookup_here(entry.name) + assert getter_entry, "Getter not found in scope %s: %s" % (property_scope, property_scope.entries) + function = NameNode(pos, name=entry.name, entry=getter_entry, type=getter_entry.type) + node = cls(pos, function=function, args=[obj]) + return node + def analyse_as_type(self, env): attr = self.function.as_cython_attribute() if attr == 'pointer': @@ -5588,6 +6103,7 @@ class SimpleCallNode(CallNode): self.analyse_c_function_call(env) if func_type.exception_check == '+': self.is_temp = True + return self def function_type(self): @@ -5639,8 +6155,8 @@ class SimpleCallNode(CallNode): else: alternatives = overloaded_entry.all_alternatives() - entry = PyrexTypes.best_match( - [arg.type for arg in args], alternatives, self.pos, env, args) + entry = PyrexTypes.best_match([arg.type for arg in args], + alternatives, self.pos, env, args) if not entry: self.type = PyrexTypes.error_type @@ -5723,7 +6239,7 @@ class SimpleCallNode(CallNode): # but we must make sure it cannot be collected # before we return from the function, so we create # an owned temp reference to it - if i > 0: # first argument doesn't matter + if i > 0: # first argument doesn't matter some_args_in_temps = True arg = arg.coerce_to_temp(env) args[i] = arg @@ -5752,7 +6268,7 @@ class SimpleCallNode(CallNode): # case) for i in range(actual_nargs-1): if i == 0 and self.self is not None: - continue # self is ok + continue # self is ok arg = args[i] if arg.nonlocally_immutable(): # locals, C functions, unassignable types are safe. @@ -5768,7 +6284,7 @@ class SimpleCallNode(CallNode): else: #self.args[i] = arg.coerce_to_temp(env) # instead: issue a warning - if i > 0 or i == 1 and self.self is not None: # skip first arg + if i > 0 or i == 1 and self.self is not None: # skip first arg warning(arg.pos, "Argument evaluation order in C function call is undefined and may not be as expected", 0) break @@ -5797,15 +6313,9 @@ class SimpleCallNode(CallNode): if self.is_temp and self.type.is_reference: self.type = PyrexTypes.CFakeReferenceType(self.type.ref_base_type) - # Called in 'nogil' context? - self.nogil = env.nogil - if (self.nogil and - func_type.exception_check and - func_type.exception_check != '+'): - env.use_utility_code(pyerr_occurred_withgil_utility_code) # C++ exception handler if func_type.exception_check == '+': - if func_type.exception_value is None: + if needs_cpp_exception_conversion(func_type): env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp")) self.overflowcheck = env.directives['overflowcheck'] @@ -5824,8 +6334,8 @@ class SimpleCallNode(CallNode): expected_nargs = max_nargs - func_type.optional_arg_count actual_nargs = len(self.args) for formal_arg, actual_arg in args[:expected_nargs]: - arg_code = actual_arg.result_as(formal_arg.type) - arg_list_code.append(arg_code) + arg_code = actual_arg.move_result_rhs_as(formal_arg.type) + arg_list_code.append(arg_code) if func_type.is_overridable: arg_list_code.append(str(int(self.wrapper_call or self.function.entry.is_unbound_cmethod))) @@ -5838,7 +6348,7 @@ class SimpleCallNode(CallNode): arg_list_code.append(optional_args) for actual_arg in self.args[len(formal_args):]: - arg_list_code.append(actual_arg.result()) + arg_list_code.append(actual_arg.move_result_rhs()) result = "%s(%s)" % (self.function.result(), ', '.join(arg_list_code)) return result @@ -5899,7 +6409,7 @@ class SimpleCallNode(CallNode): arg.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) for subexpr in subexprs: if subexpr is not None: @@ -5918,8 +6428,9 @@ class SimpleCallNode(CallNode): self.function.py_result(), arg_code, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif func_type.is_cfunction: + nogil = not code.funcstate.gil_owned if self.has_optional_args: actual_nargs = len(self.args) expected_nargs = len(func_type.args) - func_type.optional_arg_count @@ -5947,7 +6458,9 @@ class SimpleCallNode(CallNode): if exc_val is not None: exc_checks.append("%s == %s" % (self.result(), func_type.return_type.cast_code(exc_val))) if exc_check: - if self.nogil: + if nogil: + code.globalstate.use_utility_code( + UtilityCode.load_cached("ErrOccurredWithGIL", "Exceptions.c")) exc_checks.append("__Pyx_ErrOccurredWithGIL()") else: exc_checks.append("PyErr_Occurred()") @@ -5965,7 +6478,7 @@ class SimpleCallNode(CallNode): if func_type.exception_check == '+': translate_cpp_exception(code, self.pos, '%s%s;' % (lhs, rhs), self.result() if self.type.is_pyobject else None, - func_type.exception_value, self.nogil) + func_type.exception_value, nogil) else: if exc_checks: goto_error = code.error_goto_if(" && ".join(exc_checks), self.pos) @@ -5973,7 +6486,7 @@ class SimpleCallNode(CallNode): goto_error = "" code.putln("%s%s; %s" % (lhs, rhs, goto_error)) if self.type.is_pyobject and self.result(): - code.put_gotref(self.py_result()) + self.generate_gotref(code) if self.has_optional_args: code.funcstate.release_temp(self.opt_arg_struct) @@ -6039,10 +6552,8 @@ class PyMethodCallNode(SimpleCallNode): self_arg = code.funcstate.allocate_temp(py_object_type, manage_ref=True) code.putln("%s = NULL;" % self_arg) - arg_offset_cname = None - if len(args) > 1: - arg_offset_cname = code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False) - code.putln("%s = 0;" % arg_offset_cname) + arg_offset_cname = code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False) + code.putln("%s = 0;" % arg_offset_cname) def attribute_is_likely_method(attr): obj = attr.obj @@ -6074,116 +6585,35 @@ class PyMethodCallNode(SimpleCallNode): code.put_incref(self_arg, py_object_type) code.put_incref("function", py_object_type) # free method object as early to possible to enable reuse from CPython's freelist - code.put_decref_set(function, "function") - if len(args) > 1: - code.putln("%s = 1;" % arg_offset_cname) + code.put_decref_set(function, py_object_type, "function") + code.putln("%s = 1;" % arg_offset_cname) code.putln("}") code.putln("}") - if not args: - # fastest special case: try to avoid tuple creation - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyObjectCallNoArg", "ObjectHandling.c")) - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyObjectCallOneArg", "ObjectHandling.c")) - code.putln( - "%s = (%s) ? __Pyx_PyObject_CallOneArg(%s, %s) : __Pyx_PyObject_CallNoArg(%s);" % ( - self.result(), self_arg, - function, self_arg, - function)) - code.put_xdecref_clear(self_arg, py_object_type) - code.funcstate.release_temp(self_arg) - code.putln(code.error_goto_if_null(self.result(), self.pos)) - code.put_gotref(self.py_result()) - elif len(args) == 1: - # fastest special case: try to avoid tuple creation - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyObjectCall2Args", "ObjectHandling.c")) - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyObjectCallOneArg", "ObjectHandling.c")) - arg = args[0] - code.putln( - "%s = (%s) ? __Pyx_PyObject_Call2Args(%s, %s, %s) : __Pyx_PyObject_CallOneArg(%s, %s);" % ( - self.result(), self_arg, - function, self_arg, arg.py_result(), - function, arg.py_result())) - code.put_xdecref_clear(self_arg, py_object_type) - code.funcstate.release_temp(self_arg) - arg.generate_disposal_code(code) - arg.free_temps(code) - code.putln(code.error_goto_if_null(self.result(), self.pos)) - code.put_gotref(self.py_result()) - else: - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyFunctionFastCall", "ObjectHandling.c")) - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyCFunctionFastCall", "ObjectHandling.c")) - for test_func, call_prefix in [('PyFunction_Check', 'Py'), ('__Pyx_PyFastCFunction_Check', 'PyC')]: - code.putln("#if CYTHON_FAST_%sCALL" % call_prefix.upper()) - code.putln("if (%s(%s)) {" % (test_func, function)) - code.putln("PyObject *%s[%d] = {%s, %s};" % ( - Naming.quick_temp_cname, - len(args)+1, - self_arg, - ', '.join(arg.py_result() for arg in args))) - code.putln("%s = __Pyx_%sFunction_FastCall(%s, %s+1-%s, %d+%s); %s" % ( - self.result(), - call_prefix, - function, - Naming.quick_temp_cname, - arg_offset_cname, - len(args), - arg_offset_cname, - code.error_goto_if_null(self.result(), self.pos))) - code.put_xdecref_clear(self_arg, py_object_type) - code.put_gotref(self.py_result()) - for arg in args: - arg.generate_disposal_code(code) - code.putln("} else") - code.putln("#endif") - - code.putln("{") - args_tuple = code.funcstate.allocate_temp(py_object_type, manage_ref=True) - code.putln("%s = PyTuple_New(%d+%s); %s" % ( - args_tuple, len(args), arg_offset_cname, - code.error_goto_if_null(args_tuple, self.pos))) - code.put_gotref(args_tuple) - - if len(args) > 1: - code.putln("if (%s) {" % self_arg) - code.putln("__Pyx_GIVEREF(%s); PyTuple_SET_ITEM(%s, 0, %s); %s = NULL;" % ( - self_arg, args_tuple, self_arg, self_arg)) # stealing owned ref in this case - code.funcstate.release_temp(self_arg) - if len(args) > 1: - code.putln("}") - - for i, arg in enumerate(args): - arg.make_owned_reference(code) - code.put_giveref(arg.py_result()) - code.putln("PyTuple_SET_ITEM(%s, %d+%s, %s);" % ( - args_tuple, i, arg_offset_cname, arg.py_result())) - if len(args) > 1: - code.funcstate.release_temp(arg_offset_cname) - - for arg in args: - arg.generate_post_assignment_code(code) - arg.free_temps(code) - - code.globalstate.use_utility_code( - UtilityCode.load_cached("PyObjectCall", "ObjectHandling.c")) - code.putln( - "%s = __Pyx_PyObject_Call(%s, %s, NULL); %s" % ( - self.result(), - function, args_tuple, - code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + # actually call the function + code.globalstate.use_utility_code( + UtilityCode.load_cached("PyObjectFastCall", "ObjectHandling.c")) - code.put_decref_clear(args_tuple, py_object_type) - code.funcstate.release_temp(args_tuple) + code.putln("{") + code.putln("PyObject *__pyx_callargs[%d] = {%s, %s};" % ( + len(args)+1, + self_arg, + ', '.join(arg.py_result() for arg in args))) + code.putln("%s = __Pyx_PyObject_FastCall(%s, __pyx_callargs+1-%s, %d+%s);" % ( + self.result(), + function, + arg_offset_cname, + len(args), + arg_offset_cname)) - if len(args) == 1: - code.putln("}") - code.putln("}") # !CYTHON_FAST_PYCALL + code.put_xdecref_clear(self_arg, py_object_type) + code.funcstate.release_temp(self_arg) + code.funcstate.release_temp(arg_offset_cname) + for arg in args: + arg.generate_disposal_code(code) + arg.free_temps(code) + code.putln(code.error_goto_if_null(self.result(), self.pos)) + self.generate_gotref(code) if reuse_function_temp: self.function.generate_disposal_code(code) @@ -6191,6 +6621,7 @@ class PyMethodCallNode(SimpleCallNode): else: code.put_decref_clear(function, py_object_type) code.funcstate.release_temp(function) + code.putln("}") class InlinedDefNodeCallNode(CallNode): @@ -6241,7 +6672,7 @@ class InlinedDefNodeCallNode(CallNode): # but we must make sure it cannot be collected # before we return from the function, so we create # an owned temp reference to it - if i > 0: # first argument doesn't matter + if i > 0: # first argument doesn't matter some_args_in_temps = True arg = arg.coerce_to_temp(env) self.args[i] = arg @@ -6288,7 +6719,7 @@ class InlinedDefNodeCallNode(CallNode): self.function.def_node.entry.pyfunc_cname, arg_code, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class PythonCapiFunctionNode(ExprNode): @@ -6358,7 +6789,7 @@ class CachedBuiltinMethodCallNode(CallNode): self.result(), call_code, code.error_goto_if_null(self.result(), self.pos) )) - code.put_gotref(self.result()) + self.generate_gotref(code) class GeneralCallNode(CallNode): @@ -6449,7 +6880,7 @@ class GeneralCallNode(CallNode): kwargs = self.keyword_args declared_args = function_type.args if entry.is_cmethod: - declared_args = declared_args[1:] # skip 'self' + declared_args = declared_args[1:] # skip 'self' if len(pos_args) > len(declared_args): error(self.pos, "function call got too many positional arguments, " @@ -6457,8 +6888,10 @@ class GeneralCallNode(CallNode): len(pos_args))) return None - matched_args = set([ arg.name for arg in declared_args[:len(pos_args)] - if arg.name ]) + matched_args = { + arg.name for arg in declared_args[:len(pos_args)] + if arg.name + } unmatched_args = declared_args[len(pos_args):] matched_kwargs_count = 0 args = list(pos_args) @@ -6576,7 +7009,7 @@ class GeneralCallNode(CallNode): self.positional_args.py_result(), kwargs, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class AsTupleNode(ExprNode): @@ -6618,7 +7051,7 @@ class AsTupleNode(ExprNode): self.result(), cfunc, self.arg.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class MergedDictNode(ExprNode): @@ -6711,16 +7144,18 @@ class MergedDictNode(ExprNode): self.result(), item.py_result(), code.error_goto_if_null(self.result(), item.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) item.generate_disposal_code(code) if item.type is not dict_type: code.putln('} else {') - code.putln("%s = PyObject_CallFunctionObjArgs((PyObject*)&PyDict_Type, %s, NULL); %s" % ( + code.globalstate.use_utility_code(UtilityCode.load_cached( + "PyObjectCallOneArg", "ObjectHandling.c")) + code.putln("%s = __Pyx_PyObject_CallOneArg((PyObject*)&PyDict_Type, %s); %s" % ( self.result(), item.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) item.generate_disposal_code(code) code.putln('}') item.free_temps(code) @@ -6792,7 +7227,6 @@ class AttributeNode(ExprNode): is_attribute = 1 subexprs = ['obj'] - type = PyrexTypes.error_type entry = None is_called = 0 needs_none_check = True @@ -6822,6 +7256,35 @@ class AttributeNode(ExprNode): self.entry = entry.as_variable self.analyse_as_python_attribute(env) return self + elif entry and entry.is_cfunction and self.obj.type is not Builtin.type_type: + # "bound" cdef function. + # This implementation is likely a little inefficient and could be improved. + # Essentially it does: + # __import__("functools").partial(coerce_to_object(self), self.obj) + from .UtilNodes import EvalWithTempExprNode, ResultRefNode + # take self.obj out to a temp because it's used twice + obj_node = ResultRefNode(self.obj, type=self.obj.type) + obj_node.result_ctype = self.obj.result_ctype + self.obj = obj_node + unbound_node = ExprNode.coerce_to(self, dst_type, env) + utility_code=UtilityCode.load_cached( + "PyMethodNew2Arg", "ObjectHandling.c" + ) + func_type = PyrexTypes.CFuncType( + PyrexTypes.py_object_type, [ + PyrexTypes.CFuncTypeArg("func", PyrexTypes.py_object_type, None), + PyrexTypes.CFuncTypeArg("self", PyrexTypes.py_object_type, None) + ], + ) + binding_call = PythonCapiCallNode( + self.pos, + function_name="__Pyx_PyMethod_New2Arg", + func_type=func_type, + args=[unbound_node, obj_node], + utility_code=utility_code, + ) + complete_call = EvalWithTempExprNode(obj_node, binding_call) + return complete_call.analyse_types(env) return ExprNode.coerce_to(self, dst_type, env) def calculate_constant_result(self): @@ -6871,7 +7334,7 @@ class AttributeNode(ExprNode): return self.type def analyse_target_declaration(self, env): - pass + self.is_target = True def analyse_target_types(self, env): node = self.analyse_types(env, target = 1) @@ -6882,6 +7345,8 @@ class AttributeNode(ExprNode): return node def analyse_types(self, env, target = 0): + if not self.type: + self.type = PyrexTypes.error_type # default value if it isn't analysed successfully self.initialized_check = env.directives['initializedcheck'] node = self.analyse_as_cimported_attribute_node(env, target) if node is None and not target: @@ -6889,7 +7354,7 @@ class AttributeNode(ExprNode): if node is None: node = self.analyse_as_ordinary_attribute_node(env, target) assert node is not None - if node.entry: + if (node.is_attribute or node.is_name) and node.entry: node.entry.used = True if node.is_attribute: node.wrap_obj_in_nonecheck(env) @@ -6908,6 +7373,7 @@ class AttributeNode(ExprNode): or entry.is_type or entry.is_const): return self.as_name_node(env, entry, target) if self.is_cimported_module_without_shadow(env): + # TODO: search for submodule error(self.pos, "cimported module has no attribute '%s'" % self.attribute) return self return None @@ -6930,31 +7396,13 @@ class AttributeNode(ExprNode): return None ubcm_entry = entry else: - # Create a temporary entry describing the C method - # as an ordinary function. - if entry.func_cname and not hasattr(entry.type, 'op_arg_struct'): - cname = entry.func_cname - if entry.type.is_static_method or ( - env.parent_scope and env.parent_scope.is_cpp_class_scope): - ctype = entry.type - elif type.is_cpp_class: - error(self.pos, "%s not a static member of %s" % (entry.name, type)) - ctype = PyrexTypes.error_type - else: - # Fix self type. - ctype = copy.copy(entry.type) - ctype.args = ctype.args[:] - ctype.args[0] = PyrexTypes.CFuncTypeArg('self', type, 'self', None) - else: - cname = "%s->%s" % (type.vtabptr_cname, entry.cname) - ctype = entry.type - ubcm_entry = Symtab.Entry(entry.name, cname, ctype) - ubcm_entry.is_cfunction = 1 - ubcm_entry.func_cname = entry.func_cname - ubcm_entry.is_unbound_cmethod = 1 - ubcm_entry.scope = entry.scope + ubcm_entry = self._create_unbound_cmethod_entry(type, entry, env) + ubcm_entry.overloaded_alternatives = [ + self._create_unbound_cmethod_entry(type, overloaded_alternative, env) + for overloaded_alternative in entry.overloaded_alternatives + ] return self.as_name_node(env, ubcm_entry, target=False) - elif type.is_enum: + elif type.is_enum or type.is_cpp_enum: if self.attribute in type.values: for entry in type.entry.enum_values: if entry.name == self.attribute: @@ -6965,13 +7413,39 @@ class AttributeNode(ExprNode): error(self.pos, "%s not a known value of %s" % (self.attribute, type)) return None + def _create_unbound_cmethod_entry(self, type, entry, env): + # Create a temporary entry describing the unbound C method in `entry` + # as an ordinary function. + if entry.func_cname and entry.type.op_arg_struct is None: + cname = entry.func_cname + if entry.type.is_static_method or ( + env.parent_scope and env.parent_scope.is_cpp_class_scope): + ctype = entry.type + elif type.is_cpp_class: + error(self.pos, "%s not a static member of %s" % (entry.name, type)) + ctype = PyrexTypes.error_type + else: + # Fix self type. + ctype = copy.copy(entry.type) + ctype.args = ctype.args[:] + ctype.args[0] = PyrexTypes.CFuncTypeArg('self', type, 'self', None) + else: + cname = "%s->%s" % (type.vtabptr_cname, entry.cname) + ctype = entry.type + ubcm_entry = Symtab.Entry(entry.name, cname, ctype) + ubcm_entry.is_cfunction = 1 + ubcm_entry.func_cname = entry.func_cname + ubcm_entry.is_unbound_cmethod = 1 + ubcm_entry.scope = entry.scope + return ubcm_entry + def analyse_as_type(self, env): module_scope = self.obj.analyse_as_module(env) if module_scope: return module_scope.lookup_type(self.attribute) if not self.obj.is_string_literal: base_type = self.obj.analyse_as_type(env) - if base_type and hasattr(base_type, 'scope') and base_type.scope is not None: + if base_type and getattr(base_type, 'scope', None) is not None: return base_type.scope.lookup_type(self.attribute) return None @@ -7022,13 +7496,18 @@ class AttributeNode(ExprNode): self.result_ctype = py_object_type elif target and self.obj.type.is_builtin_type: error(self.pos, "Assignment to an immutable object field") + elif self.entry and self.entry.is_cproperty: + if not target: + return SimpleCallNode.for_cproperty(self.pos, self.obj, self.entry).analyse_types(env) + # TODO: implement writable C-properties? + error(self.pos, "Assignment to a read-only property") #elif self.type.is_memoryviewslice and not target: # self.is_temp = True return self def analyse_attribute(self, env, obj_type = None): # Look up attribute and set self.type and self.member. - immutable_obj = obj_type is not None # used during type inference + immutable_obj = obj_type is not None # used during type inference self.is_py_attr = 0 self.member = self.attribute if obj_type is None: @@ -7078,7 +7557,10 @@ class AttributeNode(ExprNode): # fused function go through assignment synthesis # (foo = pycfunction(foo_func_obj)) and need to go through # regular Python lookup as well - if (entry.is_variable and not entry.fused_cfunction) or entry.is_cmethod: + if entry.is_cproperty: + self.type = entry.type + return + elif (entry.is_variable and not entry.fused_cfunction) or entry.is_cmethod: self.type = entry.type self.member = entry.cname return @@ -7105,15 +7587,15 @@ class AttributeNode(ExprNode): if not obj_type.is_pyobject and not obj_type.is_error: # Expose python methods for immutable objects. if (obj_type.is_string or obj_type.is_cpp_string - or obj_type.is_buffer or obj_type.is_memoryviewslice - or obj_type.is_numeric - or (obj_type.is_ctuple and obj_type.can_coerce_to_pyobject(env)) - or (obj_type.is_struct and obj_type.can_coerce_to_pyobject(env))): + or obj_type.is_buffer or obj_type.is_memoryviewslice + or obj_type.is_numeric + or (obj_type.is_ctuple and obj_type.can_coerce_to_pyobject(env)) + or (obj_type.is_struct and obj_type.can_coerce_to_pyobject(env))): if not immutable_obj: self.obj = self.obj.coerce_to_pyobject(env) elif (obj_type.is_cfunction and (self.obj.is_name or self.obj.is_attribute) - and self.obj.entry.as_variable - and self.obj.entry.as_variable.type.is_pyobject): + and self.obj.entry.as_variable + and self.obj.entry.as_variable.type.is_pyobject): # might be an optimised builtin function => unpack it if not immutable_obj: self.obj = self.obj.coerce_to_pyobject(env) @@ -7174,9 +7656,14 @@ class AttributeNode(ExprNode): return NameNode.is_ephemeral(self) def calculate_result_code(self): - #print "AttributeNode.calculate_result_code:", self.member ### - #print "...obj node =", self.obj, "code", self.obj.result() ### - #print "...obj type", self.obj.type, "ctype", self.obj.ctype() ### + result = self.calculate_access_code() + if self.entry and self.entry.is_cpp_optional and not self.is_target: + result = "(*%s)" % result + return result + + def calculate_access_code(self): + # Does the job of calculate_result_code but doesn't dereference cpp_optionals + # Therefore allowing access to the holder variable obj = self.obj obj_code = obj.result_as(obj.type) #print "...obj_code =", obj_code ### @@ -7227,7 +7714,7 @@ class AttributeNode(ExprNode): self.obj.py_result(), code.intern_identifier(self.attribute), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif self.type.is_memoryviewslice: if self.is_memslice_transpose: # transpose the slice @@ -7238,10 +7725,11 @@ class AttributeNode(ExprNode): return code.putln("%s = %s;" % (self.result(), self.obj.result())) - code.put_incref_memoryviewslice(self.result(), have_gil=True) + code.put_incref_memoryviewslice(self.result(), self.type, + have_gil=True) - T = "__pyx_memslice_transpose(&%s) == 0" - code.putln(code.error_goto_if(T % self.result(), self.pos)) + T = "__pyx_memslice_transpose(&%s)" % self.result() + code.putln(code.error_goto_if_neg(T, self.pos)) elif self.initialized_check: code.putln( 'if (unlikely(!%s.memview)) {' @@ -7249,6 +7737,14 @@ class AttributeNode(ExprNode): '"Memoryview is not initialized");' '%s' '}' % (self.result(), code.error_goto(self.pos))) + elif self.entry.is_cpp_optional and self.initialized_check: + if self.is_target: + undereferenced_result = self.result() + else: + assert not self.is_temp # calculate_access_code() only makes sense for non-temps + undereferenced_result = self.calculate_access_code() + unbound_check_code = self.type.cpp_optional_check_for_null_code(undereferenced_result) + code.put_error_if_unbound(self.pos, self.entry, unbound_check_code=unbound_check_code) else: # result_code contains what is needed, but we may need to insert # a check and raise an exception @@ -7261,15 +7757,12 @@ class AttributeNode(ExprNode): def generate_disposal_code(self, code): if self.is_temp and self.type.is_memoryviewslice and self.is_memslice_transpose: # mirror condition for putting the memview incref here: - code.put_xdecref_memoryviewslice( - self.result(), have_gil=True) - code.putln("%s.memview = NULL;" % self.result()) - code.putln("%s.data = NULL;" % self.result()) + code.put_xdecref_clear(self.result(), self.type, have_gil=True) else: ExprNode.generate_disposal_code(self, code) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): self.obj.generate_evaluation_code(code) if self.is_py_attr: code.globalstate.use_utility_code( @@ -7282,8 +7775,9 @@ class AttributeNode(ExprNode): rhs.generate_disposal_code(code) rhs.free_temps(code) elif self.obj.type.is_complex: - code.putln("__Pyx_SET_C%s(%s, %s);" % ( + code.putln("__Pyx_SET_C%s%s(%s, %s);" % ( self.member.upper(), + self.obj.type.implementation_suffix, self.obj.result_as(self.obj.type), rhs.result_as(self.ctype()))) rhs.generate_disposal_code(code) @@ -7292,8 +7786,8 @@ class AttributeNode(ExprNode): select_code = self.result() if self.type.is_pyobject and self.use_managed_ref: rhs.make_owned_reference(code) - code.put_giveref(rhs.py_result()) - code.put_gotref(select_code) + rhs.generate_giveref(code) + code.put_gotref(select_code, self.type) code.put_decref(select_code, self.ctype()) elif self.type.is_memoryviewslice: from . import MemoryView @@ -7304,7 +7798,7 @@ class AttributeNode(ExprNode): code.putln( "%s = %s;" % ( select_code, - rhs.result_as(self.ctype()))) + rhs.move_result_rhs_as(self.ctype()))) #rhs.result())) rhs.generate_post_assignment_code(code) rhs.free_temps(code) @@ -7333,6 +7827,12 @@ class AttributeNode(ExprNode): style, text = 'c_attr', 'c attribute (%s)' code.annotate(self.pos, AnnotationItem(style, text % self.type, size=len(self.attribute))) + def get_known_standard_library_import(self): + module_name = self.obj.get_known_standard_library_import() + if module_name: + return StringEncoding.EncodedString("%s.%s" % (module_name, self.attribute)) + return None + #------------------------------------------------------------------- # @@ -7435,9 +7935,10 @@ class SequenceNode(ExprNode): arg = arg.analyse_types(env) self.args[i] = arg.coerce_to_pyobject(env) if self.mult_factor: - self.mult_factor = self.mult_factor.analyse_types(env) - if not self.mult_factor.type.is_int: - self.mult_factor = self.mult_factor.coerce_to_pyobject(env) + mult_factor = self.mult_factor.analyse_types(env) + if not mult_factor.type.is_int: + mult_factor = mult_factor.coerce_to_pyobject(env) + self.mult_factor = mult_factor.coerce_to_simple(env) self.is_temp = 1 # not setting self.type here, subtypes do this return self @@ -7539,7 +8040,7 @@ class SequenceNode(ExprNode): len(self.args), ', '.join(arg.py_result() for arg in self.args), code.error_goto_if_null(target, self.pos))) - code.put_gotref(target) + code.put_gotref(target, py_object_type) elif self.type.is_ctuple: for i, arg in enumerate(self.args): code.putln("%s.f%s = %s;" % ( @@ -7556,7 +8057,7 @@ class SequenceNode(ExprNode): code.putln("%s = %s(%s%s); %s" % ( target, create_func, arg_count, size_factor, code.error_goto_if_null(target, self.pos))) - code.put_gotref(target) + code.put_gotref(target, py_object_type) if c_mult: # FIXME: can't use a temp variable here as the code may @@ -7580,7 +8081,7 @@ class SequenceNode(ExprNode): arg = self.args[i] if c_mult or not arg.result_in_temp(): code.put_incref(arg.result(), arg.ctype()) - code.put_giveref(arg.py_result()) + arg.generate_giveref(code) code.putln("%s(%s, %s, %s);" % ( set_item_func, target, @@ -7597,7 +8098,7 @@ class SequenceNode(ExprNode): Naming.quick_temp_cname, target, mult_factor.py_result(), code.error_goto_if_null(Naming.quick_temp_cname, self.pos) )) - code.put_gotref(Naming.quick_temp_cname) + code.put_gotref(Naming.quick_temp_cname, py_object_type) code.put_decref(target, py_object_type) code.putln('%s = %s;' % (target, Naming.quick_temp_cname)) code.putln('}') @@ -7619,7 +8120,7 @@ class SequenceNode(ExprNode): self.mult_factor.generate_disposal_code(code) def generate_assignment_code(self, rhs, code, overloaded_assignment=False, - exception_check=None, exception_value=None): + exception_check=None, exception_value=None): if self.starred_assignment: self.generate_starred_assignment_code(rhs, code) else: @@ -7683,10 +8184,12 @@ class SequenceNode(ExprNode): # list/tuple => check size code.putln("Py_ssize_t size = __Pyx_PySequence_SIZE(sequence);") code.putln("if (unlikely(size != %d)) {" % len(self.args)) - code.globalstate.use_utility_code(raise_too_many_values_to_unpack) + code.globalstate.use_utility_code( + UtilityCode.load_cached("RaiseTooManyValuesToUnpack", "ObjectHandling.c")) code.putln("if (size > %d) __Pyx_RaiseTooManyValuesError(%d);" % ( len(self.args), len(self.args))) - code.globalstate.use_utility_code(raise_need_more_values_to_unpack) + code.globalstate.use_utility_code( + UtilityCode.load_cached("RaiseNeedMoreValuesToUnpack", "ObjectHandling.c")) code.putln("else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size);") # < 0 => exception code.putln(code.error_goto(self.pos)) @@ -7715,7 +8218,7 @@ class SequenceNode(ExprNode): code.putln("%s = PySequence_ITEM(sequence, %d); %s" % ( item.result(), i, code.error_goto_if_null(item.result(), self.pos))) - code.put_gotref(item.result()) + code.put_gotref(item.result(), item.type) else: code.putln("{") code.putln("Py_ssize_t i;") @@ -7725,7 +8228,7 @@ class SequenceNode(ExprNode): code.putln("for (i=0; i < %s; i++) {" % len(self.unpacked_items)) code.putln("PyObject* item = PySequence_ITEM(sequence, i); %s" % ( code.error_goto_if_null('item', self.pos))) - code.put_gotref('item') + code.put_gotref('item', py_object_type) code.putln("*(temps[i]) = item;") code.putln("}") code.putln("}") @@ -7749,9 +8252,11 @@ class SequenceNode(ExprNode): code.putln("}") def generate_generic_parallel_unpacking_code(self, code, rhs, unpacked_items, use_loop, terminate=True): - code.globalstate.use_utility_code(raise_need_more_values_to_unpack) - code.globalstate.use_utility_code(UtilityCode.load_cached("IterFinish", "ObjectHandling.c")) - code.putln("Py_ssize_t index = -1;") # must be at the start of a C block! + code.globalstate.use_utility_code( + UtilityCode.load_cached("RaiseNeedMoreValuesToUnpack", "ObjectHandling.c")) + code.globalstate.use_utility_code( + UtilityCode.load_cached("IterFinish", "ObjectHandling.c")) + code.putln("Py_ssize_t index = -1;") # must be at the start of a C block! if use_loop: code.putln("PyObject** temps[%s] = {%s};" % ( @@ -7764,11 +8269,11 @@ class SequenceNode(ExprNode): iterator_temp, rhs.py_result(), code.error_goto_if_null(iterator_temp, self.pos))) - code.put_gotref(iterator_temp) + code.put_gotref(iterator_temp, py_object_type) rhs.generate_disposal_code(code) iternext_func = code.funcstate.allocate_temp(self._func_iternext_type, manage_ref=False) - code.putln("%s = Py_TYPE(%s)->tp_iternext;" % ( + code.putln("%s = __Pyx_PyObject_GetIterNextFunc(%s);" % ( iternext_func, iterator_temp)) unpacking_error_label = code.new_label('unpacking_failed') @@ -7777,7 +8282,7 @@ class SequenceNode(ExprNode): code.putln("for (index=0; index < %s; index++) {" % len(unpacked_items)) code.put("PyObject* item = %s; if (unlikely(!item)) " % unpack_code) code.put_goto(unpacking_error_label) - code.put_gotref("item") + code.put_gotref("item", py_object_type) code.putln("*(temps[index]) = item;") code.putln("}") else: @@ -7789,7 +8294,7 @@ class SequenceNode(ExprNode): unpack_code, item.result())) code.put_goto(unpacking_error_label) - code.put_gotref(item.py_result()) + item.generate_gotref(code) if terminate: code.globalstate.use_utility_code( @@ -7842,11 +8347,14 @@ class SequenceNode(ExprNode): starred_target.allocate(code) target_list = starred_target.result() - code.putln("%s = PySequence_List(%s); %s" % ( + code.putln("%s = %s(%s); %s" % ( target_list, + "__Pyx_PySequence_ListKeepNew" if ( + not iterator_temp and rhs.is_temp and rhs.type in (py_object_type, list_type)) + else "PySequence_List", iterator_temp or rhs.py_result(), code.error_goto_if_null(target_list, self.pos))) - code.put_gotref(target_list) + starred_target.generate_gotref(code) if iterator_temp: code.put_decref_clear(iterator_temp, py_object_type) @@ -7855,7 +8363,8 @@ class SequenceNode(ExprNode): rhs.generate_disposal_code(code) if unpacked_fixed_items_right: - code.globalstate.use_utility_code(raise_need_more_values_to_unpack) + code.globalstate.use_utility_code( + UtilityCode.load_cached("RaiseNeedMoreValuesToUnpack", "ObjectHandling.c")) length_temp = code.funcstate.allocate_temp(PyrexTypes.c_py_ssize_t_type, manage_ref=False) code.putln('%s = PyList_GET_SIZE(%s);' % (length_temp, target_list)) code.putln("if (unlikely(%s < %d)) {" % (length_temp, len(unpacked_fixed_items_right))) @@ -7877,7 +8386,7 @@ class SequenceNode(ExprNode): code.putln("%s = PySequence_ITEM(%s, %s-%d); " % ( item.py_result(), target_list, length_temp, i+1)) code.putln('#endif') - code.put_gotref(item.py_result()) + item.generate_gotref(code) coerced_arg.generate_evaluation_code(code) code.putln('#if !CYTHON_COMPILING_IN_CPYTHON') @@ -7885,12 +8394,12 @@ class SequenceNode(ExprNode): code.putln('%s = PySequence_GetSlice(%s, 0, %s-%d); %s' % ( sublist_temp, target_list, length_temp, len(unpacked_fixed_items_right), code.error_goto_if_null(sublist_temp, self.pos))) - code.put_gotref(sublist_temp) + code.put_gotref(sublist_temp, py_object_type) code.funcstate.release_temp(length_temp) code.put_decref(target_list, py_object_type) code.putln('%s = %s; %s = NULL;' % (target_list, sublist_temp, sublist_temp)) code.putln('#else') - code.putln('(void)%s;' % sublist_temp) # avoid warning about unused variable + code.putln('CYTHON_UNUSED_VAR(%s);' % sublist_temp) code.funcstate.release_temp(sublist_temp) code.putln('#endif') @@ -7925,6 +8434,12 @@ class TupleNode(SequenceNode): return env.declare_tuple_type(self.pos, arg_types).type def analyse_types(self, env, skip_children=False): + # reset before re-analysing + if self.is_literal: + self.is_literal = False + if self.is_partly_literal: + self.is_partly_literal = False + if len(self.args) == 0: self.is_temp = False self.is_literal = True @@ -7955,7 +8470,7 @@ class TupleNode(SequenceNode): node.is_temp = False node.is_literal = True else: - if not node.mult_factor.type.is_pyobject: + if not node.mult_factor.type.is_pyobject and not node.mult_factor.type.is_int: node.mult_factor = node.mult_factor.coerce_to_pyobject(env) node.is_temp = True node.is_partly_literal = True @@ -7977,7 +8492,13 @@ class TupleNode(SequenceNode): return self.coerce_to_ctuple(dst_type, env) elif dst_type is tuple_type or dst_type is py_object_type: coerced_args = [arg.coerce_to_pyobject(env) for arg in self.args] - return TupleNode(self.pos, args=coerced_args, type=tuple_type, is_temp=1).analyse_types(env, skip_children=True) + return TupleNode( + self.pos, + args=coerced_args, + type=tuple_type, + mult_factor=self.mult_factor, + is_temp=1, + ).analyse_types(env, skip_children=True) else: return self.coerce_to_pyobject(env).coerce_to(dst_type, env) elif dst_type.is_ctuple and not self.mult_factor: @@ -8031,15 +8552,23 @@ class TupleNode(SequenceNode): # constant is not yet initialised const_code.mark_pos(self.pos) self.generate_sequence_packing_code(const_code, tuple_target, plain=not self.is_literal) - const_code.put_giveref(tuple_target) + const_code.put_giveref(tuple_target, py_object_type) if self.is_literal: self.result_code = tuple_target + elif self.mult_factor.type.is_int: + code.globalstate.use_utility_code( + UtilityCode.load_cached("PySequenceMultiply", "ObjectHandling.c")) + code.putln('%s = __Pyx_PySequence_Multiply(%s, %s); %s' % ( + self.result(), tuple_target, self.mult_factor.result(), + code.error_goto_if_null(self.result(), self.pos) + )) + self.generate_gotref(code) else: code.putln('%s = PyNumber_Multiply(%s, %s); %s' % ( self.result(), tuple_target, self.mult_factor.py_result(), code.error_goto_if_null(self.result(), self.pos) )) - code.put_gotref(self.py_result()) + self.generate_gotref(code) else: self.type.entry.used = True self.generate_sequence_packing_code(code) @@ -8203,97 +8732,6 @@ class ListNode(SequenceNode): raise InternalError("List type never specified") -class ScopedExprNode(ExprNode): - # Abstract base class for ExprNodes that have their own local - # scope, such as generator expressions. - # - # expr_scope Scope the inner scope of the expression - - subexprs = [] - expr_scope = None - - # does this node really have a local scope, e.g. does it leak loop - # variables or not? non-leaking Py3 behaviour is default, except - # for list comprehensions where the behaviour differs in Py2 and - # Py3 (set in Parsing.py based on parser context) - has_local_scope = True - - def init_scope(self, outer_scope, expr_scope=None): - if expr_scope is not None: - self.expr_scope = expr_scope - elif self.has_local_scope: - self.expr_scope = Symtab.GeneratorExpressionScope(outer_scope) - else: - self.expr_scope = None - - def analyse_declarations(self, env): - self.init_scope(env) - - def analyse_scoped_declarations(self, env): - # this is called with the expr_scope as env - pass - - def analyse_types(self, env): - # no recursion here, the children will be analysed separately below - return self - - def analyse_scoped_expressions(self, env): - # this is called with the expr_scope as env - return self - - def generate_evaluation_code(self, code): - # set up local variables and free their references on exit - generate_inner_evaluation_code = super(ScopedExprNode, self).generate_evaluation_code - if not self.has_local_scope or not self.expr_scope.var_entries: - # no local variables => delegate, done - generate_inner_evaluation_code(code) - return - - code.putln('{ /* enter inner scope */') - py_entries = [] - for _, entry in sorted(item for item in self.expr_scope.entries.items() if item[0]): - if not entry.in_closure: - if entry.type.is_pyobject and entry.used: - py_entries.append(entry) - if not py_entries: - # no local Python references => no cleanup required - generate_inner_evaluation_code(code) - code.putln('} /* exit inner scope */') - return - - # must free all local Python references at each exit point - old_loop_labels = code.new_loop_labels() - old_error_label = code.new_error_label() - - generate_inner_evaluation_code(code) - - # normal (non-error) exit - self._generate_vars_cleanup(code, py_entries) - - # error/loop body exit points - exit_scope = code.new_label('exit_scope') - code.put_goto(exit_scope) - for label, old_label in ([(code.error_label, old_error_label)] + - list(zip(code.get_loop_labels(), old_loop_labels))): - if code.label_used(label): - code.put_label(label) - self._generate_vars_cleanup(code, py_entries) - code.put_goto(old_label) - code.put_label(exit_scope) - code.putln('} /* exit inner scope */') - - code.set_loop_labels(old_loop_labels) - code.error_label = old_error_label - - def _generate_vars_cleanup(self, code, py_entries): - for entry in py_entries: - if entry.is_cglobal: - code.put_var_gotref(entry) - code.put_decref_set(entry.cname, "Py_None") - else: - code.put_var_xdecref_clear(entry) - - class ComprehensionNode(ScopedExprNode): # A list/set/dict comprehension @@ -8306,8 +8744,14 @@ class ComprehensionNode(ScopedExprNode): return self.type def analyse_declarations(self, env): - self.append.target = self # this is used in the PyList_Append of the inner loop + self.append.target = self # this is used in the PyList_Append of the inner loop self.init_scope(env) + # setup loop scope + if isinstance(self.loop, Nodes._ForInStatNode): + assert isinstance(self.loop.iterator, ScopedExprNode), self.loop.iterator + self.loop.iterator.init_scope(None, env) + else: + assert isinstance(self.loop, Nodes.ForFromStatNode), self.loop def analyse_scoped_declarations(self, env): self.loop.analyse_declarations(env) @@ -8341,7 +8785,7 @@ class ComprehensionNode(ScopedExprNode): self.result(), create_code, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) self.loop.generate_execution_code(code) def annotate(self, code): @@ -8466,7 +8910,7 @@ class InlinedGeneratorExpressionNode(ExprNode): code.putln("%s = __Pyx_Generator_Next(%s); %s" % ( self.result(), self.gen.result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) class MergedSequenceNode(ExprNode): @@ -8574,10 +9018,12 @@ class MergedSequenceNode(ExprNode): else: code.putln("%s = %s(%s); %s" % ( self.result(), - 'PySet_New' if is_set else 'PySequence_List', + 'PySet_New' if is_set + else "__Pyx_PySequence_ListKeepNew" if item.is_temp and item.type in (py_object_type, list_type) + else "PySequence_List", item.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) item.generate_disposal_code(code) item.free_temps(code) @@ -8627,7 +9073,7 @@ class MergedSequenceNode(ExprNode): self.result(), Naming.quick_temp_cname, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) code.putln("}") for helper in sorted(helpers): @@ -8660,7 +9106,7 @@ class SetNode(ExprNode): return False def calculate_constant_result(self): - self.constant_result = set([arg.constant_result for arg in self.args]) + self.constant_result = {arg.constant_result for arg in self.args} def compile_time_value(self, denv): values = [arg.compile_time_value(denv) for arg in self.args] @@ -8677,7 +9123,7 @@ class SetNode(ExprNode): "%s = PySet_New(0); %s" % ( self.result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) for arg in self.args: code.put_error_if_neg( self.pos, @@ -8764,7 +9210,7 @@ class DictNode(ExprNode): error(item.key.pos, "Invalid struct field identifier") item.key = StringNode(item.key.pos, value="<error>") else: - key = str(item.key.value) # converts string literals to unicode in Py3 + key = str(item.key.value) # converts string literals to unicode in Py3 member = dst_type.scope.lookup_here(key) if not member: error(item.key.pos, "struct '%s' has no field '%s'" % (dst_type, key)) @@ -8774,8 +9220,7 @@ class DictNode(ExprNode): value = value.arg item.value = value.coerce_to(member.type, env) else: - self.type = error_type - error(self.pos, "Cannot interpret dict as type '%s'" % dst_type) + return super(DictNode, self).coerce_to(dst_type, env) return self def release_errors(self): @@ -8799,7 +9244,7 @@ class DictNode(ExprNode): self.result(), len(self.key_value_pairs), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) keys_seen = set() key_type = None @@ -8848,10 +9293,17 @@ class DictNode(ExprNode): if self.exclude_null_values: code.putln('}') else: - code.putln("%s.%s = %s;" % ( - self.result(), - item.key.value, - item.value.result())) + if item.value.type.is_array: + code.putln("memcpy(%s.%s, %s, sizeof(%s));" % ( + self.result(), + item.key.value, + item.value.result(), + item.value.result())) + else: + code.putln("%s.%s = %s;" % ( + self.result(), + item.key.value, + item.value.result())) item.generate_disposal_code(code) item.free_temps(code) @@ -8863,6 +9315,11 @@ class DictNode(ExprNode): for item in self.key_value_pairs: item.annotate(code) + def as_python_dict(self): + # returns a dict with constant keys and Node values + # (only works on DictNodes where the keys are ConstNodes or PyConstNode) + return dict([(key.value, value) for key, value in self.key_value_pairs]) + class DictItemNode(ExprNode): # Represents a single item in a DictNode @@ -8871,7 +9328,7 @@ class DictItemNode(ExprNode): # value ExprNode subexprs = ['key', 'value'] - nogil_check = None # Parent DictNode takes care of it + nogil_check = None # Parent DictNode takes care of it def calculate_constant_result(self): self.constant_result = ( @@ -8927,7 +9384,7 @@ class SortedDictKeysNode(ExprNode): code.putln('%s = PyDict_Keys(%s); %s' % ( self.result(), dict_result, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) else: # originally used PyMapping_Keys() here, but that may return a tuple code.globalstate.use_utility_code(UtilityCode.load_cached( @@ -8936,11 +9393,11 @@ class SortedDictKeysNode(ExprNode): code.putln('%s = __Pyx_PyObject_CallMethod0(%s, %s); %s' % ( self.result(), dict_result, keys_cname, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) code.putln("if (unlikely(!PyList_Check(%s))) {" % self.result()) - code.put_decref_set(self.result(), "PySequence_List(%s)" % self.result()) + self.generate_decref_set(code, "PySequence_List(%s)" % self.result()) code.putln(code.error_goto_if_null(self.result(), self.pos)) - code.put_gotref(self.py_result()) + self.generate_gotref(code) code.putln("}") code.put_error_if_neg( self.pos, 'PyList_Sort(%s)' % self.py_result()) @@ -8970,6 +9427,9 @@ class ClassNode(ExprNode, ModuleNameMixin): type = py_object_type is_temp = True + def analyse_annotations(self, env): + pass + def infer_type(self, env): # TODO: could return 'type' in some cases return py_object_type @@ -9008,7 +9468,7 @@ class ClassNode(ExprNode, ModuleNameMixin): qualname, py_mod_name, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class Py3ClassNode(ExprNode): @@ -9021,9 +9481,11 @@ class Py3ClassNode(ExprNode): # class_def_node PyClassDefNode PyClassDefNode defining this class # calculate_metaclass bool should call CalculateMetaclass() # allow_py2_metaclass bool should look for Py2 metaclass + # force_type bool always create a "new style" class, even with no bases subexprs = [] type = py_object_type + force_type = False is_temp = True def infer_type(self, env): @@ -9038,6 +9500,26 @@ class Py3ClassNode(ExprNode): gil_message = "Constructing Python class" + def analyse_annotations(self, env): + from .AutoDocTransforms import AnnotationWriter + position = self.class_def_node.pos + dict_items = [ + DictItemNode( + entry.pos, + key=IdentifierStringNode(entry.pos, value=entry.name), + value=entry.annotation.string + ) + for entry in env.entries.values() if entry.annotation + ] + # Annotations dict shouldn't exist for classes which don't declare any. + if dict_items: + annotations_dict = DictNode(position, key_value_pairs=dict_items) + lhs = NameNode(position, name=StringEncoding.EncodedString(u"__annotations__")) + lhs.entry = env.lookup_here(lhs.name) or env.declare_var(lhs.name, dict_type, position) + node = SingleAssignmentNode(position, lhs=lhs, rhs=annotations_dict) + node.analyse_declarations(env) + self.class_def_node.body.stats.insert(0, node) + def generate_result_code(self, code): code.globalstate.use_utility_code(UtilityCode.load_cached("Py3ClassCreate", "ObjectHandling.c")) cname = code.intern_identifier(self.name) @@ -9045,6 +9527,8 @@ class Py3ClassNode(ExprNode): mkw = class_def_node.mkw.py_result() if class_def_node.mkw else 'NULL' if class_def_node.metaclass: metaclass = class_def_node.metaclass.py_result() + elif self.force_type: + metaclass = "((PyObject*)&PyType_Type)" else: metaclass = "((PyObject*)&__Pyx_DefaultClassType)" code.putln( @@ -9058,7 +9542,7 @@ class Py3ClassNode(ExprNode): self.calculate_metaclass, self.allow_py2_metaclass, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class PyClassMetaclassNode(ExprNode): @@ -9094,7 +9578,7 @@ class PyClassMetaclassNode(ExprNode): "%s = %s; %s" % ( self.result(), call, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class PyClassNamespaceNode(ExprNode, ModuleNameMixin): @@ -9136,7 +9620,7 @@ class PyClassNamespaceNode(ExprNode, ModuleNameMixin): py_mod_name, doc_code, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class ClassCellInjectorNode(ExprNode): @@ -9155,7 +9639,7 @@ class ClassCellInjectorNode(ExprNode): '%s = PyList_New(0); %s' % ( self.result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) def generate_injection_code(self, code, classobj_cname): assert self.is_active @@ -9197,7 +9681,6 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): # from a PyMethodDef struct. # # pymethdef_cname string PyMethodDef structure - # self_object ExprNode or None # binding bool # def_node DefNode the Python function node # module_name EncodedString Name of defining module @@ -9206,7 +9689,6 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): subexprs = ['code_object', 'defaults_tuple', 'defaults_kwdict', 'annotations_dict'] - self_object = None code_object = None binding = False def_node = None @@ -9255,33 +9737,35 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): must_use_constants = env.is_c_class_scope or (self.def_node.is_wrapper and env.is_module_scope) for arg in self.def_node.args: - if arg.default and not must_use_constants: - if not arg.default.is_literal: - arg.is_dynamic = True - if arg.type.is_pyobject: - nonliteral_objects.append(arg) + if arg.default: + if not must_use_constants: + if arg.default.is_literal: + arg.default = DefaultLiteralArgNode(arg.pos, arg.default) else: - nonliteral_other.append(arg) - else: - arg.default = DefaultLiteralArgNode(arg.pos, arg.default) - if arg.kw_only: - default_kwargs.append(arg) - else: - default_args.append(arg) + arg.is_dynamic = True + if arg.type.is_pyobject: + nonliteral_objects.append(arg) + else: + nonliteral_other.append(arg) + if arg.default.type and arg.default.type.can_coerce_to_pyobject(env): + if arg.kw_only: + default_kwargs.append(arg) + else: + default_args.append(arg) if arg.annotation: - arg.annotation = self.analyse_annotation(env, arg.annotation) - annotations.append((arg.pos, arg.name, arg.annotation)) + arg.annotation = arg.annotation.analyse_types(env) + annotations.append((arg.pos, arg.name, arg.annotation.string)) for arg in (self.def_node.star_arg, self.def_node.starstar_arg): if arg and arg.annotation: - arg.annotation = self.analyse_annotation(env, arg.annotation) - annotations.append((arg.pos, arg.name, arg.annotation)) + arg.annotation = arg.annotation.analyse_types(env) + annotations.append((arg.pos, arg.name, arg.annotation.string)) annotation = self.def_node.return_type_annotation if annotation: - annotation = self.analyse_annotation(env, annotation) - self.def_node.return_type_annotation = annotation - annotations.append((annotation.pos, StringEncoding.EncodedString("return"), annotation)) + self.def_node.return_type_annotation = annotation.analyse_types(env) + annotations.append((annotation.pos, StringEncoding.EncodedString("return"), + annotation.string)) if nonliteral_objects or nonliteral_other: module_scope = env.global_scope() @@ -9289,7 +9773,10 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): scope = Symtab.StructOrUnionScope(cname) self.defaults = [] for arg in nonliteral_objects: - entry = scope.declare_var(arg.name, arg.type, None, + type_ = arg.type + if type_.is_buffer: + type_ = type_.base + entry = scope.declare_var(arg.name, type_, None, Naming.arg_prefix + arg.name, allow_pyobject=True) self.defaults.append((arg, entry)) @@ -9321,7 +9808,8 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): value=arg.default) for arg in default_kwargs]) self.defaults_kwdict = defaults_kwdict.analyse_types(env) - else: + elif not self.specialized_cpdefs: + # Fused dispatch functions do not support (dynamic) default arguments, only the specialisations do. if default_args: defaults_tuple = DefaultsTupleNode( self.pos, default_args, self.defaults_struct) @@ -9358,31 +9846,13 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): for pos, name, value in annotations]) self.annotations_dict = annotations_dict.analyse_types(env) - def analyse_annotation(self, env, annotation): - if annotation is None: - return None - atype = annotation.analyse_as_type(env) - if atype is not None: - # Keep parsed types as strings as they might not be Python representable. - annotation = UnicodeNode( - annotation.pos, - value=StringEncoding.EncodedString(atype.declaration_code('', for_display=True))) - annotation = annotation.analyse_types(env) - if not annotation.type.is_pyobject: - annotation = annotation.coerce_to_pyobject(env) - return annotation - def may_be_none(self): return False gil_message = "Constructing Python function" - def self_result_code(self): - if self.self_object is None: - self_result = "NULL" - else: - self_result = self.self_object.py_result() - return self_result + def closure_result_code(self): + return "NULL" def generate_result_code(self, code): if self.binding: @@ -9396,11 +9866,11 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): '%s = PyCFunction_NewEx(&%s, %s, %s); %s' % ( self.result(), self.pymethdef_cname, - self.self_result_code(), + self.closure_result_code(), py_mod_name, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) def generate_cyfunction_code(self, code): if self.specialized_cpdefs: @@ -9431,6 +9901,9 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): if def_node.local_scope.parent_scope.is_c_class_scope and not def_node.entry.is_anonymous: flags.append('__Pyx_CYFUNCTION_CCLASS') + if def_node.is_coroutine: + flags.append('__Pyx_CYFUNCTION_COROUTINE') + if flags: flags = ' | '.join(flags) else: @@ -9443,13 +9916,13 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): self.pymethdef_cname, flags, self.get_py_qualified_name(code), - self.self_result_code(), + self.closure_result_code(), self.get_py_mod_name(code), Naming.moddict_cname, code_object_result, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) if def_node.requires_classobj: assert code.pyclass_stack, "pyclass_stack is empty" @@ -9459,7 +9932,7 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): 'PyList_Append(%s, %s);' % ( class_node.class_cell.result(), self.result())) - code.put_giveref(self.py_result()) + self.generate_giveref(code) if self.defaults: code.putln( @@ -9475,27 +9948,29 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): if self.defaults_tuple: code.putln('__Pyx_CyFunction_SetDefaultsTuple(%s, %s);' % ( self.result(), self.defaults_tuple.py_result())) - if self.defaults_kwdict: - code.putln('__Pyx_CyFunction_SetDefaultsKwDict(%s, %s);' % ( - self.result(), self.defaults_kwdict.py_result())) - if def_node.defaults_getter and not self.specialized_cpdefs: - # Fused functions do not support dynamic defaults, only their specialisations can have them for now. - code.putln('__Pyx_CyFunction_SetDefaultsGetter(%s, %s);' % ( - self.result(), def_node.defaults_getter.entry.pyfunc_cname)) - if self.annotations_dict: - code.putln('__Pyx_CyFunction_SetAnnotationsDict(%s, %s);' % ( - self.result(), self.annotations_dict.py_result())) + if not self.specialized_cpdefs: + # disable introspection functions for fused dispatcher function since the user never sees it + # TODO: this is mostly disabled because the attributes end up pointing to ones belonging + # to the specializations - ideally this would be fixed instead + if self.defaults_kwdict: + code.putln('__Pyx_CyFunction_SetDefaultsKwDict(%s, %s);' % ( + self.result(), self.defaults_kwdict.py_result())) + if def_node.defaults_getter: + code.putln('__Pyx_CyFunction_SetDefaultsGetter(%s, %s);' % ( + self.result(), def_node.defaults_getter.entry.pyfunc_cname)) + if self.annotations_dict: + code.putln('__Pyx_CyFunction_SetAnnotationsDict(%s, %s);' % ( + self.result(), self.annotations_dict.py_result())) class InnerFunctionNode(PyCFunctionNode): # Special PyCFunctionNode that depends on a closure class - # binding = True - needs_self_code = True + needs_closure_code = True - def self_result_code(self): - if self.needs_self_code: + def closure_result_code(self): + if self.needs_closure_code: return "((PyObject*)%s)" % Naming.cur_scope_cname return "NULL" @@ -9552,10 +10027,17 @@ class CodeObjectNode(ExprNode): flags.append('CO_VARARGS') if self.def_node.starstar_arg: flags.append('CO_VARKEYWORDS') - - code.putln("%s = (PyObject*)__Pyx_PyCode_New(%d, %d, %d, 0, %s, %s, %s, %s, %s, %s, %s, %s, %s, %d, %s); %s" % ( + if self.def_node.is_asyncgen: + flags.append('CO_ASYNC_GENERATOR') + elif self.def_node.is_coroutine: + flags.append('CO_COROUTINE') + elif self.def_node.is_generator: + flags.append('CO_GENERATOR') + + code.putln("%s = (PyObject*)__Pyx_PyCode_New(%d, %d, %d, %d, 0, %s, %s, %s, %s, %s, %s, %s, %s, %s, %d, %s); %s" % ( self.result_code, len(func.args) - func.num_kwonly_args, # argcount + func.num_posonly_args, # posonlyargcount (Py3.8+ only) func.num_kwonly_args, # kwonlyargcount (Py3 only) len(self.varnames.args), # nlocals '|'.join(flags) or '0', # flags @@ -9674,11 +10156,14 @@ class LambdaNode(InnerFunctionNode): name = StringEncoding.EncodedString('<lambda>') def analyse_declarations(self, env): + if hasattr(self, "lambda_name"): + # this if-statement makes it safe to run twice + return self.lambda_name = self.def_node.lambda_name = env.next_id('lambda') self.def_node.no_assignment_synthesis = True self.def_node.pymethdef_required = True - self.def_node.analyse_declarations(env) self.def_node.is_cyfunction = True + self.def_node.analyse_declarations(env) self.pymethdef_cname = self.def_node.entry.pymethdef_cname env.add_lambda_def(self.def_node) @@ -9698,11 +10183,22 @@ class GeneratorExpressionNode(LambdaNode): # # loop ForStatNode the for-loop, containing a YieldExprNode # def_node DefNode the underlying generator 'def' node + # call_parameters [ExprNode] (Internal) parameters passed to the DefNode call name = StringEncoding.EncodedString('genexpr') binding = False + child_attrs = LambdaNode.child_attrs + ["call_parameters"] + subexprs = LambdaNode.subexprs + ["call_parameters"] + + def __init__(self, pos, *args, **kwds): + super(GeneratorExpressionNode, self).__init__(pos, *args, **kwds) + self.call_parameters = [] + def analyse_declarations(self, env): + if hasattr(self, "genexpr_name"): + # this if-statement makes it safe to run twice + return self.genexpr_name = env.next_id('genexpr') super(GeneratorExpressionNode, self).analyse_declarations(env) # No pymethdef required @@ -9711,15 +10207,24 @@ class GeneratorExpressionNode(LambdaNode): self.def_node.is_cyfunction = False # Force genexpr signature self.def_node.entry.signature = TypeSlots.pyfunction_noargs + # setup loop scope + if isinstance(self.loop, Nodes._ForInStatNode): + assert isinstance(self.loop.iterator, ScopedExprNode) + self.loop.iterator.init_scope(None, env) + else: + assert isinstance(self.loop, Nodes.ForFromStatNode) def generate_result_code(self, code): + args_to_call = ([self.closure_result_code()] + + [ cp.result() for cp in self.call_parameters ]) + args_to_call = ", ".join(args_to_call) code.putln( '%s = %s(%s); %s' % ( self.result(), self.def_node.entry.pyfunc_cname, - self.self_result_code(), + args_to_call, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class YieldExprNode(ExprNode): @@ -9778,11 +10283,15 @@ class YieldExprNode(ExprNode): for cname, type, manage_ref in code.funcstate.temps_in_use(): save_cname = code.funcstate.closure_temps.allocate_temp(type) saved.append((cname, save_cname, type)) - if type.is_pyobject: - code.put_xgiveref(cname) + if type.is_cpp_class: + code.globalstate.use_utility_code( + UtilityCode.load_cached("MoveIfSupported", "CppSupport.cpp")) + cname = "__PYX_STD_MOVE_IF_SUPPORTED(%s)" % cname + else: + code.put_xgiveref(cname, type) code.putln('%s->%s = %s;' % (Naming.cur_scope_cname, save_cname, cname)) - code.put_xgiveref(Naming.retval_cname) + code.put_xgiveref(Naming.retval_cname, py_object_type) profile = code.globalstate.directives['profile'] linetrace = code.globalstate.directives['linetrace'] if profile or linetrace: @@ -9810,10 +10319,15 @@ class YieldExprNode(ExprNode): code.put_label(label_name) for cname, save_cname, type in saved: - code.putln('%s = %s->%s;' % (cname, Naming.cur_scope_cname, save_cname)) + save_cname = "%s->%s" % (Naming.cur_scope_cname, save_cname) + if type.is_cpp_class: + save_cname = "__PYX_STD_MOVE_IF_SUPPORTED(%s)" % save_cname + code.putln('%s = %s;' % (cname, save_cname)) if type.is_pyobject: - code.putln('%s->%s = 0;' % (Naming.cur_scope_cname, save_cname)) - code.put_xgotref(cname) + code.putln('%s = 0;' % save_cname) + code.put_xgotref(cname, type) + elif type.is_memoryviewslice: + code.putln('%s.memview = NULL; %s.data = NULL;' % (save_cname, save_cname)) self.generate_sent_value_handling_code(code, Naming.sent_value_cname) if self.result_is_used: self.allocate_temp_result(code) @@ -9841,7 +10355,7 @@ class _YieldDelegationExprNode(YieldExprNode): self.arg.free_temps(code) elif decref_source: code.put_decref_clear(source_cname, py_object_type) - code.put_xgotref(Naming.retval_cname) + code.put_xgotref(Naming.retval_cname, py_object_type) code.putln("if (likely(%s)) {" % Naming.retval_cname) self.generate_yield_code(code) @@ -9857,7 +10371,7 @@ class _YieldDelegationExprNode(YieldExprNode): # YieldExprNode has allocated the result temp for us code.putln("%s = NULL;" % self.result()) code.put_error_if_neg(self.pos, "__Pyx_PyGen_FetchStopIterationValue(&%s)" % self.result()) - code.put_gotref(self.result()) + self.generate_gotref(code) def handle_iteration_exception(self, code): code.putln("PyObject* exc_type = __Pyx_PyErr_Occurred();") @@ -9949,7 +10463,7 @@ class GlobalsExprNode(AtomicExprNode): code.putln('%s = __Pyx_Globals(); %s' % ( self.result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) class LocalsDictItemNode(DictItemNode): @@ -10037,6 +10551,7 @@ class UnopNode(ExprNode): subexprs = ['operand'] infix = True + is_inc_dec_op = False def calculate_constant_result(self): func = compile_time_unary_operators[self.operator] @@ -10139,7 +10654,7 @@ class UnopNode(ExprNode): function, self.operand.py_result(), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) def type_error(self): if not self.operand.type.is_error: @@ -10148,7 +10663,10 @@ class UnopNode(ExprNode): self.type = PyrexTypes.error_type def analyse_cpp_operation(self, env, overload_check=True): - entry = env.lookup_operator(self.operator, [self.operand]) + operand_types = [self.operand.type] + if self.is_inc_dec_op and not self.is_prefix: + operand_types.append(PyrexTypes.c_int_type) + entry = env.lookup_operator_for_types(self.pos, self.operator, operand_types) if overload_check and not entry: self.type_error() return @@ -10157,12 +10675,17 @@ class UnopNode(ExprNode): self.exception_value = entry.type.exception_value if self.exception_check == '+': self.is_temp = True - if self.exception_value is None: + if needs_cpp_exception_conversion(self): env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp")) else: self.exception_check = '' self.exception_value = '' - cpp_type = self.operand.type.find_cpp_operation_type(self.operator) + if self.is_inc_dec_op and not self.is_prefix: + cpp_type = self.operand.type.find_cpp_operation_type( + self.operator, operand_type=PyrexTypes.c_int_type + ) + else: + cpp_type = self.operand.type.find_cpp_operation_type(self.operator) if overload_check and cpp_type is None: error(self.pos, "'%s' operator not defined for %s" % ( self.operator, type)) @@ -10291,7 +10814,10 @@ class DereferenceNode(CUnopNode): def analyse_c_operation(self, env): if self.operand.type.is_ptr: - self.type = self.operand.type.base_type + if env.is_cpp: + self.type = PyrexTypes.CReferenceType(self.operand.type.base_type) + else: + self.type = self.operand.type.base_type else: self.type_error() @@ -10301,6 +10827,17 @@ class DereferenceNode(CUnopNode): class DecrementIncrementNode(CUnopNode): # unary ++/-- operator + is_inc_dec_op = True + + def type_error(self): + if not self.operand.type.is_error: + if self.is_prefix: + error(self.pos, "No match for 'operator%s' (operand type is '%s')" % + (self.operator, self.operand.type)) + else: + error(self.pos, "No 'operator%s(int)' declared for postfix '%s' (operand type is '%s')" % + (self.operator, self.operator, self.operand.type)) + self.type = PyrexTypes.error_type def analyse_c_operation(self, env): if self.operand.type.is_numeric: @@ -10503,8 +11040,10 @@ class TypecastNode(ExprNode): if self.type.is_complex: operand_result = self.operand.result() if self.operand.type.is_complex: - real_part = self.type.real_type.cast_code("__Pyx_CREAL(%s)" % operand_result) - imag_part = self.type.real_type.cast_code("__Pyx_CIMAG(%s)" % operand_result) + real_part = self.type.real_type.cast_code( + self.operand.type.real_code(operand_result)) + imag_part = self.type.real_type.cast_code( + self.operand.type.imag_code(operand_result)) else: real_part = self.type.real_type.cast_code(operand_result) imag_part = "0" @@ -10717,7 +11256,7 @@ class CythonArrayNode(ExprNode): type_info, code.error_goto_if_null(format_temp, self.pos), )) - code.put_gotref(format_temp) + code.put_gotref(format_temp, py_object_type) buildvalue_fmt = " __PYX_BUILD_PY_SSIZE_T " * len(shapes) code.putln('%s = Py_BuildValue((char*) "(" %s ")", %s); %s' % ( @@ -10726,15 +11265,14 @@ class CythonArrayNode(ExprNode): ", ".join(shapes), code.error_goto_if_null(shapes_temp, self.pos), )) - code.put_gotref(shapes_temp) + code.put_gotref(shapes_temp, py_object_type) - tup = (self.result(), shapes_temp, itemsize, format_temp, - self.mode, self.operand.result()) - code.putln('%s = __pyx_array_new(' - '%s, %s, PyBytes_AS_STRING(%s), ' - '(char *) "%s", (char *) %s);' % tup) - code.putln(code.error_goto_if_null(self.result(), self.pos)) - code.put_gotref(self.result()) + code.putln('%s = __pyx_array_new(%s, %s, PyBytes_AS_STRING(%s), (char *) "%s", (char *) %s); %s' % ( + self.result(), + shapes_temp, itemsize, format_temp, self.mode, self.operand.result(), + code.error_goto_if_null(self.result(), self.pos), + )) + self.generate_gotref(code) def dispose(temp): code.put_decref_clear(temp, py_object_type) @@ -10843,7 +11381,11 @@ class SizeofVarNode(SizeofNode): if operand_as_type: self.arg_type = operand_as_type if self.arg_type.is_fused: - self.arg_type = self.arg_type.specialize(env.fused_to_specific) + try: + self.arg_type = self.arg_type.specialize(env.fused_to_specific) + except CannotSpecialize: + error(self.operand.pos, + "Type cannot be specialized since it is not a fused argument to this function") self.__class__ = SizeofTypeNode self.check_type() else: @@ -10864,8 +11406,6 @@ class TypeidNode(ExprNode): # arg_type ExprNode # is_variable boolean - type = PyrexTypes.error_type - subexprs = ['operand'] arg_type = None @@ -10883,19 +11423,25 @@ class TypeidNode(ExprNode): cpp_message = 'typeid operator' def analyse_types(self, env): + if not self.type: + self.type = PyrexTypes.error_type # default value if it isn't analysed successfully self.cpp_check(env) type_info = self.get_type_info_type(env) if not type_info: self.error("The 'libcpp.typeinfo' module must be cimported to use the typeid() operator") return self + if self.operand is None: + return self # already analysed, no need to repeat self.type = type_info - as_type = self.operand.analyse_as_type(env) + as_type = self.operand.analyse_as_specialized_type(env) if as_type: self.arg_type = as_type self.is_type = True + self.operand = None # nothing further uses self.operand - will only cause problems if its used in code generation else: self.arg_type = self.operand.analyse_types(env) self.is_type = False + self.operand = None # nothing further uses self.operand - will only cause problems if its used in code generation if self.arg_type.type.is_pyobject: self.error("Cannot use typeid on a Python object") return self @@ -10937,11 +11483,11 @@ class TypeofNode(ExprNode): literal = None type = py_object_type - subexprs = ['literal'] # 'operand' will be ignored after type analysis! + subexprs = ['literal'] # 'operand' will be ignored after type analysis! def analyse_types(self, env): self.operand = self.operand.analyse_types(env) - value = StringEncoding.EncodedString(str(self.operand.type)) #self.operand.type.typeof_name()) + value = StringEncoding.EncodedString(str(self.operand.type)) #self.operand.type.typeof_name()) literal = StringNode(self.pos, value=value) literal = literal.analyse_types(env) self.literal = literal.coerce_to_pyobject(env) @@ -11100,7 +11646,7 @@ class BinopNode(ExprNode): # Used by NumBinopNodes to break up expressions involving multiple # operators so that exceptions can be handled properly. self.is_temp = 1 - if self.exception_value is None: + if needs_cpp_exception_conversion(self): env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp")) if func_type.is_ptr: func_type = func_type.base_type @@ -11155,6 +11701,8 @@ class BinopNode(ExprNode): self.operand1.is_ephemeral() or self.operand2.is_ephemeral()) def generate_result_code(self, code): + type1 = self.operand1.type + type2 = self.operand2.type if self.type.is_pythran_expr: code.putln("// Pythran binop") code.putln("__Pyx_call_destructor(%s);" % self.result()) @@ -11171,21 +11719,20 @@ class BinopNode(ExprNode): self.operand1.pythran_result(), self.operator, self.operand2.pythran_result())) - elif self.operand1.type.is_pyobject: + elif type1.is_pyobject or type2.is_pyobject: function = self.py_operation_function(code) - if self.operator == '**': - extra_args = ", Py_None" - else: - extra_args = "" + extra_args = ", Py_None" if self.operator == '**' else "" + op1_result = self.operand1.py_result() if type1.is_pyobject else self.operand1.result() + op2_result = self.operand2.py_result() if type2.is_pyobject else self.operand2.result() code.putln( "%s = %s(%s, %s%s); %s" % ( self.result(), function, - self.operand1.py_result(), - self.operand2.py_result(), + op1_result, + op2_result, extra_args, code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) elif self.is_temp: # C++ overloaded operators with exception values are currently all # handled through temporaries. @@ -11309,8 +11856,8 @@ class NumBinopNode(BinopNode): def c_types_okay(self, type1, type2): #print "NumBinopNode.c_types_okay:", type1, type2 ### - return (type1.is_numeric or type1.is_enum) \ - and (type2.is_numeric or type2.is_enum) + return (type1.is_numeric or type1.is_enum) \ + and (type2.is_numeric or type2.is_enum) def generate_evaluation_code(self, code): if self.overflow_check: @@ -11421,7 +11968,7 @@ class AddNode(NumBinopNode): def py_operation_function(self, code): type1, type2 = self.operand1.type, self.operand2.type - + func = None if type1 is unicode_type or type2 is unicode_type: if type1 in (unicode_type, str_type) and type2 in (unicode_type, str_type): is_unicode_concat = True @@ -11433,10 +11980,22 @@ class AddNode(NumBinopNode): is_unicode_concat = False if is_unicode_concat: - if self.operand1.may_be_none() or self.operand2.may_be_none(): - return '__Pyx_PyUnicode_ConcatSafe' - else: - return '__Pyx_PyUnicode_Concat' + if self.inplace or self.operand1.is_temp: + code.globalstate.use_utility_code( + UtilityCode.load_cached("UnicodeConcatInPlace", "ObjectHandling.c")) + func = '__Pyx_PyUnicode_Concat' + elif type1 is str_type and type2 is str_type: + code.globalstate.use_utility_code( + UtilityCode.load_cached("StrConcatInPlace", "ObjectHandling.c")) + func = '__Pyx_PyStr_Concat' + + if func: + # any necessary utility code will be got by "NumberAdd" in generate_evaluation_code + if self.inplace or self.operand1.is_temp: + func += 'InPlace' # upper case to indicate unintuitive macro + if self.operand1.may_be_none() or self.operand2.may_be_none(): + func += 'Safe' + return func return super(AddNode, self).py_operation_function(code) @@ -11456,22 +12015,76 @@ class SubNode(NumBinopNode): class MulNode(NumBinopNode): # '*' operator. + is_sequence_mul = False + + def analyse_types(self, env): + self.operand1 = self.operand1.analyse_types(env) + self.operand2 = self.operand2.analyse_types(env) + self.is_sequence_mul = self.calculate_is_sequence_mul() + + # TODO: we could also optimise the case of "[...] * 2 * n", i.e. with an existing 'mult_factor' + if self.is_sequence_mul: + operand1 = self.operand1 + operand2 = self.operand2 + if operand1.is_sequence_constructor and operand1.mult_factor is None: + return self.analyse_sequence_mul(env, operand1, operand2) + elif operand2.is_sequence_constructor and operand2.mult_factor is None: + return self.analyse_sequence_mul(env, operand2, operand1) + + self.analyse_operation(env) + return self + + @staticmethod + def is_builtin_seqmul_type(type): + return type.is_builtin_type and type in builtin_sequence_types and type is not memoryview_type + + def calculate_is_sequence_mul(self): + type1 = self.operand1.type + type2 = self.operand2.type + if type1 is long_type or type1.is_int: + # normalise to (X * int) + type1, type2 = type2, type1 + if type2 is long_type or type2.is_int: + if type1.is_string or type1.is_ctuple: + return True + if self.is_builtin_seqmul_type(type1): + return True + return False + + def analyse_sequence_mul(self, env, seq, mult): + assert seq.mult_factor is None + seq = seq.coerce_to_pyobject(env) + seq.mult_factor = mult + return seq.analyse_types(env) + + def coerce_operands_to_pyobjects(self, env): + if self.is_sequence_mul: + # Keep operands as they are, but ctuples must become Python tuples to multiply them. + if self.operand1.type.is_ctuple: + self.operand1 = self.operand1.coerce_to_pyobject(env) + elif self.operand2.type.is_ctuple: + self.operand2 = self.operand2.coerce_to_pyobject(env) + return + super(MulNode, self).coerce_operands_to_pyobjects(env) def is_py_operation_types(self, type1, type2): - if ((type1.is_string and type2.is_int) or - (type2.is_string and type1.is_int)): - return 1 - else: - return NumBinopNode.is_py_operation_types(self, type1, type2) + return self.is_sequence_mul or super(MulNode, self).is_py_operation_types(type1, type2) + + def py_operation_function(self, code): + if self.is_sequence_mul: + code.globalstate.use_utility_code( + UtilityCode.load_cached("PySequenceMultiply", "ObjectHandling.c")) + return "__Pyx_PySequence_Multiply" if self.operand1.type.is_pyobject else "__Pyx_PySequence_Multiply_Left" + return super(MulNode, self).py_operation_function(code) def infer_builtin_types_operation(self, type1, type2): - # let's assume that whatever builtin type you multiply a string with - # will either return a string of the same type or fail with an exception - string_types = (bytes_type, bytearray_type, str_type, basestring_type, unicode_type) - if type1 in string_types and type2.is_builtin_type: - return type1 - if type2 in string_types and type1.is_builtin_type: - return type2 + # let's assume that whatever builtin type you multiply a builtin sequence type with + # will either return a sequence of the same type or fail with an exception + if type1.is_builtin_type and type2.is_builtin_type: + if self.is_builtin_seqmul_type(type1): + return type1 + if self.is_builtin_seqmul_type(type2): + return type2 # multiplication of containers/numbers with an integer value # always (?) returns the same type if type1.is_int: @@ -11608,7 +12221,7 @@ class DivNode(NumBinopNode): minus1_check = '(!(((%s)-1) > 0)) && unlikely(%s == (%s)-1)' % ( type_of_op2, self.operand2.result(), type_of_op2) code.putln("else if (sizeof(%s) == sizeof(long) && %s " - " && unlikely(UNARY_NEG_WOULD_OVERFLOW(%s))) {" % ( + " && unlikely(__Pyx_UNARY_NEG_WOULD_OVERFLOW(%s))) {" % ( self.type.empty_declaration_code(), minus1_check, self.operand1.result())) @@ -11676,10 +12289,10 @@ _find_formatting_types = re.compile( br")").findall # These format conversion types can never trigger a Unicode string conversion in Py2. -_safe_bytes_formats = set([ +_safe_bytes_formats = frozenset({ # Excludes 's' and 'r', which can generate non-bytes strings. b'd', b'i', b'o', b'u', b'x', b'X', b'e', b'E', b'f', b'F', b'g', b'G', b'c', b'b', b'a', -]) +}) class ModNode(DivNode): @@ -11779,12 +12392,21 @@ class ModNode(DivNode): class PowNode(NumBinopNode): # '**' operator. + is_cpow = None + type_was_inferred = False # was the result type affected by cpow==False? + # Intended to allow it to be changed if the node is coerced. + + def _check_cpow(self, env): + if self.is_cpow is not None: + return # already set + self.is_cpow = env.directives['cpow'] + + def infer_type(self, env): + self._check_cpow(env) + return super(PowNode, self).infer_type(env) + def analyse_types(self, env): - if not env.directives['cpow']: - # Note - the check here won't catch cpow directives that don't use '**' - # but that's probably OK for a placeholder forward compatibility directive - error(self.pos, "The 'cpow' directive is provided for forward compatibility " - "and must be True") + self._check_cpow(env) return super(PowNode, self).analyse_types(env) def analyse_c_operation(self, env): @@ -11810,6 +12432,48 @@ class PowNode(NumBinopNode): error(self.pos, "got unexpected types for C power operator: %s, %s" % (self.operand1.type, self.operand2.type)) + def compute_c_result_type(self, type1, type2): + from numbers import Real + c_result_type = None + op1_is_definitely_positive = ( + self.operand1.has_constant_result() + and self.operand1.constant_result >= 0 + ) or ( + type1.is_int and type1.signed == 0 # definitely unsigned + ) + type2_is_int = type2.is_int or ( + self.operand2.has_constant_result() and + isinstance(self.operand2.constant_result, Real) and + int(self.operand2.constant_result) == self.operand2.constant_result + ) + needs_widening = False + if self.is_cpow: + c_result_type = super(PowNode, self).compute_c_result_type(type1, type2) + if not self.operand2.has_constant_result(): + needs_widening = ( + isinstance(self.operand2.constant_result, _py_int_types) and self.operand2.constant_result < 0 + ) + elif op1_is_definitely_positive or type2_is_int: # cpow==False + # if type2 is an integer then we can't end up going from real to complex + c_result_type = super(PowNode, self).compute_c_result_type(type1, type2) + if not self.operand2.has_constant_result(): + needs_widening = type2.is_int and type2.signed + if needs_widening: + self.type_was_inferred = True + else: + needs_widening = ( + isinstance(self.operand2.constant_result, _py_int_types) and self.operand2.constant_result < 0 + ) + elif self.c_types_okay(type1, type2): + # Allowable result types are double or complex double. + # Return the special "soft complex" type to store it as a + # complex number but with specialized coercions to Python + c_result_type = PyrexTypes.soft_complex_type + self.type_was_inferred = True + if needs_widening: + c_result_type = PyrexTypes.widest_numeric_type(c_result_type, PyrexTypes.c_double_type) + return c_result_type + def calculate_result_code(self): # Work around MSVC overloading ambiguity. def typecast(operand): @@ -11834,6 +12498,50 @@ class PowNode(NumBinopNode): return '__Pyx_PyNumber_PowerOf2' return super(PowNode, self).py_operation_function(code) + def coerce_to(self, dst_type, env): + if dst_type == self.type: + return self + if (self.is_cpow is None and self.type_was_inferred and + (dst_type.is_float or dst_type.is_int)): + # if we're trying to coerce this directly to a C float or int + # then fall back to the cpow == True behaviour since this is + # almost certainly the user intent. + # However, ensure that the operand types are suitable C types + if self.type is PyrexTypes.soft_complex_type: + def check_types(operand, recurse=True): + if operand.type.is_float or operand.type.is_int: + return True, operand + if recurse and isinstance(operand, CoerceToComplexNode): + return check_types(operand.arg, recurse=False), operand.arg + return False, None + msg_detail = "a non-complex C numeric type" + elif dst_type.is_int: + def check_types(operand): + if operand.type.is_int: + return True, operand + else: + # int, int doesn't seem to involve coercion nodes + return False, None + msg_detail = "an integer C numeric type" + else: + def check_types(operand): + return False, None + check_op1, op1 = check_types(self.operand1) + check_op2, op2 = check_types(self.operand2) + if check_op1 and check_op2: + warning(self.pos, "Treating '**' as if 'cython.cpow(True)' since it " + "is directly assigned to a %s. " + "This is likely to be fragile and we recommend setting " + "'cython.cpow' explicitly." % msg_detail) + self.is_cpow = True + self.operand1 = op1 + self.operand2 = op2 + result = self.analyse_types(env) + if result.type != dst_type: + result = result.coerce_to(dst_type, env) + return result + return super(PowNode, self).coerce_to(dst_type, env) + class BoolBinopNode(ExprNode): """ @@ -12080,6 +12788,9 @@ class BoolBinopResultNode(ExprNode): code.putln("}") self.arg.free_temps(code) + def analyse_types(self, env): + return self + class CondExprNode(ExprNode): # Short-circuiting conditional expression. @@ -12217,6 +12928,7 @@ class CmpNode(object): special_bool_cmp_function = None special_bool_cmp_utility_code = None + special_bool_extra_args = [] def infer_type(self, env): # TODO: Actually implement this (after merging with -unstable). @@ -12433,6 +13145,21 @@ class CmpNode(object): self.special_bool_cmp_utility_code = UtilityCode.load_cached("StrEquals", "StringTools.c") self.special_bool_cmp_function = "__Pyx_PyString_Equals" return True + elif result_is_bool: + from .Optimize import optimise_numeric_binop + result = optimise_numeric_binop( + "Eq" if self.operator == "==" else "Ne", + self, + PyrexTypes.c_bint_type, + operand1, + self.operand2 + ) + if result: + (self.special_bool_cmp_function, + self.special_bool_cmp_utility_code, + self.special_bool_extra_args, + _) = result + return True elif self.operator in ('in', 'not_in'): if self.operand2.type is Builtin.dict_type: self.operand2 = self.operand2.as_none_safe_node("'NoneType' object is not iterable") @@ -12458,7 +13185,7 @@ class CmpNode(object): return False def generate_operation_code(self, code, result_code, - operand1, op , operand2): + operand1, op, operand2): if self.type.is_pyobject: error_clause = code.error_goto_if_null got_ref = "__Pyx_XGOTREF(%s); " % result_code @@ -12482,6 +13209,9 @@ class CmpNode(object): result2 = operand2.py_result() else: result2 = operand2.result() + special_bool_extra_args_result = ", ".join([ + extra_arg.result() for extra_arg in self.special_bool_extra_args + ]) if self.special_bool_cmp_utility_code: code.globalstate.use_utility_code(self.special_bool_cmp_utility_code) code.putln( @@ -12489,14 +13219,17 @@ class CmpNode(object): result_code, coerce_result, self.special_bool_cmp_function, - result1, result2, richcmp_constants[op], + result1, result2, + special_bool_extra_args_result if self.special_bool_extra_args else richcmp_constants[op], got_ref, error_clause(result_code, self.pos))) elif operand1.type.is_pyobject and op not in ('is', 'is_not'): assert op not in ('in', 'not_in'), op - code.putln("%s = PyObject_RichCompare(%s, %s, %s); %s%s" % ( + assert self.type.is_pyobject or self.type is PyrexTypes.c_bint_type + code.putln("%s = PyObject_RichCompare%s(%s, %s, %s); %s%s" % ( result_code, + "" if self.type.is_pyobject else "Bool", operand1.py_result(), operand2.py_result(), richcmp_constants[op], @@ -12563,7 +13296,8 @@ class PrimaryCmpNode(ExprNode, CmpNode): # Instead, we override all the framework methods # which use it. - child_attrs = ['operand1', 'operand2', 'coerced_operand2', 'cascade'] + child_attrs = ['operand1', 'operand2', 'coerced_operand2', 'cascade', + 'special_bool_extra_args'] cascade = None coerced_operand2 = None @@ -12591,6 +13325,12 @@ class PrimaryCmpNode(ExprNode, CmpNode): operand1 = self.operand1.compile_time_value(denv) return self.cascaded_compile_time_value(operand1, denv) + def unify_cascade_type(self): + cdr = self.cascade + while cdr: + cdr.type = self.type + cdr = cdr.cascade + def analyse_types(self, env): self.operand1 = self.operand1.analyse_types(env) self.operand2 = self.operand2.analyse_types(env) @@ -12640,16 +13380,16 @@ class PrimaryCmpNode(ExprNode, CmpNode): elif self.find_special_bool_compare_function(env, self.operand1): if not self.operand1.type.is_pyobject: self.operand1 = self.operand1.coerce_to_pyobject(env) - common_type = None # if coercion needed, the method call above has already done it - self.is_pycmp = False # result is bint + common_type = None # if coercion needed, the method call above has already done it + self.is_pycmp = False # result is bint else: common_type = py_object_type self.is_pycmp = True elif self.find_special_bool_compare_function(env, self.operand1): if not self.operand1.type.is_pyobject: self.operand1 = self.operand1.coerce_to_pyobject(env) - common_type = None # if coercion needed, the method call above has already done it - self.is_pycmp = False # result is bint + common_type = None # if coercion needed, the method call above has already done it + self.is_pycmp = False # result is bint else: common_type = self.find_common_type(env, self.operator, self.operand1) self.is_pycmp = common_type.is_pyobject @@ -12669,10 +13409,7 @@ class PrimaryCmpNode(ExprNode, CmpNode): self.type = PyrexTypes.py_object_type else: self.type = PyrexTypes.c_bint_type - cdr = self.cascade - while cdr: - cdr.type = self.type - cdr = cdr.cascade + self.unify_cascade_type() if self.is_pycmp or self.cascade or self.special_bool_cmp_function: # 1) owned reference, 2) reused value, 3) potential function error return value self.is_temp = 1 @@ -12696,7 +13433,7 @@ class PrimaryCmpNode(ExprNode, CmpNode): self.exception_value = func_type.exception_value if self.exception_check == '+': self.is_temp = True - if self.exception_value is None: + if needs_cpp_exception_conversion(self): env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp")) if len(func_type.args) == 1: self.operand2 = self.operand2.coerce_to(func_type.args[0].type, env) @@ -12731,6 +13468,7 @@ class PrimaryCmpNode(ExprNode, CmpNode): self.operand2, env, result_is_bool=True) if operand2 is not self.operand2: self.coerced_operand2 = operand2 + self.unify_cascade_type() return self # TODO: check if we can optimise parts of the cascade here return ExprNode.coerce_to_boolean(self, env) @@ -12791,6 +13529,8 @@ class PrimaryCmpNode(ExprNode, CmpNode): def generate_evaluation_code(self, code): self.operand1.generate_evaluation_code(code) self.operand2.generate_evaluation_code(code) + for extra_arg in self.special_bool_extra_args: + extra_arg.generate_evaluation_code(code) if self.is_temp: self.allocate_temp_result(code) self.generate_operation_code(code, self.result(), @@ -12833,11 +13573,12 @@ class CascadedCmpNode(Node, CmpNode): # operand2 ExprNode # cascade CascadedCmpNode - child_attrs = ['operand2', 'coerced_operand2', 'cascade'] + child_attrs = ['operand2', 'coerced_operand2', 'cascade', + 'special_bool_extra_args'] cascade = None coerced_operand2 = None - constant_result = constant_value_not_set # FIXME: where to calculate this? + constant_result = constant_value_not_set # FIXME: where to calculate this? def infer_type(self, env): # TODO: Actually implement this (after merging with -unstable). @@ -12897,6 +13638,8 @@ class CascadedCmpNode(Node, CmpNode): if needs_evaluation: operand1.generate_evaluation_code(code) self.operand2.generate_evaluation_code(code) + for extra_arg in self.special_bool_extra_args: + extra_arg.generate_evaluation_code(code) self.generate_operation_code(code, result, operand1, self.operator, self.operand2) if self.cascade: @@ -12984,6 +13727,9 @@ class CoercionNode(ExprNode): code.annotate((file, line, col-1), AnnotationItem( style='coerce', tag='coerce', text='[%s] to [%s]' % (self.arg.type, self.type))) + def analyse_types(self, env): + return self + class CoerceToMemViewSliceNode(CoercionNode): """ @@ -13035,9 +13781,10 @@ class PyTypeTestNode(CoercionNode): exact_builtin_type = True def __init__(self, arg, dst_type, env, notnone=False): - # The arg is know to be a Python object, and + # The arg is known to be a Python object, and # the dst_type is known to be an extension type. - assert dst_type.is_extension_type or dst_type.is_builtin_type, "PyTypeTest on non extension type" + assert dst_type.is_extension_type or dst_type.is_builtin_type, \ + "PyTypeTest for %s against non extension type %s" % (arg.type, dst_type) CoercionNode.__init__(self, arg) self.type = dst_type self.result_ctype = arg.ctype() @@ -13088,6 +13835,8 @@ class PyTypeTestNode(CoercionNode): type_test = self.type.type_test_code( self.arg.py_result(), self.notnone, exact=self.exact_builtin_type) + code.globalstate.use_utility_code(UtilityCode.load_cached( + "RaiseUnexpectedTypeError", "ObjectHandling.c")) else: type_test = self.type.type_test_code( self.arg.py_result(), self.notnone) @@ -13131,7 +13880,7 @@ class NoneCheckNode(CoercionNode): self.exception_message = exception_message self.exception_format_args = tuple(exception_format_args or ()) - nogil_check = None # this node only guards an operation that would fail already + nogil_check = None # this node only guards an operation that would fail already def analyse_types(self, env): return self @@ -13254,7 +14003,7 @@ class CoerceToPyTypeNode(CoercionNode): def coerce_to_boolean(self, env): arg_type = self.arg.type if (arg_type == PyrexTypes.c_bint_type or - (arg_type.is_pyobject and arg_type.name == 'bool')): + (arg_type.is_pyobject and arg_type.name == 'bool')): return self.arg.coerce_to_temp(env) else: return CoerceToBooleanNode(self, env) @@ -13278,7 +14027,7 @@ class CoerceToPyTypeNode(CoercionNode): self.target_type), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class CoerceIntToBytesNode(CoerceToPyTypeNode): @@ -13318,13 +14067,16 @@ class CoerceIntToBytesNode(CoerceToPyTypeNode): code.error_goto_if_null(self.result(), self.pos))) if temp is not None: code.funcstate.release_temp(temp) - code.put_gotref(self.py_result()) + self.generate_gotref(code) class CoerceFromPyTypeNode(CoercionNode): # This node is used to convert a Python object # to a C data type. + # Allow 'None' to map to a difference C value independent of the coercion, e.g. to 'NULL' or '0'. + special_none_cvalue = None + def __init__(self, result_type, arg, env): CoercionNode.__init__(self, arg) self.type = result_type @@ -13354,9 +14106,12 @@ class CoerceFromPyTypeNode(CoercionNode): NoneCheckNode.generate_if_needed(self.arg, code, "expected bytes, NoneType found") code.putln(self.type.from_py_call_code( - self.arg.py_result(), self.result(), self.pos, code, from_py_function=from_py_function)) + self.arg.py_result(), self.result(), self.pos, code, + from_py_function=from_py_function, + special_none_cvalue=self.special_none_cvalue, + )) if self.type.is_pyobject: - code.put_gotref(self.py_result()) + self.generate_gotref(code) def nogil_check(self, env): error(self.pos, "Coercion from Python not allowed without the GIL") @@ -13413,6 +14168,9 @@ class CoerceToBooleanNode(CoercionNode): self.arg.py_result(), code.error_goto_if_neg(self.result(), self.pos))) + def analyse_types(self, env): + return self + class CoerceToComplexNode(CoercionNode): @@ -13425,8 +14183,8 @@ class CoerceToComplexNode(CoercionNode): def calculate_result_code(self): if self.arg.type.is_complex: - real_part = "__Pyx_CREAL(%s)" % self.arg.result() - imag_part = "__Pyx_CIMAG(%s)" % self.arg.result() + real_part = self.arg.type.real_code(self.arg.result()) + imag_part = self.arg.type.imag_code(self.arg.result()) else: real_part = self.arg.result() imag_part = "0" @@ -13438,6 +14196,33 @@ class CoerceToComplexNode(CoercionNode): def generate_result_code(self, code): pass + def analyse_types(self, env): + return self + + +def coerce_from_soft_complex(arg, dst_type, env): + from .UtilNodes import HasGilNode + cfunc_type = PyrexTypes.CFuncType( + PyrexTypes.c_double_type, + [ PyrexTypes.CFuncTypeArg("value", PyrexTypes.soft_complex_type, None), + PyrexTypes.CFuncTypeArg("have_gil", PyrexTypes.c_bint_type, None) ], + exception_value="-1", + exception_check=True, + nogil=True # We can acquire the GIL internally on failure + ) + call = PythonCapiCallNode( + arg.pos, + "__Pyx_SoftComplexToDouble", + cfunc_type, + utility_code = UtilityCode.load_cached("SoftComplexToDouble", "Complex.c"), + args = [arg, HasGilNode(arg.pos)], + ) + call = call.analyse_types(env) + if call.type != dst_type: + call = call.coerce_to(dst_type, env) + return call + + class CoerceToTempNode(CoercionNode): # This node is used to force the result of another node # to be stored in a temporary. It is only used if the @@ -13457,6 +14242,9 @@ class CoerceToTempNode(CoercionNode): # The arg is always already analysed return self + def may_be_none(self): + return self.arg.may_be_none() + def coerce_to_boolean(self, env): self.arg = self.arg.coerce_to_boolean(env) if self.arg.is_simple(): @@ -13471,11 +14259,12 @@ class CoerceToTempNode(CoercionNode): code.putln("%s = %s;" % ( self.result(), self.arg.result_as(self.ctype()))) if self.use_managed_ref: - if self.type.is_pyobject: + if not self.type.is_memoryviewslice: code.put_incref(self.result(), self.ctype()) - elif self.type.is_memoryviewslice: - code.put_incref_memoryviewslice(self.result(), - not self.in_nogil_context) + else: + code.put_incref_memoryviewslice(self.result(), self.type, + have_gil=not self.in_nogil_context) + class ProxyNode(CoercionNode): """ @@ -13491,22 +14280,24 @@ class ProxyNode(CoercionNode): def __init__(self, arg): super(ProxyNode, self).__init__(arg) self.constant_result = arg.constant_result - self._proxy_type() + self.update_type_and_entry() def analyse_types(self, env): self.arg = self.arg.analyse_expressions(env) - self._proxy_type() + self.update_type_and_entry() return self def infer_type(self, env): return self.arg.infer_type(env) - def _proxy_type(self): - if hasattr(self.arg, 'type'): - self.type = self.arg.type + def update_type_and_entry(self): + type = getattr(self.arg, 'type', None) + if type: + self.type = type self.result_ctype = self.arg.result_ctype - if hasattr(self.arg, 'entry'): - self.entry = self.arg.entry + arg_entry = getattr(self.arg, 'entry', None) + if arg_entry: + self.entry = arg_entry def generate_result_code(self, code): self.arg.generate_result_code(code) @@ -13537,17 +14328,19 @@ class CloneNode(CoercionNode): # disposal code for it. The original owner of the argument # node is responsible for doing those things. - subexprs = [] # Arg is not considered a subexpr + subexprs = [] # Arg is not considered a subexpr nogil_check = None def __init__(self, arg): CoercionNode.__init__(self, arg) self.constant_result = arg.constant_result - if hasattr(arg, 'type'): - self.type = arg.type + type = getattr(arg, 'type', None) + if type: + self.type = type self.result_ctype = arg.result_ctype - if hasattr(arg, 'entry'): - self.entry = arg.entry + arg_entry = getattr(arg, 'entry', None) + if arg_entry: + self.entry = arg_entry def result(self): return self.arg.result() @@ -13565,8 +14358,9 @@ class CloneNode(CoercionNode): self.type = self.arg.type self.result_ctype = self.arg.result_ctype self.is_temp = 1 - if hasattr(self.arg, 'entry'): - self.entry = self.arg.entry + arg_entry = getattr(self.arg, 'entry', None) + if arg_entry: + self.entry = arg_entry return self def coerce_to(self, dest_type, env): @@ -13575,7 +14369,7 @@ class CloneNode(CoercionNode): return super(CloneNode, self).coerce_to(dest_type, env) def is_simple(self): - return True # result is always in a temp (or a name) + return True # result is always in a temp (or a name) def generate_evaluation_code(self, code): pass @@ -13586,10 +14380,38 @@ class CloneNode(CoercionNode): def generate_disposal_code(self, code): pass + def generate_post_assignment_code(self, code): + # if we're assigning from a CloneNode then it's "giveref"ed away, so it does + # need a matching incref (ideally this should happen before the assignment though) + if self.is_temp: # should usually be true + code.put_incref(self.result(), self.ctype()) + def free_temps(self, code): pass +class CppOptionalTempCoercion(CoercionNode): + """ + Used only in CoerceCppTemps - handles cases the temp is actually a OptionalCppClassType (and thus needs dereferencing when on the rhs) + """ + is_temp = False + + @property + def type(self): + return self.arg.type + + def calculate_result_code(self): + return "(*%s)" % self.arg.result() + + def generate_result_code(self, code): + pass + + def _make_move_result_rhs(self, result, optional=False): + # this wouldn't normally get moved (because it isn't a temp), but force it to be because it + # is a thin wrapper around a temp + return super(CppOptionalTempCoercion, self)._make_move_result_rhs(result, optional=False) + + class CMethodSelfCloneNode(CloneNode): # Special CloneNode for the self argument of builtin C methods # that accepts subtypes of the builtin type. This is safe only @@ -13641,77 +14463,215 @@ class DocstringRefNode(ExprNode): self.result(), self.body.result(), code.intern_identifier(StringEncoding.EncodedString("__doc__")), code.error_goto_if_null(self.result(), self.pos))) - code.put_gotref(self.result()) + self.generate_gotref(code) + +class AnnotationNode(ExprNode): + # Deals with the two possible uses of an annotation. + # 1. The post PEP-563 use where an annotation is stored + # as a string + # 2. The Cython use where the annotation can indicate an + # object type + # + # Doesn't handle the pre PEP-563 version where the + # annotation is evaluated into a Python Object. + subexprs = [] + # 'untyped' is set for fused specializations: + # Once a fused function has been created we don't want + # annotations to override an already set type. + untyped = False -#------------------------------------------------------------------------------------ -# -# Runtime support code -# -#------------------------------------------------------------------------------------ - -pyerr_occurred_withgil_utility_code= UtilityCode( -proto = """ -static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void); /* proto */ -""", -impl = """ -static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void) { - int err; - #ifdef WITH_THREAD - PyGILState_STATE _save = PyGILState_Ensure(); - #endif - err = !!PyErr_Occurred(); - #ifdef WITH_THREAD - PyGILState_Release(_save); - #endif - return err; -} -""" -) + def __init__(self, pos, expr, string=None): + """string is expected to already be a StringNode or None""" + ExprNode.__init__(self, pos) + if string is None: + # import doesn't work at top of file? + from .AutoDocTransforms import AnnotationWriter + string = StringEncoding.EncodedString( + AnnotationWriter(description="annotation").write(expr)) + string = StringNode(pos, unicode_value=string, value=string.as_utf8_string()) + self.string = string + self.expr = expr -#------------------------------------------------------------------------------------ + def analyse_types(self, env): + return self # nothing needs doing -raise_unbound_local_error_utility_code = UtilityCode( -proto = """ -static CYTHON_INLINE void __Pyx_RaiseUnboundLocalError(const char *varname); -""", -impl = """ -static CYTHON_INLINE void __Pyx_RaiseUnboundLocalError(const char *varname) { - PyErr_Format(PyExc_UnboundLocalError, "local variable '%s' referenced before assignment", varname); -} -""") - -raise_closure_name_error_utility_code = UtilityCode( -proto = """ -static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname); -""", -impl = """ -static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname) { - PyErr_Format(PyExc_NameError, "free variable '%s' referenced before assignment in enclosing scope", varname); -} -""") - -# Don't inline the function, it should really never be called in production -raise_unbound_memoryview_utility_code_nogil = UtilityCode( -proto = """ -static void __Pyx_RaiseUnboundMemoryviewSliceNogil(const char *varname); -""", -impl = """ -static void __Pyx_RaiseUnboundMemoryviewSliceNogil(const char *varname) { - #ifdef WITH_THREAD - PyGILState_STATE gilstate = PyGILState_Ensure(); - #endif - __Pyx_RaiseUnboundLocalError(varname); - #ifdef WITH_THREAD - PyGILState_Release(gilstate); - #endif -} -""", -requires = [raise_unbound_local_error_utility_code]) + def analyse_as_type(self, env): + # for compatibility when used as a return_type_node, have this interface too + return self.analyse_type_annotation(env)[1] + + def _warn_on_unknown_annotation(self, env, annotation): + """Method checks for cases when user should be warned that annotation contains unknown types.""" + if annotation.is_name: + # Validate annotation in form `var: type` + if not env.lookup(annotation.name): + warning(annotation.pos, + "Unknown type declaration '%s' in annotation, ignoring" % self.string.value, level=1) + elif annotation.is_attribute and annotation.obj.is_name: + # Validate annotation in form `var: module.type` + if not env.lookup(annotation.obj.name): + # `module` is undeclared + warning(annotation.pos, + "Unknown type declaration '%s' in annotation, ignoring" % self.string.value, level=1) + elif annotation.obj.is_cython_module: + # `module` is cython + module_scope = annotation.obj.analyse_as_module(env) + if module_scope and not module_scope.lookup_type(annotation.attribute): + error(annotation.pos, + "Unknown type declaration '%s' in annotation" % self.string.value) + else: + module_scope = annotation.obj.analyse_as_module(env) + if module_scope and module_scope.pxd_file_loaded: + warning(annotation.pos, + "Unknown type declaration '%s' in annotation, ignoring" % self.string.value, level=1) + else: + warning(annotation.pos, "Unknown type declaration in annotation, ignoring") + + def analyse_type_annotation(self, env, assigned_value=None): + if self.untyped: + # Already applied as a fused type, not re-evaluating it here. + return [], None + annotation = self.expr + explicit_pytype = explicit_ctype = False + if annotation.is_dict_literal: + warning(annotation.pos, + "Dicts should no longer be used as type annotations. Use 'cython.int' etc. directly.", level=1) + for name, value in annotation.key_value_pairs: + if not name.is_string_literal: + continue + if name.value in ('type', b'type'): + explicit_pytype = True + if not explicit_ctype: + annotation = value + elif name.value in ('ctype', b'ctype'): + explicit_ctype = True + annotation = value + if explicit_pytype and explicit_ctype: + warning(annotation.pos, "Duplicate type declarations found in signature annotation", level=1) + elif isinstance(annotation, TupleNode): + warning(annotation.pos, + "Tuples cannot be declared as simple tuples of types. Use 'tuple[type1, type2, ...]'.", level=1) + return [], None + + with env.new_c_type_context(in_c_type_context=explicit_ctype): + arg_type = annotation.analyse_as_type(env) + + if arg_type is None: + self._warn_on_unknown_annotation(env, annotation) + return [], arg_type + + if annotation.is_string_literal: + warning(annotation.pos, + "Strings should no longer be used for type declarations. Use 'cython.int' etc. directly.", + level=1) + if explicit_pytype and not explicit_ctype and not (arg_type.is_pyobject or arg_type.equivalent_type): + warning(annotation.pos, + "Python type declaration in signature annotation does not refer to a Python type") + if arg_type.is_complex: + # creating utility code needs to be special-cased for complex types + arg_type.create_declaration_utility_code(env) + + # Check for declaration modifiers, e.g. "typing.Optional[...]" or "dataclasses.InitVar[...]" + modifiers = annotation.analyse_pytyping_modifiers(env) if annotation.is_subscript else [] + + return modifiers, arg_type + + +class AssignmentExpressionNode(ExprNode): + """ + Also known as a named expression or the walrus operator + + Arguments + lhs - NameNode - not stored directly as an attribute of the node + rhs - ExprNode -#------------------------------------------------------------------------------------ + Attributes + rhs - ExprNode + assignment - SingleAssignmentNode + """ + # subexprs and child_attrs are intentionally different here, because the assignment is not an expression + subexprs = ["rhs"] + child_attrs = ["rhs", "assignment"] # This order is important for control-flow (i.e. xdecref) to be right + + is_temp = False + assignment = None + clone_node = None + + def __init__(self, pos, lhs, rhs, **kwds): + super(AssignmentExpressionNode, self).__init__(pos, **kwds) + self.rhs = ProxyNode(rhs) + assign_expr_rhs = CloneNode(self.rhs) + self.assignment = SingleAssignmentNode( + pos, lhs=lhs, rhs=assign_expr_rhs, is_assignment_expression=True) + + @property + def type(self): + return self.rhs.type + + @property + def target_name(self): + return self.assignment.lhs.name -raise_too_many_values_to_unpack = UtilityCode.load_cached("RaiseTooManyValuesToUnpack", "ObjectHandling.c") -raise_need_more_values_to_unpack = UtilityCode.load_cached("RaiseNeedMoreValuesToUnpack", "ObjectHandling.c") -tuple_unpacking_error_code = UtilityCode.load_cached("UnpackTupleError", "ObjectHandling.c") + def infer_type(self, env): + return self.rhs.infer_type(env) + + def analyse_declarations(self, env): + self.assignment.analyse_declarations(env) + + def analyse_types(self, env): + # we're trying to generate code that looks roughly like: + # __pyx_t_1 = rhs + # lhs = __pyx_t_1 + # __pyx_t_1 + # (plus any reference counting that's needed) + + self.rhs = self.rhs.analyse_types(env) + if not self.rhs.arg.is_temp: + if not self.rhs.arg.is_literal: + # for anything but the simplest cases (where it can be used directly) + # we convert rhs to a temp, because CloneNode requires arg to be a temp + self.rhs.arg = self.rhs.arg.coerce_to_temp(env) + else: + # For literals we can optimize by just using the literal twice + # + # We aren't including `self.rhs.is_name` in this optimization + # because that goes wrong for assignment expressions run in + # parallel. e.g. `(a := b) + (b := a + c)`) + # This is a special case of https://github.com/cython/cython/issues/4146 + # TODO - once that's fixed general revisit this code and possibly + # use coerce_to_simple + self.assignment.rhs = copy.copy(self.rhs) + + # TODO - there's a missed optimization in the code generation stage + # for self.rhs.arg.is_temp: an incref/decref pair can be removed + # (but needs a general mechanism to do that) + self.assignment = self.assignment.analyse_types(env) + return self + + def coerce_to(self, dst_type, env): + if dst_type == self.assignment.rhs.type: + # in this quite common case (for example, when both lhs, and self are being coerced to Python) + # we can optimize the coercion out by sharing it between + # this and the assignment + old_rhs_arg = self.rhs.arg + if isinstance(old_rhs_arg, CoerceToTempNode): + old_rhs_arg = old_rhs_arg.arg + rhs_arg = old_rhs_arg.coerce_to(dst_type, env) + if rhs_arg is not old_rhs_arg: + self.rhs.arg = rhs_arg + self.rhs.update_type_and_entry() + # clean up the old coercion node that the assignment has likely generated + if (isinstance(self.assignment.rhs, CoercionNode) + and not isinstance(self.assignment.rhs, CloneNode)): + self.assignment.rhs = self.assignment.rhs.arg + self.assignment.rhs.type = self.assignment.rhs.arg.type + return self + return super(AssignmentExpressionNode, self).coerce_to(dst_type, env) + + def calculate_result_code(self): + return self.rhs.result() + + def generate_result_code(self, code): + # we have to do this manually because it isn't a subexpression + self.assignment.generate_execution_code(code) diff --git a/Cython/Compiler/FlowControl.pxd b/Cython/Compiler/FlowControl.pxd index c87370b81..5338d4fe4 100644 --- a/Cython/Compiler/FlowControl.pxd +++ b/Cython/Compiler/FlowControl.pxd @@ -1,30 +1,30 @@ -from __future__ import absolute_import +# cython: language_level=3 cimport cython from .Visitor cimport CythonTransform, TreeVisitor cdef class ControlBlock: - cdef public set children - cdef public set parents - cdef public set positions - cdef public list stats - cdef public dict gen - cdef public set bounded - - # Big integer bitsets - cdef public object i_input - cdef public object i_output - cdef public object i_gen - cdef public object i_kill - cdef public object i_state - - cpdef bint empty(self) - cpdef detach(self) - cpdef add_child(self, block) + cdef public set children + cdef public set parents + cdef public set positions + cdef public list stats + cdef public dict gen + cdef public set bounded + + # Big integer bitsets + cdef public object i_input + cdef public object i_output + cdef public object i_gen + cdef public object i_kill + cdef public object i_state + + cpdef bint empty(self) + cpdef detach(self) + cpdef add_child(self, block) cdef class ExitBlock(ControlBlock): - cpdef bint empty(self) + cpdef bint empty(self) cdef class NameAssignment: cdef public bint is_arg @@ -36,6 +36,7 @@ cdef class NameAssignment: cdef public set refs cdef public object bit cdef public object inferred_type + cdef public object rhs_scope cdef class AssignmentList: cdef public object bit @@ -47,51 +48,50 @@ cdef class AssignmentCollector(TreeVisitor): @cython.final cdef class ControlFlow: - cdef public set blocks - cdef public set entries - cdef public list loops - cdef public list exceptions + cdef public set blocks + cdef public set entries + cdef public list loops + cdef public list exceptions + + cdef public ControlBlock entry_point + cdef public ExitBlock exit_point + cdef public ControlBlock block - cdef public ControlBlock entry_point - cdef public ExitBlock exit_point - cdef public ControlBlock block + cdef public dict assmts - cdef public dict assmts + cdef public Py_ssize_t in_try_block - cpdef newblock(self, ControlBlock parent=*) - cpdef nextblock(self, ControlBlock parent=*) - cpdef bint is_tracked(self, entry) - cpdef bint is_statically_assigned(self, entry) - cpdef mark_position(self, node) - cpdef mark_assignment(self, lhs, rhs, entry) - cpdef mark_argument(self, lhs, rhs, entry) - cpdef mark_deletion(self, node, entry) - cpdef mark_reference(self, node, entry) + cpdef newblock(self, ControlBlock parent=*) + cpdef nextblock(self, ControlBlock parent=*) + cpdef bint is_tracked(self, entry) + cpdef bint is_statically_assigned(self, entry) + cpdef mark_position(self, node) + cpdef mark_assignment(self, lhs, rhs, entry, rhs_scope=*) + cpdef mark_argument(self, lhs, rhs, entry) + cpdef mark_deletion(self, node, entry) + cpdef mark_reference(self, node, entry) - @cython.locals(block=ControlBlock, parent=ControlBlock, unreachable=set) - cpdef normalize(self) + @cython.locals(block=ControlBlock, parent=ControlBlock, unreachable=set) + cpdef normalize(self) - @cython.locals(bit=object, assmts=AssignmentList, - block=ControlBlock) - cpdef initialize(self) + @cython.locals(bit=object, assmts=AssignmentList, block=ControlBlock) + cpdef initialize(self) - @cython.locals(assmts=AssignmentList, assmt=NameAssignment) - cpdef set map_one(self, istate, entry) + @cython.locals(assmts=AssignmentList, assmt=NameAssignment) + cpdef set map_one(self, istate, entry) - @cython.locals(block=ControlBlock, parent=ControlBlock) - cdef reaching_definitions(self) + @cython.locals(block=ControlBlock, parent=ControlBlock) + cdef reaching_definitions(self) cdef class Uninitialized: - pass + pass cdef class Unknown: pass - cdef class MessageCollection: cdef set messages - @cython.locals(dirty=bint, block=ControlBlock, parent=ControlBlock, assmt=NameAssignment) cdef check_definitions(ControlFlow flow, dict compiler_directives) @@ -101,11 +101,11 @@ cdef class ControlFlowAnalysis(CythonTransform): cdef object gv_ctx cdef object constant_folder cdef set reductions - cdef list env_stack - cdef list stack + cdef list stack # a stack of (env, flow) tuples cdef object env cdef ControlFlow flow + cdef object object_expr cdef bint in_inplace_assignment - cpdef mark_assignment(self, lhs, rhs=*) + cpdef mark_assignment(self, lhs, rhs=*, rhs_scope=*) cpdef mark_position(self, node) diff --git a/Cython/Compiler/FlowControl.py b/Cython/Compiler/FlowControl.py index df04471f9..294bce9ee 100644 --- a/Cython/Compiler/FlowControl.py +++ b/Cython/Compiler/FlowControl.py @@ -1,21 +1,22 @@ +# cython: language_level=3str +# cython: auto_pickle=True + from __future__ import absolute_import import cython -cython.declare(PyrexTypes=object, ExprNodes=object, Nodes=object, - Builtin=object, InternalError=object, error=object, warning=object, - py_object_type=object, unspecified_type=object, - object_expr=object, fake_rhs_expr=object, TypedExprNode=object) +cython.declare(PyrexTypes=object, ExprNodes=object, Nodes=object, Builtin=object, + Options=object, TreeVisitor=object, CythonTransform=object, + InternalError=object, error=object, warning=object, + fake_rhs_expr=object, TypedExprNode=object) from . import Builtin from . import ExprNodes from . import Nodes from . import Options -from .PyrexTypes import py_object_type, unspecified_type from . import PyrexTypes from .Visitor import TreeVisitor, CythonTransform from .Errors import error, warning, InternalError -from .Optimize import ConstantFolding class TypedExprNode(ExprNodes.ExprNode): @@ -28,9 +29,8 @@ class TypedExprNode(ExprNodes.ExprNode): def may_be_none(self): return self._may_be_none != False -object_expr = TypedExprNode(py_object_type, may_be_none=True) # Fake rhs to silence "unused variable" warning -fake_rhs_expr = TypedExprNode(unspecified_type) +fake_rhs_expr = TypedExprNode(PyrexTypes.unspecified_type) class ControlBlock(object): @@ -52,7 +52,7 @@ class ControlBlock(object): stats = [Assignment(a), NameReference(a), NameReference(c), Assignment(b)] gen = {Entry(a): Assignment(a), Entry(b): Assignment(b)} - bounded = set([Entry(a), Entry(c)]) + bounded = {Entry(a), Entry(c)} """ @@ -110,6 +110,7 @@ class ControlFlow(object): entries set tracked entries loops list stack for loop descriptors exceptions list stack for exception descriptors + in_try_block int track if we're in a try...except or try...finally block """ def __init__(self): @@ -122,6 +123,7 @@ class ControlFlow(object): self.exit_point = ExitBlock() self.blocks.add(self.exit_point) self.block = self.entry_point + self.in_try_block = 0 def newblock(self, parent=None): """Create floating block linked to `parent` if given. @@ -160,7 +162,7 @@ class ControlFlow(object): (entry.type.is_struct_or_union or entry.type.is_complex or entry.type.is_array or - entry.type.is_cpp_class)): + (entry.type.is_cpp_class and not entry.is_cpp_optional))): # stack allocated structured variable => never uninitialised return True return False @@ -170,9 +172,9 @@ class ControlFlow(object): if self.block: self.block.positions.add(node.pos[:2]) - def mark_assignment(self, lhs, rhs, entry): + def mark_assignment(self, lhs, rhs, entry, rhs_scope=None): if self.block and self.is_tracked(entry): - assignment = NameAssignment(lhs, rhs, entry) + assignment = NameAssignment(lhs, rhs, entry, rhs_scope=rhs_scope) self.block.stats.append(assignment) self.block.gen[entry] = assignment self.entries.add(entry) @@ -203,7 +205,7 @@ class ControlFlow(object): def normalize(self): """Delete unreachable and orphan blocks.""" - queue = set([self.entry_point]) + queue = {self.entry_point} visited = set() while queue: root = queue.pop() @@ -217,7 +219,7 @@ class ControlFlow(object): visited.remove(self.entry_point) for block in visited: if block.empty(): - for parent in block.parents: # Re-parent + for parent in block.parents: # Re-parent for child in block.children: parent.add_child(child) block.detach() @@ -313,7 +315,7 @@ class ExceptionDescr(object): class NameAssignment(object): - def __init__(self, lhs, rhs, entry): + def __init__(self, lhs, rhs, entry, rhs_scope=None): if lhs.cf_state is None: lhs.cf_state = set() self.lhs = lhs @@ -324,16 +326,18 @@ class NameAssignment(object): self.is_arg = False self.is_deletion = False self.inferred_type = None + # For generator expression targets, the rhs can have a different scope than the lhs. + self.rhs_scope = rhs_scope def __repr__(self): return '%s(entry=%r)' % (self.__class__.__name__, self.entry) def infer_type(self): - self.inferred_type = self.rhs.infer_type(self.entry.scope) + self.inferred_type = self.rhs.infer_type(self.rhs_scope or self.entry.scope) return self.inferred_type def type_dependencies(self): - return self.rhs.type_dependencies(self.entry.scope) + return self.rhs.type_dependencies(self.rhs_scope or self.entry.scope) @property def type(self): @@ -373,9 +377,9 @@ class NameDeletion(NameAssignment): def infer_type(self): inferred_type = self.rhs.infer_type(self.entry.scope) - if (not inferred_type.is_pyobject and - inferred_type.can_coerce_to_pyobject(self.entry.scope)): - return py_object_type + if (not inferred_type.is_pyobject + and inferred_type.can_coerce_to_pyobject(self.entry.scope)): + return PyrexTypes.py_object_type self.inferred_type = inferred_type return inferred_type @@ -455,7 +459,7 @@ class GVContext(object): start = min(block.positions) stop = max(block.positions) srcdescr = start[0] - if not srcdescr in self.sources: + if srcdescr not in self.sources: self.sources[srcdescr] = list(srcdescr.get_lines()) lines = self.sources[srcdescr] return '\\n'.join([l.strip() for l in lines[start[1] - 1:stop[1]]]) @@ -590,7 +594,7 @@ def check_definitions(flow, compiler_directives): if (node.allow_null or entry.from_closure or entry.is_pyclass_attr or entry.type.is_error): pass # Can be uninitialized here - elif node.cf_is_null: + elif node.cf_is_null and not entry.in_closure: if entry.error_on_uninitialized or ( Options.error_on_uninitialized and ( entry.type.is_pyobject or entry.type.is_unspecified)): @@ -604,10 +608,12 @@ def check_definitions(flow, compiler_directives): "local variable '%s' referenced before assignment" % entry.name) elif warn_maybe_uninitialized: + msg = "local variable '%s' might be referenced before assignment" % entry.name + if entry.in_closure: + msg += " (maybe initialized inside a closure)" messages.warning( node.pos, - "local variable '%s' might be referenced before assignment" - % entry.name) + msg) elif Unknown in node.cf_state: # TODO: better cross-closure analysis to know when inner functions # are being called before a variable is being set, and when @@ -621,7 +627,7 @@ def check_definitions(flow, compiler_directives): # Unused result for assmt in assignments: if (not assmt.refs and not assmt.entry.is_pyclass_attr - and not assmt.entry.in_closure): + and not assmt.entry.in_closure): if assmt.entry.cf_references and warn_unused_result: if assmt.is_arg: messages.warning(assmt.pos, "Unused argument value '%s'" % @@ -661,7 +667,7 @@ class AssignmentCollector(TreeVisitor): self.assignments = [] def visit_Node(self): - self._visitchildren(self, None) + self._visitchildren(self, None, None) def visit_SingleAssignmentNode(self, node): self.assignments.append((node.lhs, node.rhs)) @@ -673,30 +679,37 @@ class AssignmentCollector(TreeVisitor): class ControlFlowAnalysis(CythonTransform): + def find_in_stack(self, env): + if env == self.env: + return self.flow + for e, flow in reversed(self.stack): + if e is env: + return flow + assert False + def visit_ModuleNode(self, node): - self.gv_ctx = GVContext() + dot_output = self.current_directives['control_flow.dot_output'] + self.gv_ctx = GVContext() if dot_output else None + + from .Optimize import ConstantFolding self.constant_folder = ConstantFolding() # Set of NameNode reductions self.reductions = set() self.in_inplace_assignment = False - self.env_stack = [] self.env = node.scope - self.stack = [] self.flow = ControlFlow() + self.stack = [] # a stack of (env, flow) tuples + self.object_expr = TypedExprNode(PyrexTypes.py_object_type, may_be_none=True) self.visitchildren(node) check_definitions(self.flow, self.current_directives) - dot_output = self.current_directives['control_flow.dot_output'] if dot_output: annotate_defs = self.current_directives['control_flow.dot_annotate_defs'] - fp = open(dot_output, 'wt') - try: + with open(dot_output, 'wt') as fp: self.gv_ctx.render(fp, 'module', annotate_defs=annotate_defs) - finally: - fp.close() return node def visit_FuncDefNode(self, node): @@ -704,9 +717,8 @@ class ControlFlowAnalysis(CythonTransform): if arg.default: self.visitchildren(arg) self.visitchildren(node, ('decorators',)) - self.env_stack.append(self.env) + self.stack.append((self.env, self.flow)) self.env = node.local_scope - self.stack.append(self.flow) self.flow = ControlFlow() # Collect all entries @@ -744,10 +756,10 @@ class ControlFlowAnalysis(CythonTransform): check_definitions(self.flow, self.current_directives) self.flow.blocks.add(self.flow.entry_point) - self.gv_ctx.add(GV(node.local_scope.name, self.flow)) + if self.gv_ctx is not None: + self.gv_ctx.add(GV(node.local_scope.name, self.flow)) - self.flow = self.stack.pop() - self.env = self.env_stack.pop() + self.env, self.flow = self.stack.pop() return node def visit_DefNode(self, node): @@ -760,7 +772,7 @@ class ControlFlowAnalysis(CythonTransform): def visit_CTypeDefNode(self, node): return node - def mark_assignment(self, lhs, rhs=None): + def mark_assignment(self, lhs, rhs=None, rhs_scope=None): if not self.flow.block: return if self.flow.exceptions: @@ -769,19 +781,22 @@ class ControlFlowAnalysis(CythonTransform): self.flow.nextblock() if not rhs: - rhs = object_expr + rhs = self.object_expr if lhs.is_name: if lhs.entry is not None: entry = lhs.entry else: entry = self.env.lookup(lhs.name) - if entry is None: # TODO: This shouldn't happen... + if entry is None: # TODO: This shouldn't happen... return - self.flow.mark_assignment(lhs, rhs, entry) + self.flow.mark_assignment(lhs, rhs, entry, rhs_scope=rhs_scope) elif lhs.is_sequence_constructor: for i, arg in enumerate(lhs.args): - if not rhs or arg.is_starred: - item_node = None + if arg.is_starred: + # "a, *b = x" assigns a list to "b" + item_node = TypedExprNode(Builtin.list_type, may_be_none=False, pos=arg.pos) + elif rhs is self.object_expr: + item_node = rhs else: item_node = rhs.inferable_item_node(i) self.mark_assignment(arg, item_node) @@ -806,7 +821,7 @@ class ControlFlowAnalysis(CythonTransform): return node def visit_AssignmentNode(self, node): - raise InternalError("Unhandled assignment node") + raise InternalError("Unhandled assignment node %s" % type(node)) def visit_SingleAssignmentNode(self, node): self._visit(node.rhs) @@ -916,6 +931,26 @@ class ControlFlowAnalysis(CythonTransform): self.flow.block = None return node + def visit_AssertStatNode(self, node): + """Essentially an if-condition that wraps a RaiseStatNode. + """ + self.mark_position(node) + next_block = self.flow.newblock() + parent = self.flow.block + # failure case + parent = self.flow.nextblock(parent) + self._visit(node.condition) + self.flow.nextblock() + self._visit(node.exception) + if self.flow.block: + self.flow.block.add_child(next_block) + parent.add_child(next_block) + if next_block.parents: + self.flow.block = next_block + else: + self.flow.block = None + return node + def visit_WhileStatNode(self, node): condition_block = self.flow.nextblock() next_block = self.flow.newblock() @@ -951,10 +986,11 @@ class ControlFlowAnalysis(CythonTransform): is_special = False sequence = node.iterator.sequence target = node.target + env = node.iterator.expr_scope or self.env if isinstance(sequence, ExprNodes.SimpleCallNode): function = sequence.function if sequence.self is None and function.is_name: - entry = self.env.lookup(function.name) + entry = env.lookup(function.name) if not entry or entry.is_builtin: if function.name == 'reversed' and len(sequence.args) == 1: sequence = sequence.args[0] @@ -962,30 +998,32 @@ class ControlFlowAnalysis(CythonTransform): if target.is_sequence_constructor and len(target.args) == 2: iterator = sequence.args[0] if iterator.is_name: - iterator_type = iterator.infer_type(self.env) + iterator_type = iterator.infer_type(env) if iterator_type.is_builtin_type: # assume that builtin types have a length within Py_ssize_t self.mark_assignment( target.args[0], ExprNodes.IntNode(target.pos, value='PY_SSIZE_T_MAX', - type=PyrexTypes.c_py_ssize_t_type)) + type=PyrexTypes.c_py_ssize_t_type), + rhs_scope=node.iterator.expr_scope) target = target.args[1] sequence = sequence.args[0] if isinstance(sequence, ExprNodes.SimpleCallNode): function = sequence.function if sequence.self is None and function.is_name: - entry = self.env.lookup(function.name) + entry = env.lookup(function.name) if not entry or entry.is_builtin: if function.name in ('range', 'xrange'): is_special = True for arg in sequence.args[:2]: - self.mark_assignment(target, arg) + self.mark_assignment(target, arg, rhs_scope=node.iterator.expr_scope) if len(sequence.args) > 2: self.mark_assignment(target, self.constant_folder( ExprNodes.binop_node(node.pos, '+', sequence.args[0], - sequence.args[2]))) + sequence.args[2])), + rhs_scope=node.iterator.expr_scope) if not is_special: # A for-loop basically translates to subsequent calls to @@ -994,7 +1032,7 @@ class ControlFlowAnalysis(CythonTransform): # Python strings, etc., while correctly falling back to an # object type when the base type cannot be handled. - self.mark_assignment(target, node.item) + self.mark_assignment(target, node.item, rhs_scope=node.iterator.expr_scope) def visit_AsyncForStatNode(self, node): return self.visit_ForInStatNode(node) @@ -1013,7 +1051,7 @@ class ControlFlowAnalysis(CythonTransform): elif isinstance(node, Nodes.AsyncForStatNode): # not entirely correct, but good enough for now self.mark_assignment(node.target, node.item) - else: # Parallel + else: # Parallel self.mark_assignment(node.target) # Body block @@ -1140,7 +1178,9 @@ class ControlFlowAnalysis(CythonTransform): ## XXX: children nodes self.flow.block.add_child(entry_point) self.flow.nextblock() + self.flow.in_try_block += 1 self._visit(node.body) + self.flow.in_try_block -= 1 self.flow.exceptions.pop() # After exception @@ -1200,7 +1240,9 @@ class ControlFlowAnalysis(CythonTransform): self.flow.block = body_block body_block.add_child(entry_point) self.flow.nextblock() + self.flow.in_try_block += 1 self._visit(node.body) + self.flow.in_try_block -= 1 self.flow.exceptions.pop() if self.flow.loops: self.flow.loops[-1].exceptions.pop() @@ -1219,6 +1261,8 @@ class ControlFlowAnalysis(CythonTransform): if self.flow.exceptions: self.flow.block.add_child(self.flow.exceptions[-1].entry_point) self.flow.block = None + if self.flow.in_try_block: + node.in_try_block = True return node def visit_ReraiseStatNode(self, node): @@ -1287,34 +1331,47 @@ class ControlFlowAnalysis(CythonTransform): def visit_ComprehensionNode(self, node): if node.expr_scope: - self.env_stack.append(self.env) + self.stack.append((self.env, self.flow)) self.env = node.expr_scope # Skip append node here self._visit(node.loop) if node.expr_scope: - self.env = self.env_stack.pop() + self.env, _ = self.stack.pop() return node def visit_ScopedExprNode(self, node): + # currently this is written to deal with these two types + # (with comprehensions covered in their own function) + assert isinstance(node, (ExprNodes.IteratorNode, ExprNodes.AsyncIteratorNode)), node if node.expr_scope: - self.env_stack.append(self.env) + self.stack.append((self.env, self.flow)) + self.flow = self.find_in_stack(node.expr_scope) self.env = node.expr_scope self.visitchildren(node) if node.expr_scope: - self.env = self.env_stack.pop() + self.env, self.flow = self.stack.pop() return node def visit_PyClassDefNode(self, node): self.visitchildren(node, ('dict', 'metaclass', 'mkw', 'bases', 'class_result')) self.flow.mark_assignment(node.target, node.classobj, - self.env.lookup(node.name)) - self.env_stack.append(self.env) + self.env.lookup(node.target.name)) + self.stack.append((self.env, self.flow)) self.env = node.scope self.flow.nextblock() + if node.doc_node: + self.flow.mark_assignment(node.doc_node, fake_rhs_expr, node.doc_node.entry) self.visitchildren(node, ('body',)) self.flow.nextblock() - self.env = self.env_stack.pop() + self.env, _ = self.stack.pop() + return node + + def visit_CClassDefNode(self, node): + # just make sure the nodes scope is findable in-case there is a list comprehension in it + self.stack.append((node.scope, self.flow)) + self.visitchildren(node) + self.stack.pop() return node def visit_AmpersandNode(self, node): diff --git a/Cython/Compiler/FusedNode.py b/Cython/Compiler/FusedNode.py index 26d6ffd3d..7876916db 100644 --- a/Cython/Compiler/FusedNode.py +++ b/Cython/Compiler/FusedNode.py @@ -7,6 +7,7 @@ from . import (ExprNodes, PyrexTypes, MemoryView, from .ExprNodes import CloneNode, ProxyNode, TupleNode from .Nodes import FuncDefNode, CFuncDefNode, StatListNode, DefNode from ..Utils import OrderedSet +from .Errors import error, CannotSpecialize class FusedCFuncDefNode(StatListNode): @@ -141,7 +142,14 @@ class FusedCFuncDefNode(StatListNode): copied_node = copy.deepcopy(self.node) # Make the types in our CFuncType specific. - type = copied_node.type.specialize(fused_to_specific) + try: + type = copied_node.type.specialize(fused_to_specific) + except CannotSpecialize: + # unlike for the argument types, specializing the return type can fail + error(copied_node.pos, "Return type is a fused type that cannot " + "be determined from the function arguments") + self.py_func = None # this is just to let the compiler exit gracefully + return entry = copied_node.entry type.specialize_entry(entry, cname) @@ -220,6 +228,10 @@ class FusedCFuncDefNode(StatListNode): arg.type = arg.type.specialize(fused_to_specific) if arg.type.is_memoryviewslice: arg.type.validate_memslice_dtype(arg.pos) + if arg.annotation: + # TODO might be nice if annotations were specialized instead? + # (Or might be hard to do reliably) + arg.annotation.untyped = True def create_new_local_scope(self, node, env, f2s): """ @@ -264,12 +276,12 @@ class FusedCFuncDefNode(StatListNode): Returns whether an error was issued and whether we should stop in in order to prevent a flood of errors. """ - num_errors = Errors.num_errors + num_errors = Errors.get_errors_count() transform = ParseTreeTransforms.ReplaceFusedTypeChecks( copied_node.local_scope) transform(copied_node) - if Errors.num_errors > num_errors: + if Errors.get_errors_count() > num_errors: return False return True @@ -309,25 +321,21 @@ class FusedCFuncDefNode(StatListNode): def _buffer_check_numpy_dtype_setup_cases(self, pyx_code): "Setup some common cases to match dtypes against specializations" - if pyx_code.indenter("if kind in b'iu':"): + with pyx_code.indenter("if kind in b'iu':"): pyx_code.putln("pass") pyx_code.named_insertion_point("dtype_int") - pyx_code.dedent() - if pyx_code.indenter("elif kind == b'f':"): + with pyx_code.indenter("elif kind == b'f':"): pyx_code.putln("pass") pyx_code.named_insertion_point("dtype_float") - pyx_code.dedent() - if pyx_code.indenter("elif kind == b'c':"): + with pyx_code.indenter("elif kind == b'c':"): pyx_code.putln("pass") pyx_code.named_insertion_point("dtype_complex") - pyx_code.dedent() - if pyx_code.indenter("elif kind == b'O':"): + with pyx_code.indenter("elif kind == b'O':"): pyx_code.putln("pass") pyx_code.named_insertion_point("dtype_object") - pyx_code.dedent() match = "dest_sig[{{dest_sig_idx}}] = '{{specialized_type_name}}'" no_match = "dest_sig[{{dest_sig_idx}}] = None" @@ -364,11 +372,10 @@ class FusedCFuncDefNode(StatListNode): if final_type.is_pythran_expr: cond += ' and arg_is_pythran_compatible' - if codewriter.indenter("if %s:" % cond): + with codewriter.indenter("if %s:" % cond): #codewriter.putln("print 'buffer match found based on numpy dtype'") codewriter.putln(self.match) codewriter.putln("break") - codewriter.dedent() def _buffer_parse_format_string_check(self, pyx_code, decl_code, specialized_type, env): @@ -394,15 +401,30 @@ class FusedCFuncDefNode(StatListNode): pyx_code.context.update( specialized_type_name=specialized_type.specialization_string, - sizeof_dtype=self._sizeof_dtype(dtype)) + sizeof_dtype=self._sizeof_dtype(dtype), + ndim_dtype=specialized_type.ndim, + dtype_is_struct_obj=int(dtype.is_struct or dtype.is_pyobject)) + # use the memoryview object to check itemsize and ndim. + # In principle it could check more, but these are the easiest to do quickly pyx_code.put_chunk( u""" # try {{dtype}} - if itemsize == -1 or itemsize == {{sizeof_dtype}}: - memslice = {{coerce_from_py_func}}(arg, 0) + if (((itemsize == -1 and arg_as_memoryview.itemsize == {{sizeof_dtype}}) + or itemsize == {{sizeof_dtype}}) + and arg_as_memoryview.ndim == {{ndim_dtype}}): + {{if dtype_is_struct_obj}} + if __PYX_IS_PYPY2: + # I wasn't able to diagnose why, but PyPy2 fails to convert a + # memoryview to a Cython memoryview in this case + memslice = {{coerce_from_py_func}}(arg, 0) + else: + {{else}} + if True: + {{endif}} + memslice = {{coerce_from_py_func}}(arg_as_memoryview, 0) if memslice.memview: - __PYX_XDEC_MEMVIEW(&memslice, 1) + __PYX_XCLEAR_MEMVIEW(&memslice, 1) # print 'found a match for the buffer through format parsing' %s break @@ -410,7 +432,7 @@ class FusedCFuncDefNode(StatListNode): __pyx_PyErr_Clear() """ % self.match) - def _buffer_checks(self, buffer_types, pythran_types, pyx_code, decl_code, env): + def _buffer_checks(self, buffer_types, pythran_types, pyx_code, decl_code, accept_none, env): """ Generate Cython code to match objects to buffer specializations. First try to get a numpy dtype object and match it against the individual @@ -467,9 +489,35 @@ class FusedCFuncDefNode(StatListNode): self._buffer_check_numpy_dtype(pyx_code, buffer_types, pythran_types) pyx_code.dedent(2) - for specialized_type in buffer_types: - self._buffer_parse_format_string_check( - pyx_code, decl_code, specialized_type, env) + if accept_none: + # If None is acceptable, then Cython <3.0 matched None with the + # first type. This behaviour isn't ideal, but keep it for backwards + # compatibility. Better behaviour would be to see if subsequent + # arguments give a stronger match. + pyx_code.context.update( + specialized_type_name=buffer_types[0].specialization_string + ) + pyx_code.put_chunk( + """ + if arg is None: + %s + break + """ % self.match) + + # creating a Cython memoryview from a Python memoryview avoids the + # need to get the buffer multiple times, and we can + # also use it to check itemsizes etc + pyx_code.put_chunk( + """ + try: + arg_as_memoryview = memoryview(arg) + except (ValueError, TypeError): + pass + """) + with pyx_code.indenter("else:"): + for specialized_type in buffer_types: + self._buffer_parse_format_string_check( + pyx_code, decl_code, specialized_type, env) def _buffer_declarations(self, pyx_code, decl_code, all_buffer_types, pythran_types): """ @@ -481,8 +529,9 @@ class FusedCFuncDefNode(StatListNode): ctypedef struct {{memviewslice_cname}}: void *memview - void __PYX_XDEC_MEMVIEW({{memviewslice_cname}} *, int have_gil) + void __PYX_XCLEAR_MEMVIEW({{memviewslice_cname}} *, int have_gil) bint __pyx_memoryview_check(object) + bint __PYX_IS_PYPY2 "(CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION == 2)" """) pyx_code.local_variable_declarations.put_chunk( @@ -507,6 +556,12 @@ class FusedCFuncDefNode(StatListNode): ndarray = __Pyx_ImportNumPyArrayTypeIfAvailable() """) + pyx_code.imports.put_chunk( + u""" + cdef memoryview arg_as_memoryview + """ + ) + seen_typedefs = set() seen_int_dtypes = set() for buffer_type in all_buffer_types: @@ -580,6 +635,26 @@ class FusedCFuncDefNode(StatListNode): {{endif}} """) + def _fused_signature_index(self, pyx_code): + """ + Generate Cython code for constructing a persistent nested dictionary index of + fused type specialization signatures. + """ + pyx_code.put_chunk( + u""" + if not _fused_sigindex: + for sig in <dict>signatures: + sigindex_node = _fused_sigindex + *sig_series, last_type = sig.strip('()').split('|') + for sig_type in sig_series: + if sig_type not in sigindex_node: + sigindex_node[sig_type] = sigindex_node = {} + else: + sigindex_node = sigindex_node[sig_type] + sigindex_node[last_type] = sig + """ + ) + def make_fused_cpdef(self, orig_py_func, env, is_def): """ This creates the function that is indexable from Python and does @@ -616,10 +691,14 @@ class FusedCFuncDefNode(StatListNode): pyx_code.put_chunk( u""" - def __pyx_fused_cpdef(signatures, args, kwargs, defaults): + def __pyx_fused_cpdef(signatures, args, kwargs, defaults, _fused_sigindex={}): # FIXME: use a typed signature - currently fails badly because # default arguments inherit the types we specify here! + cdef list search_list + + cdef dict sn, sigindex_node + dest_sig = [None] * {{n_fused}} if kwargs is not None and not kwargs: @@ -630,7 +709,7 @@ class FusedCFuncDefNode(StatListNode): # instance check body """) - pyx_code.indent() # indent following code to function body + pyx_code.indent() # indent following code to function body pyx_code.named_insertion_point("imports") pyx_code.named_insertion_point("func_defs") pyx_code.named_insertion_point("local_variable_declarations") @@ -661,19 +740,20 @@ class FusedCFuncDefNode(StatListNode): self._unpack_argument(pyx_code) # 'unrolled' loop, first match breaks out of it - if pyx_code.indenter("while 1:"): + with pyx_code.indenter("while 1:"): if normal_types: self._fused_instance_checks(normal_types, pyx_code, env) if buffer_types or pythran_types: env.use_utility_code(Code.UtilityCode.load_cached("IsLittleEndian", "ModuleSetupCode.c")) - self._buffer_checks(buffer_types, pythran_types, pyx_code, decl_code, env) + self._buffer_checks( + buffer_types, pythran_types, pyx_code, decl_code, + arg.accept_none, env) if has_object_fallback: pyx_code.context.update(specialized_type_name='object') pyx_code.putln(self.match) else: pyx_code.putln(self.no_match) pyx_code.putln("break") - pyx_code.dedent() fused_index += 1 all_buffer_types.update(buffer_types) @@ -687,23 +767,36 @@ class FusedCFuncDefNode(StatListNode): env.use_utility_code(Code.UtilityCode.load_cached("Import", "ImportExport.c")) env.use_utility_code(Code.UtilityCode.load_cached("ImportNumPyArray", "ImportExport.c")) + self._fused_signature_index(pyx_code) + pyx_code.put_chunk( u""" - candidates = [] - for sig in <dict>signatures: - match_found = False - src_sig = sig.strip('()').split('|') - for i in range(len(dest_sig)): - dst_type = dest_sig[i] - if dst_type is not None: - if src_sig[i] == dst_type: - match_found = True - else: - match_found = False - break + sigindex_matches = [] + sigindex_candidates = [_fused_sigindex] + + for dst_type in dest_sig: + found_matches = [] + found_candidates = [] + # Make two seperate lists: One for signature sub-trees + # with at least one definite match, and another for + # signature sub-trees with only ambiguous matches + # (where `dest_sig[i] is None`). + if dst_type is None: + for sn in sigindex_matches: + found_matches.extend(sn.values()) + for sn in sigindex_candidates: + found_candidates.extend(sn.values()) + else: + for search_list in (sigindex_matches, sigindex_candidates): + for sn in search_list: + if dst_type in sn: + found_matches.append(sn[dst_type]) + sigindex_matches = found_matches + sigindex_candidates = found_candidates + if not (found_matches or found_candidates): + break - if match_found: - candidates.append(sig) + candidates = sigindex_matches if not candidates: raise TypeError("No matching signature found") @@ -792,16 +885,18 @@ class FusedCFuncDefNode(StatListNode): for arg in self.node.args: if arg.default: arg.default = arg.default.analyse_expressions(env) - defaults.append(ProxyNode(arg.default)) + # coerce the argument to temp since CloneNode really requires a temp + defaults.append(ProxyNode(arg.default.coerce_to_temp(env))) else: defaults.append(None) for i, stat in enumerate(self.stats): stat = self.stats[i] = stat.analyse_expressions(env) - if isinstance(stat, FuncDefNode): + if isinstance(stat, FuncDefNode) and stat is not self.py_func: + # the dispatcher specifically doesn't want its defaults overriding for arg, default in zip(stat.args, defaults): if default is not None: - arg.default = CloneNode(default).coerce_to(arg.type, env) + arg.default = CloneNode(default).analyse_expressions(env).coerce_to(arg.type, env) if self.py_func: args = [CloneNode(default) for default in defaults if default] @@ -829,6 +924,10 @@ class FusedCFuncDefNode(StatListNode): else: nodes = self.nodes + # For the moment, fused functions do not support METH_FASTCALL + for node in nodes: + node.entry.signature.use_fastcall = False + signatures = [StringEncoding.EncodedString(node.specialized_signature_string) for node in nodes] keys = [ExprNodes.StringNode(node.pos, value=sig) @@ -847,8 +946,10 @@ class FusedCFuncDefNode(StatListNode): self.py_func.pymethdef_required = True self.fused_func_assignment.generate_function_definitions(env, code) + from . import Options for stat in self.stats: - if isinstance(stat, FuncDefNode) and stat.entry.used: + from_pyx = Options.cimport_from_pyx and not stat.entry.visibility == 'extern' + if isinstance(stat, FuncDefNode) and (stat.entry.used or from_pyx): code.mark_pos(stat.pos) stat.generate_function_definitions(env, code) @@ -877,7 +978,7 @@ class FusedCFuncDefNode(StatListNode): "((__pyx_FusedFunctionObject *) %s)->__signatures__ = %s;" % (self.resulting_fused_function.result(), self.__signatures__.result())) - code.put_giveref(self.__signatures__.result()) + self.__signatures__.generate_giveref(code) self.__signatures__.generate_post_assignment_code(code) self.__signatures__.free_temps(code) diff --git a/Cython/Compiler/Future.py b/Cython/Compiler/Future.py index 848792e00..8de10c0cb 100644 --- a/Cython/Compiler/Future.py +++ b/Cython/Compiler/Future.py @@ -11,5 +11,6 @@ absolute_import = _get_feature("absolute_import") nested_scopes = _get_feature("nested_scopes") # dummy generators = _get_feature("generators") # dummy generator_stop = _get_feature("generator_stop") +annotations = _get_feature("annotations") del _get_feature diff --git a/Cython/Compiler/Interpreter.py b/Cython/Compiler/Interpreter.py index 9ec391f2a..244397264 100644 --- a/Cython/Compiler/Interpreter.py +++ b/Cython/Compiler/Interpreter.py @@ -47,8 +47,8 @@ def interpret_compiletime_options(optlist, optdict, type_env=None, type_args=()) raise CompileError(node.pos, "Type not allowed here.") else: if (sys.version_info[0] >=3 and - isinstance(node, StringNode) and - node.unicode_value is not None): + isinstance(node, StringNode) and + node.unicode_value is not None): return (node.unicode_value, node.pos) return (node.compile_time_value(empty_scope), node.pos) diff --git a/Cython/Compiler/Lexicon.py b/Cython/Compiler/Lexicon.py index 72c9ceaef..c3ca05b56 100644 --- a/Cython/Compiler/Lexicon.py +++ b/Cython/Compiler/Lexicon.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # cython: language_level=3, py2_import=True # # Cython Scanner - Lexical Definitions @@ -16,28 +17,43 @@ IDENT = 'IDENT' def make_lexicon(): from ..Plex import \ Str, Any, AnyBut, AnyChar, Rep, Rep1, Opt, Bol, Eol, Eof, \ - TEXT, IGNORE, State, Lexicon - from .Scanning import Method + TEXT, IGNORE, Method, State, Lexicon, Range - letter = Any("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_") + nonzero_digit = Any("123456789") digit = Any("0123456789") bindigit = Any("01") octdigit = Any("01234567") hexdigit = Any("0123456789ABCDEFabcdef") indentation = Bol + Rep(Any(" \t")) + # The list of valid unicode identifier characters are pretty slow to generate at runtime, + # and require Python3, so are just included directly here + # (via the generated code block at the bottom of the file) + unicode_start_character = (Any(unicode_start_ch_any) | Range(unicode_start_ch_range)) + unicode_continuation_character = ( + unicode_start_character | + Any(unicode_continuation_ch_any) | Range(unicode_continuation_ch_range)) + def underscore_digits(d): return Rep1(d) + Rep(Str("_") + Rep1(d)) + def prefixed_digits(prefix, digits): + return prefix + Opt(Str("_")) + underscore_digits(digits) + decimal = underscore_digits(digit) dot = Str(".") exponent = Any("Ee") + Opt(Any("+-")) + decimal decimal_fract = (decimal + dot + Opt(decimal)) | (dot + decimal) - name = letter + Rep(letter | digit) - intconst = decimal | (Str("0") + ((Any("Xx") + underscore_digits(hexdigit)) | - (Any("Oo") + underscore_digits(octdigit)) | - (Any("Bb") + underscore_digits(bindigit)) )) + #name = letter + Rep(letter | digit) + name = unicode_start_character + Rep(unicode_continuation_character) + intconst = (prefixed_digits(nonzero_digit, digit) | # decimal literals with underscores must not start with '0' + (Str("0") + (prefixed_digits(Any("Xx"), hexdigit) | + prefixed_digits(Any("Oo"), octdigit) | + prefixed_digits(Any("Bb"), bindigit) )) | + underscore_digits(Str('0')) # 0_0_0_0... is allowed as a decimal literal + | Rep1(digit) # FIXME: remove these Py2 style decimal/octal literals (PY_VERSION_HEX < 3) + ) intsuffix = (Opt(Any("Uu")) + Opt(Any("Ll")) + Opt(Any("Ll"))) | (Opt(Any("Ll")) + Opt(Any("Ll")) + Opt(Any("Uu"))) intliteral = intconst + intsuffix fltconst = (decimal_fract + Opt(exponent)) | (decimal + exponent) @@ -58,10 +74,11 @@ def make_lexicon(): bra = Any("([{") ket = Any(")]}") + ellipsis = Str("...") punct = Any(":,;+-*/|&<>=.%`~^?!@") diphthong = Str("==", "<>", "!=", "<=", ">=", "<<", ">>", "**", "//", "+=", "-=", "*=", "/=", "%=", "|=", "^=", "&=", - "<<=", ">>=", "**=", "//=", "->", "@=") + "<<=", ">>=", "**=", "//=", "->", "@=", "&&", "||", ':=') spaces = Rep1(Any(" \t\f")) escaped_newline = Str("\\\n") lineterm = Eol + Opt(Str("\n")) @@ -69,11 +86,11 @@ def make_lexicon(): comment = Str("#") + Rep(AnyBut("\n")) return Lexicon([ - (name, IDENT), + (name, Method('normalize_ident')), (intliteral, Method('strip_underscores', symbol='INT')), (fltconst, Method('strip_underscores', symbol='FLOAT')), (imagconst, Method('strip_underscores', symbol='IMAG')), - (punct | diphthong, TEXT), + (ellipsis | punct | diphthong, TEXT), (bra, Method('open_bracket_action')), (ket, Method('close_bracket_action')), @@ -136,3 +153,50 @@ def make_lexicon(): #debug_file = scanner_dump_file ) + +# BEGIN GENERATED CODE +# generated with: +# cpython 3.10.0a0 (heads/master:2b0e654f91, May 29 2020, 16:17:52) + +unicode_start_ch_any = ( + u"_ªµºˬˮͿΆΌՙەۿܐޱߺࠚࠤࠨऽॐলঽৎৼਫ਼ઽૐૹଽୱஃஜௐఽಀಽೞഽൎලาຄລາຽໆༀဿၡႎჇჍቘዀៗៜᢪᪧᳺὙ" + u"ὛὝιⁱⁿℂℇℕℤΩℨⅎⴧⴭⵯꣻꧏꩺꪱꫀꫂיִמּﹱﹳﹷﹹﹻﹽ𐠈𐠼𐨀𐼧𑅄𑅇𑅶𑇚𑇜𑊈𑌽𑍐𑓇𑙄𑚸𑤉𑤿𑥁𑧡𑧣𑨀𑨺𑩐𑪝𑱀𑵆𑶘𑾰𖽐𖿣𝒢" + u"𝒻𝕆𞅎𞥋𞸤𞸧𞸹𞸻𞹂𞹇𞹉𞹋𞹔𞹗𞹙𞹛𞹝𞹟𞹤𞹾" +) +unicode_start_ch_range = ( + u"AZazÀÖØöøˁˆˑˠˤͰʹͶͷͻͽΈΊΎΡΣϵϷҁҊԯԱՖՠֈאתׯײؠيٮٯٱۓۥۦۮۯۺۼܒܯݍޥߊߪߴߵࠀࠕ" + u"ࡀࡘࡠࡪࢠࢴࢶࣇऄहक़ॡॱঀঅঌএঐওনপরশহড়ঢ়য়ৡৰৱਅਊਏਐਓਨਪਰਲਲ਼ਵਸ਼ਸਹਖ਼ੜੲੴઅઍએઑઓનપરલળવહ" + u"ૠૡଅଌଏଐଓନପରଲଳଵହଡ଼ଢ଼ୟୡஅஊஎஐஒகஙசஞடணதநபமஹఅఌఎఐఒనపహౘౚౠౡಅಌಎಐಒನಪಳವಹೠೡೱೲ" + u"ഄഌഎഐഒഺൔൖൟൡൺൿඅඖකනඳරවෆกะเๆກຂຆຊຌຣວະເໄໜໟཀཇཉཬྈྌကဪၐၕၚၝၥၦၮၰၵႁႠჅაჺჼቈ" + u"ቊቍቐቖቚቝበኈኊኍነኰኲኵኸኾዂዅወዖዘጐጒጕጘፚᎀᎏᎠᏵᏸᏽᐁᙬᙯᙿᚁᚚᚠᛪᛮᛸᜀᜌᜎᜑᜠᜱᝀᝑᝠᝬᝮᝰកឳᠠᡸᢀᢨ" + u"ᢰᣵᤀᤞᥐᥭᥰᥴᦀᦫᦰᧉᨀᨖᨠᩔᬅᬳᭅᭋᮃᮠᮮᮯᮺᯥᰀᰣᱍᱏᱚᱽᲀᲈᲐᲺᲽᲿᳩᳬᳮᳳᳵᳶᴀᶿḀἕἘἝἠὅὈὍὐὗὟώᾀᾴ" + u"ᾶᾼῂῄῆῌῐΐῖΊῠῬῲῴῶῼₐₜℊℓ℘ℝKℹℼℿⅅⅉⅠↈⰀⰮⰰⱞⱠⳤⳫⳮⳲⳳⴀⴥⴰⵧⶀⶖⶠⶦⶨⶮⶰⶶⶸⶾⷀⷆⷈⷎⷐⷖ" + u"ⷘⷞ々〇〡〩〱〵〸〼ぁゖゝゟァヺーヿㄅㄯㄱㆎㆠㆿㇰㇿ㐀䶿一鿼ꀀꒌꓐꓽꔀꘌꘐꘟꘪꘫꙀꙮꙿꚝꚠꛯꜗꜟꜢꞈꞋꞿꟂꟊꟵꠁꠃꠅꠇꠊ" + u"ꠌꠢꡀꡳꢂꢳꣲꣷꣽꣾꤊꤥꤰꥆꥠꥼꦄꦲꧠꧤꧦꧯꧺꧾꨀꨨꩀꩂꩄꩋꩠꩶꩾꪯꪵꪶꪹꪽꫛꫝꫠꫪꫲꫴꬁꬆꬉꬎꬑꬖꬠꬦꬨꬮꬰꭚꭜꭩꭰꯢ" + u"가힣ힰퟆퟋퟻ豈舘並龎ffstﬓﬗײַﬨשׁזּטּלּנּסּףּפּצּﮱﯓﱝﱤﴽﵐﶏﶒﷇﷰﷹﹿﻼＡＺａｚｦﾝﾠﾾￂￇￊￏￒￗￚￜ𐀀𐀋𐀍𐀦𐀨𐀺" + u"𐀼𐀽𐀿𐁍𐁐𐁝𐂀𐃺𐅀𐅴𐊀𐊜𐊠𐋐𐌀𐌟𐌭𐍊𐍐𐍵𐎀𐎝𐎠𐏃𐏈𐏏𐏑𐏕𐐀𐒝𐒰𐓓𐓘𐓻𐔀𐔧𐔰𐕣𐘀𐜶𐝀𐝕𐝠𐝧𐠀𐠅𐠊𐠵𐠷𐠸𐠿𐡕𐡠𐡶𐢀𐢞𐣠𐣲𐣴𐣵" + u"𐤀𐤕𐤠𐤹𐦀𐦷𐦾𐦿𐨐𐨓𐨕𐨗𐨙𐨵𐩠𐩼𐪀𐪜𐫀𐫇𐫉𐫤𐬀𐬵𐭀𐭕𐭠𐭲𐮀𐮑𐰀𐱈𐲀𐲲𐳀𐳲𐴀𐴣𐺀𐺩𐺰𐺱𐼀𐼜𐼰𐽅𐾰𐿄𐿠𐿶𑀃𑀷𑂃𑂯𑃐𑃨𑄃𑄦𑅐𑅲" + u"𑆃𑆲𑇁𑇄𑈀𑈑𑈓𑈫𑊀𑊆𑊊𑊍𑊏𑊝𑊟𑊨𑊰𑋞𑌅𑌌𑌏𑌐𑌓𑌨𑌪𑌰𑌲𑌳𑌵𑌹𑍝𑍡𑐀𑐴𑑇𑑊𑑟𑑡𑒀𑒯𑓄𑓅𑖀𑖮𑗘𑗛𑘀𑘯𑚀𑚪𑜀𑜚𑠀𑠫𑢠𑣟𑣿𑤆𑤌𑤓" + u"𑤕𑤖𑤘𑤯𑦠𑦧𑦪𑧐𑨋𑨲𑩜𑪉𑫀𑫸𑰀𑰈𑰊𑰮𑱲𑲏𑴀𑴆𑴈𑴉𑴋𑴰𑵠𑵥𑵧𑵨𑵪𑶉𑻠𑻲𒀀𒎙𒐀𒑮𒒀𒕃𓀀𓐮𔐀𔙆𖠀𖨸𖩀𖩞𖫐𖫭𖬀𖬯𖭀𖭃𖭣𖭷𖭽𖮏𖹀𖹿" + u"𖼀𖽊𖾓𖾟𖿠𖿡𗀀𘟷𘠀𘳕𘴀𘴈𛀀𛄞𛅐𛅒𛅤𛅧𛅰𛋻𛰀𛱪𛱰𛱼𛲀𛲈𛲐𛲙𝐀𝑔𝑖𝒜𝒞𝒟𝒥𝒦𝒩𝒬𝒮𝒹𝒽𝓃𝓅𝔅𝔇𝔊𝔍𝔔𝔖𝔜𝔞𝔹𝔻𝔾𝕀𝕄𝕊𝕐𝕒𝚥" + u"𝚨𝛀𝛂𝛚𝛜𝛺𝛼𝜔𝜖𝜴𝜶𝝎𝝐𝝮𝝰𝞈𝞊𝞨𝞪𝟂𝟄𝟋𞄀𞄬𞄷𞄽𞋀𞋫𞠀𞣄𞤀𞥃𞸀𞸃𞸅𞸟𞸡𞸢𞸩𞸲𞸴𞸷𞹍𞹏𞹑𞹒𞹡𞹢𞹧𞹪𞹬𞹲𞹴𞹷𞹹𞹼𞺀𞺉𞺋𞺛" + u"𞺡𞺣𞺥𞺩𞺫𞺻𠀀𪛝𪜀𫜴𫝀𫠝𫠠𬺡𬺰𮯠丽𪘀" +) +unicode_continuation_ch_any = ( + u"··়ׇֿٰܑ߽ৗ਼৾ੑੵ઼଼ஂௗ಼ൗ්ූัັ༹༵༷࿆᳭ᢩ៝᳴⁔⵿⃡꙯ꠂ꠆ꠋ꠬ꧥꩃﬞꪰ꫁＿𑅳𐨿𐇽𐋠𑈾𑍗𑑞𑥀𑧤𑩇𑴺𑵇𖽏𖿤𝩵" + u"𝪄" +) +unicode_continuation_ch_range = ( + u"09ֽׁׂًؚ֑ׅ̀ͯ҃҇ׄؐ٩۪ۭۖۜ۟ۤۧۨ۰۹ܰ݊ަް߀߉࡙࡛࣓ࣣ߫߳ࠖ࠙ࠛࠣࠥࠧࠩ࠭࣡ःऺ़ाॏ॑ॗॢॣ०९ঁঃ" + u"াৄেৈো্ৢৣ০৯ਁਃਾੂੇੈੋ੍੦ੱઁઃાૅેૉો્ૢૣ૦૯ૺ૿ଁଃାୄେୈୋ୍୕ୗୢୣ୦୯ாூெைொ்௦௯ఀఄాౄ" + u"ెైొ్ౕౖౢౣ౦౯ಁಃಾೄೆೈೊ್ೕೖೢೣ೦೯ഀഃ഻഼ാൄെൈൊ്ൢൣ൦൯ඁඃාුෘෟ෦෯ෲෳำฺ็๎๐๙ຳຼ່ໍ໐໙" + u"༘༙༠༩༾༿྄ཱ྆྇ྍྗྙྼါှ၀၉ၖၙၞၠၢၤၧၭၱၴႂႍႏႝ፝፟፩፱ᜒ᜔ᜲ᜴ᝒᝓᝲᝳ឴៓០៩᠋᠍᠐᠙ᤠᤫᤰ᤻᥆᥏᧐᧚" + u"ᨗᨛᩕᩞ᩠᩿᩼᪉᪐᪙᪽ᪿᫀ᪰ᬀᬄ᬴᭄᭐᭙᭫᭳ᮀᮂᮡᮭ᮰᮹᯦᯳ᰤ᰷᱀᱉᱐᱙᳔᳨᳐᳒᳷᷹᷿᳹᷀᷻‿⁀⃥゙゚〪〯⃐⃜⃰⳯⳱ⷠⷿ" + u"꘠꘩ꙴ꙽ꚞꚟ꛰꛱ꠣꠧꢀꢁꢴꣅ꣐꣙꣠꣱ꣿ꤉ꤦ꤭ꥇ꥓ꦀꦃ꦳꧀꧐꧙꧰꧹ꨩꨶꩌꩍ꩐꩙ꩻꩽꪴꪲꪷꪸꪾ꪿ꫫꫯꫵ꫶ꯣꯪ꯬꯭꯰꯹︀️︠︯" + u"︳︴﹍﹏０９ﾞﾟ𐍶𐍺𐒠𐒩𐨁𐨃𐨅𐨆𐨌𐨺𐫦𐨏𐨸𐫥𐴤𐴧𐴰𐴹𐽆𐽐𐺫𐺬𑀀𑀂𑀸𑁆𑁦𑁯𑁿𑂂𑂰𑂺𑃰𑃹𑄀𑄂𑄧𑄴𑄶𑄿𑅅𑅆𑆀𑆂𑆳𑇀𑇉𑇌𑇎𑇙𑈬𑈷" + u"𑋟𑋪𑋰𑋹𑌀𑌃𑌻𑌼𑌾𑍄𑍇𑍈𑍋𑍍𑍢𑍣𑍦𑍬𑍰𑍴𑐵𑑆𑑐𑑙𑒰𑓃𑓐𑓙𑖯𑖵𑖸𑗀𑗜𑗝𑘰𑙀𑙐𑙙𑚫𑚷𑛀𑛉𑜝𑜫𑜰𑜹𑠬𑠺𑣠𑣩𑤰𑤵𑤷𑤸𑤻𑤾𑥂𑥃𑥐𑥙" + u"𑧑𑧗𑧚𑧠𑨁𑨊𑨳𑨹𑨻𑨾𑩑𑩛𑪊𑪙𑰯𑰶𑰸𑰿𑱐𑱙𑲒𑲧𑲩𑲶𑴱𑴶𑴼𑴽𑴿𑵅𑵐𑵙𑶊𑶎𑶐𑶑𑶓𑶗𑶠𑶩𑻳𑻶𖩠𖩩𖫰𖫴𖬰𖬶𖭐𖭙𖽑𖾇𖾏𖾒𖿰𖿱𛲝𛲞𝅩𝅥" + u"𝅲𝅻𝆂𝆋𝅭𝆅𝆪𝆭𝉂𝉄𝟎𝟿𝨀𝨶𝨻𝩬𝪛𝪟𝪡𝪯𞀀𞀆𞀈𞀘𞀛𞀡𞀣𞀤𞀦𞀪𞄰𞄶𞅀𞅉𞋬𞋹𞥊𞣐𞣖𞥄𞥐𞥙🯰🯹" +) + +# END GENERATED CODE diff --git a/Cython/Compiler/Main.py b/Cython/Compiler/Main.py index 561ac222d..36813975d 100644 --- a/Cython/Compiler/Main.py +++ b/Cython/Compiler/Main.py @@ -2,15 +2,15 @@ # Cython Top Level # -from __future__ import absolute_import +from __future__ import absolute_import, print_function import os import re import sys import io -if sys.version_info[:2] < (2, 6) or (3, 0) <= sys.version_info[:2] < (3, 3): - sys.stderr.write("Sorry, Cython requires Python 2.6+ or 3.3+, found %d.%d\n" % tuple(sys.version_info[:2])) +if sys.version_info[:2] < (2, 7) or (3, 0) <= sys.version_info[:2] < (3, 3): + sys.stderr.write("Sorry, Cython requires Python 2.7 or 3.3+, found %d.%d\n" % tuple(sys.version_info[:2])) sys.exit(1) try: @@ -30,30 +30,28 @@ from .Errors import PyrexError, CompileError, error, warning from .Symtab import ModuleScope from .. import Utils from . import Options +from .Options import CompilationOptions, default_options +from .CmdLine import parse_command_line +from .Lexicon import (unicode_start_ch_any, unicode_continuation_ch_any, + unicode_start_ch_range, unicode_continuation_ch_range) -from . import Version # legacy import needed by old PyTables versions -version = Version.version # legacy attribute - use "Cython.__version__" instead -module_name_pattern = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$") +def _make_range_re(chrs): + out = [] + for i in range(0, len(chrs), 2): + out.append(u"{0}-{1}".format(chrs[i], chrs[i+1])) + return u"".join(out) -verbose = 0 +# py2 version looked like r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$" +module_name_pattern = u"[{0}{1}][{0}{2}{1}{3}]*".format( + unicode_start_ch_any, _make_range_re(unicode_start_ch_range), + unicode_continuation_ch_any, + _make_range_re(unicode_continuation_ch_range)) +module_name_pattern = re.compile(u"{0}(\\.{0})*$".format(module_name_pattern)) -standard_include_path = os.path.abspath(os.path.join(os.path.dirname(__file__), - os.path.pardir, 'Includes')) -class CompilationData(object): - # Bundles the information that is passed from transform to transform. - # (For now, this is only) - - # While Context contains every pxd ever loaded, path information etc., - # this only contains the data related to a single compilation pass - # - # pyx ModuleNode Main code tree of this compilation. - # pxds {string : ModuleNode} Trees for the pxds used in the pyx. - # codewriter CCodeWriter Where to output final code. - # options CompilationOptions - # result CompilationResult - pass +standard_include_path = os.path.abspath( + os.path.join(os.path.dirname(os.path.dirname(__file__)), 'Includes')) class Context(object): @@ -93,10 +91,17 @@ class Context(object): if language_level is not None: self.set_language_level(language_level) + self.legacy_implicit_noexcept = self.compiler_directives.get('legacy_implicit_noexcept', False) + self.gdb_debug_outputwriter = None + @classmethod + def from_options(cls, options): + return cls(options.include_path, options.compiler_directives, + options.cplus, options.language_level, options=options) + def set_language_level(self, level): - from .Future import print_function, unicode_literals, absolute_import, division + from .Future import print_function, unicode_literals, absolute_import, division, generator_stop future_directives = set() if level == '3str': level = 3 @@ -105,7 +110,7 @@ class Context(object): if level >= 3: future_directives.add(unicode_literals) if level >= 3: - future_directives.update([print_function, absolute_import, division]) + future_directives.update([print_function, absolute_import, division, generator_stop]) self.language_level = level self.future_directives = future_directives if level >= 3: @@ -123,15 +128,6 @@ class Context(object): self._interned[key] = value return value - def intern_value(self, value, *key): - key = (type(value), value) + key - try: - return self._interned[key] - except KeyError: - pass - self._interned[key] = value - return value - # pipeline creation functions can now be found in Pipeline.py def process_pxd(self, source_desc, scope, module_name): @@ -149,6 +145,29 @@ class Context(object): def nonfatal_error(self, exc): return Errors.report_error(exc) + def _split_qualified_name(self, qualified_name): + # Splits qualified_name into parts in form of 2-tuples: (PART_NAME, IS_PACKAGE). + qualified_name_parts = qualified_name.split('.') + last_part = qualified_name_parts.pop() + qualified_name_parts = [(p, True) for p in qualified_name_parts] + if last_part != '__init__': + # If Last part is __init__, then it is omitted. Otherwise, we need to check whether we can find + # __init__.pyx/__init__.py file to determine if last part is package or not. + is_package = False + for suffix in ('.py', '.pyx'): + path = self.search_include_directories( + qualified_name, suffix=suffix, source_pos=None, source_file_path=None) + if path: + is_package = self._is_init_file(path) + break + + qualified_name_parts.append((last_part, is_package)) + return qualified_name_parts + + @staticmethod + def _is_init_file(path): + return os.path.basename(path) in ('__init__.pyx', '__init__.py', '__init__.pxd') if path else False + def find_module(self, module_name, relative_to=None, pos=None, need_pxd=1, absolute_fallback=True): # Finds and returns the module scope corresponding to @@ -179,7 +198,7 @@ class Context(object): if not module_name_pattern.match(qualified_name): raise CompileError(pos or (module_name, 0, 0), - "'%s' is not a valid module name" % module_name) + u"'%s' is not a valid module name" % module_name) if relative_to: if debug_find_module: @@ -188,16 +207,16 @@ class Context(object): if not scope: pxd_pathname = self.find_pxd_file(qualified_name, pos) if pxd_pathname: - scope = relative_to.find_submodule(module_name) + is_package = self._is_init_file(pxd_pathname) + scope = relative_to.find_submodule(module_name, as_package=is_package) if not scope: if debug_find_module: print("...trying absolute import") if absolute_fallback: qualified_name = module_name scope = self - for name in qualified_name.split("."): - scope = scope.find_submodule(name) - + for name, is_package in self._split_qualified_name(qualified_name): + scope = scope.find_submodule(name, as_package=is_package) if debug_find_module: print("...scope = %s" % scope) if not scope.pxd_file_loaded: @@ -215,8 +234,9 @@ class Context(object): # Set pxd_file_loaded such that we don't need to # look for the non-existing pxd file next time. scope.pxd_file_loaded = True - package_pathname = self.search_include_directories(qualified_name, ".py", pos) - if package_pathname and package_pathname.endswith('__init__.py'): + package_pathname = self.search_include_directories( + qualified_name, suffix=".py", source_pos=pos) + if package_pathname and package_pathname.endswith(Utils.PACKAGE_FILES): pass else: error(pos, "'%s.pxd' not found" % qualified_name.replace('.', os.sep)) @@ -238,7 +258,7 @@ class Context(object): pass return scope - def find_pxd_file(self, qualified_name, pos, sys_path=True): + def find_pxd_file(self, qualified_name, pos=None, sys_path=True, source_file_path=None): # Search include path (and sys.path if sys_path is True) for # the .pxd file corresponding to the given fully-qualified # module name. @@ -247,53 +267,36 @@ class Context(object): # the directory containing the source file is searched first # for a dotted filename, and its containing package root # directory is searched first for a non-dotted filename. - pxd = self.search_include_directories(qualified_name, ".pxd", pos, sys_path=sys_path) - if pxd is None: # XXX Keep this until Includes/Deprecated is removed - if (qualified_name.startswith('python') or - qualified_name in ('stdlib', 'stdio', 'stl')): - standard_include_path = os.path.abspath(os.path.normpath( - os.path.join(os.path.dirname(__file__), os.path.pardir, 'Includes'))) - deprecated_include_path = os.path.join(standard_include_path, 'Deprecated') - self.include_directories.append(deprecated_include_path) - try: - pxd = self.search_include_directories(qualified_name, ".pxd", pos) - finally: - self.include_directories.pop() - if pxd: - name = qualified_name - if name.startswith('python'): - warning(pos, "'%s' is deprecated, use 'cpython'" % name, 1) - elif name in ('stdlib', 'stdio'): - warning(pos, "'%s' is deprecated, use 'libc.%s'" % (name, name), 1) - elif name in ('stl'): - warning(pos, "'%s' is deprecated, use 'libcpp.*.*'" % name, 1) + pxd = self.search_include_directories( + qualified_name, suffix=".pxd", source_pos=pos, sys_path=sys_path, source_file_path=source_file_path) if pxd is None and Options.cimport_from_pyx: return self.find_pyx_file(qualified_name, pos) return pxd - def find_pyx_file(self, qualified_name, pos): + def find_pyx_file(self, qualified_name, pos=None, source_file_path=None): # Search include path for the .pyx file corresponding to the # given fully-qualified module name, as for find_pxd_file(). - return self.search_include_directories(qualified_name, ".pyx", pos) + return self.search_include_directories( + qualified_name, suffix=".pyx", source_pos=pos, source_file_path=source_file_path) - def find_include_file(self, filename, pos): + def find_include_file(self, filename, pos=None, source_file_path=None): # Search list of include directories for filename. # Reports an error and returns None if not found. - path = self.search_include_directories(filename, "", pos, - include=True) + path = self.search_include_directories( + filename, source_pos=pos, include=True, source_file_path=source_file_path) if not path: error(pos, "'%s' not found" % filename) return path - def search_include_directories(self, qualified_name, suffix, pos, - include=False, sys_path=False): + def search_include_directories(self, qualified_name, + suffix=None, source_pos=None, include=False, sys_path=False, source_file_path=None): include_dirs = self.include_directories if sys_path: include_dirs = include_dirs + sys.path # include_dirs must be hashable for caching in @cached_function include_dirs = tuple(include_dirs + [standard_include_path]) - return search_include_directories(include_dirs, qualified_name, - suffix, pos, include) + return search_include_directories( + include_dirs, qualified_name, suffix or "", source_pos, include, source_file_path) def find_root_package_dir(self, file_path): return Utils.find_root_package_dir(file_path) @@ -307,15 +310,14 @@ class Context(object): c_time = Utils.modification_time(output_path) if Utils.file_newer_than(source_path, c_time): return 1 - pos = [source_path] pxd_path = Utils.replace_suffix(source_path, ".pxd") if os.path.exists(pxd_path) and Utils.file_newer_than(pxd_path, c_time): return 1 for kind, name in self.read_dependency_file(source_path): if kind == "cimport": - dep_path = self.find_pxd_file(name, pos) + dep_path = self.find_pxd_file(name, source_file_path=source_path) elif kind == "include": - dep_path = self.search_include_directories(name, pos) + dep_path = self.search_include_directories(name, source_file_path=source_path) else: continue if dep_path and Utils.file_newer_than(dep_path, c_time): @@ -332,11 +334,10 @@ class Context(object): def read_dependency_file(self, source_path): dep_path = Utils.replace_suffix(source_path, ".dep") if os.path.exists(dep_path): - f = open(dep_path, "rU") - chunks = [ line.strip().split(" ", 1) - for line in f.readlines() - if " " in line.strip() ] - f.close() + with open(dep_path, "rU") as f: + chunks = [ line.split(" ", 1) + for line in (l.strip() for l in f) + if " " in line ] return chunks else: return () @@ -345,12 +346,12 @@ class Context(object): # Look up a top-level module. Returns None if not found. return self.modules.get(name, None) - def find_submodule(self, name): + def find_submodule(self, name, as_package=False): # Find a top-level module, creating a new one if needed. scope = self.lookup_submodule(name) if not scope: scope = ModuleScope(name, - parent_module = None, context = self) + parent_module = None, context = self, is_package=as_package) self.modules[name] = scope return scope @@ -360,7 +361,7 @@ class Context(object): source_filename = source_desc.filename scope.cpp = self.cpp # Parse the given source file and return a parse tree. - num_errors = Errors.num_errors + num_errors = Errors.get_errors_count() try: with Utils.open_source_file(source_filename) as f: from . import Parsing @@ -379,7 +380,7 @@ class Context(object): #traceback.print_exc() raise self._report_decode_error(source_desc, e) - if Errors.num_errors > num_errors: + if Errors.get_errors_count() > num_errors: raise CompileError() return tree @@ -419,20 +420,19 @@ class Context(object): return ".".join(names) def setup_errors(self, options, result): - Errors.reset() # clear any remaining error state + Errors.init_thread() if options.use_listing_file: path = result.listing_file = Utils.replace_suffix(result.main_source_file, ".lis") else: path = None - Errors.open_listing_file(path=path, - echo_to_stderr=options.errors_to_stderr) + Errors.open_listing_file(path=path, echo_to_stderr=options.errors_to_stderr) def teardown_errors(self, err, options, result): source_desc = result.compilation_source.source_desc if not isinstance(source_desc, FileSourceDescriptor): raise RuntimeError("Only file sources for code supported") Errors.close_listing_file() - result.num_errors = Errors.num_errors + result.num_errors = Errors.get_errors_count() if result.num_errors > 0: err = True if err and result.c_file: @@ -473,22 +473,29 @@ def create_default_resultobj(compilation_source, options): def run_pipeline(source, options, full_module_name=None, context=None): from . import Pipeline + # ensure that the inputs are unicode (for Python 2) + if sys.version_info[0] == 2: + source = Utils.decode_filename(source) + if full_module_name: + full_module_name = Utils.decode_filename(full_module_name) + source_ext = os.path.splitext(source)[1] - options.configure_language_defaults(source_ext[1:]) # py/pyx + options.configure_language_defaults(source_ext[1:]) # py/pyx if context is None: - context = options.create_context() + context = Context.from_options(options) # Set up source object cwd = os.getcwd() abs_path = os.path.abspath(source) full_module_name = full_module_name or context.extract_module_name(source, options) + full_module_name = EncodedString(full_module_name) Utils.raise_error_if_module_name_forbidden(full_module_name) if options.relative_path_in_code_position_comments: rel_path = full_module_name.replace('.', os.sep) + source_ext if not abs_path.endswith(rel_path): - rel_path = source # safety measure to prevent printing incorrect paths + rel_path = source # safety measure to prevent printing incorrect paths else: rel_path = abs_path source_desc = FileSourceDescriptor(abs_path, rel_path) @@ -512,6 +519,12 @@ def run_pipeline(source, options, full_module_name=None, context=None): pipeline = Pipeline.create_pyx_pipeline(context, options, result) context.setup_errors(options, result) + + if '.' in full_module_name and '.' in os.path.splitext(os.path.basename(abs_path))[0]: + warning((source_desc, 1, 0), + "Dotted filenames ('%s') are deprecated." + " Please use the normal Python package directory layout." % os.path.basename(abs_path), level=1) + err, enddata = Pipeline.run_pipeline(pipeline, source) context.teardown_errors(err, options, result) if err is None and options.depfile: @@ -538,146 +551,6 @@ class CompilationSource(object): self.cwd = cwd -class CompilationOptions(object): - r""" - See default_options at the end of this module for a list of all possible - options and CmdLine.usage and CmdLine.parse_command_line() for their - meaning. - """ - def __init__(self, defaults=None, **kw): - self.include_path = [] - if defaults: - if isinstance(defaults, CompilationOptions): - defaults = defaults.__dict__ - else: - defaults = default_options - - options = dict(defaults) - options.update(kw) - - # let's assume 'default_options' contains a value for most known compiler options - # and validate against them - unknown_options = set(options) - set(default_options) - # ignore valid options that are not in the defaults - unknown_options.difference_update(['include_path']) - if unknown_options: - message = "got unknown compilation option%s, please remove: %s" % ( - 's' if len(unknown_options) > 1 else '', - ', '.join(unknown_options)) - raise ValueError(message) - - directive_defaults = Options.get_directive_defaults() - directives = dict(options['compiler_directives']) # copy mutable field - # check for invalid directives - unknown_directives = set(directives) - set(directive_defaults) - if unknown_directives: - message = "got unknown compiler directive%s: %s" % ( - 's' if len(unknown_directives) > 1 else '', - ', '.join(unknown_directives)) - raise ValueError(message) - options['compiler_directives'] = directives - if directives.get('np_pythran', False) and not options['cplus']: - import warnings - warnings.warn("C++ mode forced when in Pythran mode!") - options['cplus'] = True - if 'language_level' in directives and 'language_level' not in kw: - options['language_level'] = directives['language_level'] - elif not options.get('language_level'): - options['language_level'] = directive_defaults.get('language_level') - if 'formal_grammar' in directives and 'formal_grammar' not in kw: - options['formal_grammar'] = directives['formal_grammar'] - if options['cache'] is True: - options['cache'] = os.path.join(Utils.get_cython_cache_dir(), 'compiler') - - self.__dict__.update(options) - - def configure_language_defaults(self, source_extension): - if source_extension == 'py': - if self.compiler_directives.get('binding') is None: - self.compiler_directives['binding'] = True - - def create_context(self): - return Context(self.include_path, self.compiler_directives, - self.cplus, self.language_level, options=self) - - def get_fingerprint(self): - r""" - Return a string that contains all the options that are relevant for cache invalidation. - """ - # Collect only the data that can affect the generated file(s). - data = {} - - for key, value in self.__dict__.items(): - if key in ['show_version', 'errors_to_stderr', 'verbose', 'quiet']: - # verbosity flags have no influence on the compilation result - continue - elif key in ['output_file', 'output_dir']: - # ignore the exact name of the output file - continue - elif key in ['timestamps']: - # the cache cares about the content of files, not about the timestamps of sources - continue - elif key in ['cache']: - # hopefully caching has no influence on the compilation result - continue - elif key in ['compiler_directives']: - # directives passed on to the C compiler do not influence the generated C code - continue - elif key in ['include_path']: - # this path changes which headers are tracked as dependencies, - # it has no influence on the generated C code - continue - elif key in ['working_path']: - # this path changes where modules and pxd files are found; - # their content is part of the fingerprint anyway, their - # absolute path does not matter - continue - elif key in ['create_extension']: - # create_extension() has already mangled the options, e.g., - # embedded_metadata, when the fingerprint is computed so we - # ignore it here. - continue - elif key in ['build_dir']: - # the (temporary) directory where we collect dependencies - # has no influence on the C output - continue - elif key in ['use_listing_file', 'generate_pxi', 'annotate', 'annotate_coverage_xml']: - # all output files are contained in the cache so the types of - # files generated must be part of the fingerprint - data[key] = value - elif key in ['formal_grammar', 'evaluate_tree_assertions']: - # these bits can change whether compilation to C passes/fails - data[key] = value - elif key in ['embedded_metadata', 'emit_linenums', 'c_line_in_traceback', 'gdb_debug', 'relative_path_in_code_position_comments']: - # the generated code contains additional bits when these are set - data[key] = value - elif key in ['cplus', 'language_level', 'compile_time_env', 'np_pythran']: - # assorted bits that, e.g., influence the parser - data[key] = value - elif key == ['capi_reexport_cincludes']: - if self.capi_reexport_cincludes: - # our caching implementation does not yet include fingerprints of all the header files - raise NotImplementedError('capi_reexport_cincludes is not compatible with Cython caching') - elif key == ['common_utility_include_dir']: - if self.common_utility_include_dir: - raise NotImplementedError('common_utility_include_dir is not compatible with Cython caching yet') - else: - # any unexpected option should go into the fingerprint; it's better - # to recompile than to return incorrect results from the cache. - data[key] = value - - def to_fingerprint(item): - r""" - Recursively turn item into a string, turning dicts into lists with - deterministic ordering. - """ - if isinstance(item, dict): - item = sorted([(repr(key), to_fingerprint(value)) for key, value in item.items()]) - return repr(item) - - return to_fingerprint(data) - - class CompilationResult(object): """ Results from the Cython compiler: @@ -739,11 +612,11 @@ def compile_multiple(sources, options): a CompilationResultSet. Performs timestamp checking and/or recursion if these are specified in the options. """ - if options.module_name and len(sources) > 1: + if len(sources) > 1 and options.module_name: raise RuntimeError('Full module name can only be set ' 'for single source compilation') # run_pipeline creates the context - # context = options.create_context() + # context = Context.from_options(options) sources = [os.path.abspath(source) for source in sources] processed = set() results = CompilationResultSet() @@ -754,7 +627,7 @@ def compile_multiple(sources, options): for source in sources: if source not in processed: if context is None: - context = options.create_context() + context = Context.from_options(options) output_filename = get_output_filename(source, cwd, options) out_of_date = context.c_file_out_of_date(source, output_filename) if (not timestamps) or out_of_date: @@ -789,55 +662,79 @@ def compile(source, options = None, full_module_name = None, **kwds): @Utils.cached_function -def search_include_directories(dirs, qualified_name, suffix, pos, include=False): +def search_include_directories(dirs, qualified_name, suffix="", pos=None, include=False, source_file_path=None): """ Search the list of include directories for the given file name. - If a source file position is given, first searches the directory - containing that file. Returns None if not found, but does not - report an error. + If a source file path or position is given, first searches the directory + containing that file. Returns None if not found, but does not report an error. The 'include' option will disable package dereferencing. """ - - if pos: + if pos and not source_file_path: file_desc = pos[0] if not isinstance(file_desc, FileSourceDescriptor): raise RuntimeError("Only file sources for code supported") + source_file_path = file_desc.filename + if source_file_path: if include: - dirs = (os.path.dirname(file_desc.filename),) + dirs + dirs = (os.path.dirname(source_file_path),) + dirs else: - dirs = (Utils.find_root_package_dir(file_desc.filename),) + dirs + dirs = (Utils.find_root_package_dir(source_file_path),) + dirs + # search for dotted filename e.g. <dir>/foo.bar.pxd dotted_filename = qualified_name if suffix: dotted_filename += suffix - if not include: - names = qualified_name.split('.') - package_names = tuple(names[:-1]) - module_name = names[-1] - module_filename = module_name + suffix - package_filename = "__init__" + suffix - for dirname in dirs: path = os.path.join(dirname, dotted_filename) if os.path.exists(path): + if not include and '.' in qualified_name and '.' in os.path.splitext(dotted_filename)[0]: + warning(pos, "Dotted filenames ('%s') are deprecated." + " Please use the normal Python package directory layout." % dotted_filename, level=1) return path - if not include: - package_dir = Utils.check_package_dir(dirname, package_names) + # search for filename in package structure e.g. <dir>/foo/bar.pxd or <dir>/foo/bar/__init__.pxd + if not include: + + names = qualified_name.split('.') + package_names = tuple(names[:-1]) + module_name = names[-1] + + # search for standard packages first - PEP420 + namespace_dirs = [] + for dirname in dirs: + package_dir, is_namespace = Utils.check_package_dir(dirname, package_names) if package_dir is not None: - path = os.path.join(package_dir, module_filename) - if os.path.exists(path): - return path - path = os.path.join(package_dir, module_name, - package_filename) - if os.path.exists(path): + if is_namespace: + namespace_dirs.append(package_dir) + continue + path = search_module_in_dir(package_dir, module_name, suffix) + if path: return path + + # search for namespaces second - PEP420 + for package_dir in namespace_dirs: + path = search_module_in_dir(package_dir, module_name, suffix) + if path: + return path + return None +@Utils.cached_function +def search_module_in_dir(package_dir, module_name, suffix): + # matches modules of the form: <dir>/foo/bar.pxd + path = Utils.find_versioned_file(package_dir, module_name, suffix) + + # matches modules of the form: <dir>/foo/bar/__init__.pxd + if not path and suffix: + path = Utils.find_versioned_file(os.path.join(package_dir, module_name), "__init__", suffix) + + return path + + # ------------------------------------------------------------------------ # # Main command-line entry point @@ -852,14 +749,23 @@ def main(command_line = 0): args = sys.argv[1:] any_failures = 0 if command_line: - from .CmdLine import parse_command_line - options, sources = parse_command_line(args) + try: + options, sources = parse_command_line(args) + except IOError as e: + # TODO: IOError can be replaced with FileNotFoundError in Cython 3.1 + import errno + if errno.ENOENT != e.errno: + # Raised IOError is not caused by missing file. + raise + print("{}: No such file or directory: '{}'".format(sys.argv[0], e.filename), file=sys.stderr) + sys.exit(1) else: options = CompilationOptions(default_options) sources = args if options.show_version: - sys.stderr.write("Cython version %s\n" % version) + from .. import __version__ + sys.stderr.write("Cython version %s\n" % __version__) if options.working_path!="": os.chdir(options.working_path) try: @@ -871,44 +777,3 @@ def main(command_line = 0): any_failures = 1 if any_failures: sys.exit(1) - - -# ------------------------------------------------------------------------ -# -# Set the default options depending on the platform -# -# ------------------------------------------------------------------------ - -default_options = dict( - show_version = 0, - use_listing_file = 0, - errors_to_stderr = 1, - cplus = 0, - output_file = None, - depfile = None, - annotate = None, - annotate_coverage_xml = None, - generate_pxi = 0, - capi_reexport_cincludes = 0, - working_path = "", - timestamps = None, - verbose = 0, - quiet = 0, - compiler_directives = {}, - embedded_metadata = {}, - evaluate_tree_assertions = False, - emit_linenums = False, - relative_path_in_code_position_comments = True, - c_line_in_traceback = True, - language_level = None, # warn but default to 2 - formal_grammar = False, - gdb_debug = False, - compile_time_env = None, - common_utility_include_dir = None, - output_dir=None, - build_dir=None, - cache=None, - create_extension=None, - module_name=None, - np_pythran=False -) diff --git a/Cython/Compiler/MemoryView.py b/Cython/Compiler/MemoryView.py index 0406d6c71..5ebd396be 100644 --- a/Cython/Compiler/MemoryView.py +++ b/Cython/Compiler/MemoryView.py @@ -22,10 +22,6 @@ ERR_UNINITIALIZED = ("Cannot check if memoryview %s is initialized without the " "GIL, consider using initializedcheck(False)") -def concat_flags(*flags): - return "(%s)" % "|".join(flags) - - format_flag = "PyBUF_FORMAT" memview_c_contiguous = "(PyBUF_C_CONTIGUOUS | PyBUF_FORMAT)" @@ -100,8 +96,15 @@ def put_acquire_memoryviewslice(lhs_cname, lhs_type, lhs_pos, rhs, code, def put_assign_to_memviewslice(lhs_cname, rhs, rhs_cname, memviewslicetype, code, have_gil=False, first_assignment=False): + if lhs_cname == rhs_cname: + # self assignment is tricky because memoryview xdecref clears the memoryview + # thus invalidating both sides of the assignment. Therefore make it actually do nothing + code.putln("/* memoryview self assignment no-op */") + return + if not first_assignment: - code.put_xdecref_memoryviewslice(lhs_cname, have_gil=have_gil) + code.put_xdecref(lhs_cname, memviewslicetype, + have_gil=have_gil) if not rhs.result_in_temp(): rhs.make_owned_memoryviewslice(code) @@ -167,7 +170,7 @@ def valid_memslice_dtype(dtype, i=0): valid_memslice_dtype(dtype.base_type, i + 1)) or dtype.is_numeric or dtype.is_pyobject or - dtype.is_fused or # accept this as it will be replaced by specializations later + dtype.is_fused or # accept this as it will be replaced by specializations later (dtype.is_typedef and valid_memslice_dtype(dtype.typedef_base_type)) ) @@ -248,7 +251,7 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry): return bufp - def generate_buffer_slice_code(self, code, indices, dst, have_gil, + def generate_buffer_slice_code(self, code, indices, dst, dst_type, have_gil, have_slices, directives): """ Slice a memoryviewslice. @@ -265,7 +268,7 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry): code.putln("%(dst)s.data = %(src)s.data;" % locals()) code.putln("%(dst)s.memview = %(src)s.memview;" % locals()) - code.put_incref_memoryviewslice(dst) + code.put_incref_memoryviewslice(dst, dst_type, have_gil=have_gil) all_dimensions_direct = all(access == 'direct' for access, packing in self.type.axes) suboffset_dim_temp = [] @@ -292,7 +295,7 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry): dim += 1 access, packing = self.type.axes[dim] - if isinstance(index, ExprNodes.SliceNode): + if index.is_slice: # slice, unspecified dimension, or part of ellipsis d = dict(locals()) for s in "start stop step".split(): @@ -404,8 +407,8 @@ def get_is_contig_utility(contig_type, ndim): return utility -def slice_iter(slice_type, slice_result, ndim, code): - if slice_type.is_c_contig or slice_type.is_f_contig: +def slice_iter(slice_type, slice_result, ndim, code, force_strided=False): + if (slice_type.is_c_contig or slice_type.is_f_contig) and not force_strided: return ContigSliceIter(slice_type, slice_result, ndim, code) else: return StridedSliceIter(slice_type, slice_result, ndim, code) @@ -489,7 +492,7 @@ def copy_c_or_fortran_cname(memview): def get_copy_new_utility(pos, from_memview, to_memview): if (from_memview.dtype != to_memview.dtype and - not (from_memview.dtype.is_const and from_memview.dtype.const_base_type == to_memview.dtype)): + not (from_memview.dtype.is_cv_qualified and from_memview.dtype.cv_base_type == to_memview.dtype)): error(pos, "dtypes must be the same!") return if len(from_memview.axes) != len(to_memview.axes): @@ -507,7 +510,8 @@ def get_copy_new_utility(pos, from_memview, to_memview): if to_memview.is_c_contig: mode = 'c' contig_flag = memview_c_contiguous - elif to_memview.is_f_contig: + else: + assert to_memview.is_f_contig mode = 'fortran' contig_flag = memview_f_contiguous @@ -654,13 +658,13 @@ def is_cf_contig(specs): is_c_contig = True elif (specs[-1] == ('direct','contig') and - all(axis == ('direct','follow') for axis in specs[:-1])): + all(axis == ('direct','follow') for axis in specs[:-1])): # c_contiguous: 'follow', 'follow', ..., 'follow', 'contig' is_c_contig = True elif (len(specs) > 1 and - specs[0] == ('direct','contig') and - all(axis == ('direct','follow') for axis in specs[1:])): + specs[0] == ('direct','contig') and + all(axis == ('direct','follow') for axis in specs[1:])): # f_contiguous: 'contig', 'follow', 'follow', ..., 'follow' is_f_contig = True @@ -809,7 +813,8 @@ context = { 'memview_struct_name': memview_objstruct_cname, 'max_dims': Options.buffer_max_dims, 'memviewslice_name': memviewslice_cname, - 'memslice_init': memslice_entry_init, + 'memslice_init': PyrexTypes.MemoryViewSliceType.default_value, + 'THREAD_LOCKS_PREALLOCATED': 8, } memviewslice_declare_code = load_memview_c_utility( "MemviewSliceStruct", @@ -835,7 +840,7 @@ overlapping_utility = load_memview_c_utility("OverlappingSlices", context) copy_contents_new_utility = load_memview_c_utility( "MemviewSliceCopyTemplate", context, - requires=[], # require cython_array_utility_code + requires=[], # require cython_array_utility_code ) view_utility_code = load_memview_cy_utility( @@ -848,9 +853,9 @@ view_utility_code = load_memview_cy_utility( is_contig_utility, overlapping_utility, copy_contents_new_utility, - ModuleNode.capsule_utility_code], + ], ) -view_utility_whitelist = ('array', 'memoryview', 'array_cwrapper', +view_utility_allowlist = ('array', 'memoryview', 'array_cwrapper', 'generic', 'strided', 'indirect', 'contiguous', 'indirect_contiguous') diff --git a/Cython/Compiler/ModuleNode.py b/Cython/Compiler/ModuleNode.py index a2400bbe2..34ef35880 100644 --- a/Cython/Compiler/ModuleNode.py +++ b/Cython/Compiler/ModuleNode.py @@ -14,6 +14,7 @@ import json import operator import os import re +import sys from .PyrexTypes import CPtrType from . import Future @@ -26,13 +27,25 @@ from . import TypeSlots from . import PyrexTypes from . import Pythran -from .Errors import error, warning +from .Errors import error, warning, CompileError from .PyrexTypes import py_object_type -from ..Utils import open_new_file, replace_suffix, decode_filename, build_hex_version -from .Code import UtilityCode, IncludeCode -from .StringEncoding import EncodedString +from ..Utils import open_new_file, replace_suffix, decode_filename, build_hex_version, is_cython_generated_file +from .Code import UtilityCode, IncludeCode, TempitaUtilityCode +from .StringEncoding import EncodedString, encoded_string_or_bytes_literal from .Pythran import has_np_pythran + +def replace_suffix_encoded(path, newsuf): + # calls replace suffix and returns a EncodedString or BytesLiteral with the encoding set + newpath = replace_suffix(path, newsuf) + return as_encoded_filename(newpath) + +def as_encoded_filename(path): + # wraps the path with either EncodedString or BytesLiteral (depending on its input type) + # and sets the encoding to the file system encoding + return encoded_string_or_bytes_literal(path, sys.getfilesystemencoding()) + + def check_c_declarations_pxd(module_node): module_node.scope.check_c_classes_pxd() return module_node @@ -50,11 +63,50 @@ def generate_c_code_config(env, options): else: emit_linenums = options.emit_linenums + if hasattr(options, "emit_code_comments"): + print('Warning: option emit_code_comments is deprecated. ' + 'Instead, use compiler directive emit_code_comments.') + return Code.CCodeConfig( emit_linenums=emit_linenums, emit_code_comments=env.directives['emit_code_comments'], c_line_in_traceback=options.c_line_in_traceback) +# The code required to generate one comparison from another. +# The keys are (from, to). +# The comparison operator always goes first, with equality possibly second. +# The first value specifies if the comparison is inverted. The second is the +# logic op to use, and the third is if the equality is inverted or not. +TOTAL_ORDERING = { + # a > b from (not a < b) and (a != b) + ('__lt__', '__gt__'): (True, '&&', True), + # a <= b from (a < b) or (a == b) + ('__lt__', '__le__'): (False, '||', False), + # a >= b from (not a < b). + ('__lt__', '__ge__'): (True, '', None), + + # a >= b from (not a <= b) or (a == b) + ('__le__', '__ge__'): (True, '||', False), + # a < b, from (a <= b) and (a != b) + ('__le__', '__lt__'): (False, '&&', True), + # a > b from (not a <= b) + ('__le__', '__gt__'): (True, '', None), + + # a < b from (not a > b) and (a != b) + ('__gt__', '__lt__'): (True, '&&', True), + # a >= b from (a > b) or (a == b) + ('__gt__', '__ge__'): (False, '||', False), + # a <= b from (not a > b) + ('__gt__', '__le__'): (True, '', None), + + # Return a <= b from (not a >= b) or (a == b) + ('__ge__', '__le__'): (True, '||', False), + # a > b from (a >= b) and (a != b) + ('__ge__', '__gt__'): (False, '&&', True), + # a < b from (not a >= b) + ('__ge__', '__lt__'): (True, '', None), +} + class ModuleNode(Nodes.Node, Nodes.BlockNode): # doc string or None @@ -69,21 +121,41 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): child_attrs = ["body"] directives = None + # internal - used in merging + pxd_stats = None + utility_code_stats = None - def merge_in(self, tree, scope, merge_scope=False): + + def merge_in(self, tree, scope, stage, merge_scope=False): # Merges in the contents of another tree, and possibly scope. With the # current implementation below, this must be done right prior # to code generation. + # Stage is one of "pxd" or "utility" to indicate pxd file or utility + # code. This helps define the order. # # Note: This way of doing it seems strange -- I believe the # right concept is to split ModuleNode into a ModuleNode and a # CodeGenerator, and tell that CodeGenerator to generate code # from multiple sources. assert isinstance(self.body, Nodes.StatListNode) + assert stage in ('pxd', 'utility') + + if self.pxd_stats is None: + self.pxd_stats = Nodes.StatListNode(self.body.pos, stats=[]) + self.utility_code_stats = Nodes.StatListNode(self.body.pos, stats=[]) + self.body.stats.insert(0, self.pxd_stats) + self.body.stats.insert(0, self.utility_code_stats) + + if scope.directives != self.scope.directives: + # merged in nodes should keep their original compiler directives + # (for example inline cdef functions) + tree = Nodes.CompilerDirectivesNode(tree.pos, body=tree, directives=scope.directives) + + target_stats = self.pxd_stats if stage == "pxd" else self.utility_code_stats if isinstance(tree, Nodes.StatListNode): - self.body.stats.extend(tree.stats) + target_stats.stats.extend(tree.stats) else: - self.body.stats.append(tree) + target_stats.stats.append(tree) self.scope.utility_code_list.extend(scope.utility_code_list) @@ -105,6 +177,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.scope.merge_in(scope) + def with_compiler_directives(self): + # When merging a utility code module into the user code we need to preserve + # the original compiler directives. This returns the body of the module node, + # wrapped in its set of directives. + body = Nodes.CompilerDirectivesNode(self.pos, directives=self.directives, body=self.body) + return body + def analyse_declarations(self, env): if has_np_pythran(env): Pythran.include_pythran_generic(env) @@ -131,8 +210,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.create_import_star_conversion_utility_code(env) for name, entry in sorted(env.entries.items()): if (entry.create_wrapper and entry.scope is env - and entry.is_type and entry.type.is_enum): - entry.type.create_type_wrapper(env) + and entry.is_type and (entry.type.is_enum or entry.type.is_cpp_enum)): + entry.type.create_type_wrapper(env) def process_implementation(self, options, result): env = self.scope @@ -151,6 +230,14 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): return 1 return 0 + def assure_safe_target(self, path, allow_failed=False): + # Check for a common gotcha for new users: naming your .pyx file after the .c file you want to wrap + if not is_cython_generated_file(path, allow_failed=allow_failed, if_not_found=True): + # Raising a fatal CompileError instead of calling error() to prevent castrating an existing file. + raise CompileError( + self.pos, 'The output file already exists and does not look like it was generated by Cython: "%s"' % + os.path.basename(path)) + def generate_h_code(self, env, options, result): def h_entries(entries, api=0, pxd=0): return [entry for entry in entries @@ -161,65 +248,99 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): h_vars = h_entries(env.var_entries) h_funcs = h_entries(env.cfunc_entries) h_extension_types = h_entries(env.c_class_entries) - if h_types or h_vars or h_funcs or h_extension_types: - result.h_file = replace_suffix(result.c_file, ".h") - h_code = Code.CCodeWriter() + + if h_types or h_vars or h_funcs or h_extension_types: + result.h_file = replace_suffix_encoded(result.c_file, ".h") + self.assure_safe_target(result.h_file) + + h_code_writer = Code.CCodeWriter() c_code_config = generate_c_code_config(env, options) - Code.GlobalState(h_code, self, c_code_config) + globalstate = Code.GlobalState(h_code_writer, self, c_code_config) + globalstate.initialize_main_h_code() # in-case utility code is used in the header + h_code_start = globalstate.parts['h_code'] + h_code_main = globalstate.parts['type_declarations'] + h_code_end = globalstate.parts['end'] if options.generate_pxi: - result.i_file = replace_suffix(result.c_file, ".pxi") + result.i_file = replace_suffix_encoded(result.c_file, ".pxi") i_code = Code.PyrexCodeWriter(result.i_file) else: i_code = None - h_code.put_generated_by() - h_guard = Naming.h_guard_prefix + self.api_name(env) - h_code.put_h_guard(h_guard) - h_code.putln("") - h_code.putln('#include "Python.h"') - self.generate_type_header_code(h_types, h_code) + h_code_start.put_generated_by() + h_guard = self.api_name(Naming.h_guard_prefix, env) + h_code_start.put_h_guard(h_guard) + h_code_start.putln("") + h_code_start.putln('#include "Python.h"') + self.generate_type_header_code(h_types, h_code_start) if options.capi_reexport_cincludes: - self.generate_includes(env, [], h_code) - h_code.putln("") - api_guard = Naming.api_guard_prefix + self.api_name(env) - h_code.putln("#ifndef %s" % api_guard) - h_code.putln("") - self.generate_extern_c_macro_definition(h_code) - h_code.putln("") - self.generate_dl_import_macro(h_code) + self.generate_includes(env, [], h_code_start) + h_code_start.putln("") + api_guard = self.api_name(Naming.api_guard_prefix, env) + h_code_start.putln("#ifndef %s" % api_guard) + h_code_start.putln("") + self.generate_extern_c_macro_definition(h_code_start, env.is_cpp()) + h_code_start.putln("") + self.generate_dl_import_macro(h_code_start) if h_extension_types: - h_code.putln("") + h_code_main.putln("") for entry in h_extension_types: - self.generate_cclass_header_code(entry.type, h_code) + self.generate_cclass_header_code(entry.type, h_code_main) if i_code: self.generate_cclass_include_code(entry.type, i_code) if h_funcs: - h_code.putln("") + h_code_main.putln("") for entry in h_funcs: - self.generate_public_declaration(entry, h_code, i_code) + self.generate_public_declaration(entry, h_code_main, i_code) if h_vars: - h_code.putln("") + h_code_main.putln("") for entry in h_vars: - self.generate_public_declaration(entry, h_code, i_code) - h_code.putln("") - h_code.putln("#endif /* !%s */" % api_guard) - h_code.putln("") - h_code.putln("/* WARNING: the interface of the module init function changed in CPython 3.5. */") - h_code.putln("/* It now returns a PyModuleDef instance instead of a PyModule instance. */") - h_code.putln("") - h_code.putln("#if PY_MAJOR_VERSION < 3") - h_code.putln("PyMODINIT_FUNC init%s(void);" % env.module_name) - h_code.putln("#else") - h_code.putln("PyMODINIT_FUNC %s(void);" % self.mod_init_func_cname('PyInit', env)) - h_code.putln("#endif") - h_code.putln("") - h_code.putln("#endif /* !%s */" % h_guard) - - f = open_new_file(result.h_file) - try: - h_code.copyto(f) - finally: - f.close() + self.generate_public_declaration(entry, h_code_main, i_code) + h_code_main.putln("") + h_code_main.putln("#endif /* !%s */" % api_guard) + h_code_main.putln("") + h_code_main.putln("/* WARNING: the interface of the module init function changed in CPython 3.5. */") + h_code_main.putln("/* It now returns a PyModuleDef instance instead of a PyModule instance. */") + h_code_main.putln("") + h_code_main.putln("#if PY_MAJOR_VERSION < 3") + if env.module_name.isascii(): + py2_mod_name = env.module_name + else: + py2_mod_name = env.module_name.encode("ascii", errors="ignore").decode("utf-8") + h_code_main.putln('#error "Unicode module names are not supported in Python 2";') + h_code_main.putln("PyMODINIT_FUNC init%s(void);" % py2_mod_name) + h_code_main.putln("#else") + py3_mod_func_name = self.mod_init_func_cname('PyInit', env) + warning_string = EncodedString('Use PyImport_AppendInittab("%s", %s) instead of calling %s directly.' % ( + py2_mod_name, py3_mod_func_name, py3_mod_func_name)) + h_code_main.putln('/* WARNING: %s from Python 3.5 */' % warning_string.rstrip('.')) + h_code_main.putln("PyMODINIT_FUNC %s(void);" % py3_mod_func_name) + h_code_main.putln("") + h_code_main.putln("#if PY_VERSION_HEX >= 0x03050000 " + "&& (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER) " + "|| (defined(__cplusplus) && __cplusplus >= 201402L))") + h_code_main.putln("#if defined(__cplusplus) && __cplusplus >= 201402L") + h_code_main.putln("[[deprecated(%s)]] inline" % warning_string.as_c_string_literal()) + h_code_main.putln("#elif defined(__GNUC__) || defined(__clang__)") + h_code_main.putln('__attribute__ ((__deprecated__(%s), __unused__)) __inline__' % ( + warning_string.as_c_string_literal())) + h_code_main.putln("#elif defined(_MSC_VER)") + h_code_main.putln('__declspec(deprecated(%s)) __inline' % ( + warning_string.as_c_string_literal())) + h_code_main.putln('#endif') + h_code_main.putln("static PyObject* __PYX_WARN_IF_%s_INIT_CALLED(PyObject* res) {" % py3_mod_func_name) + h_code_main.putln("return res;") + h_code_main.putln("}") + # Function call is converted to warning macro; uncalled (pointer) is not + h_code_main.putln('#define %s() __PYX_WARN_IF_%s_INIT_CALLED(%s())' % ( + py3_mod_func_name, py3_mod_func_name, py3_mod_func_name)) + h_code_main.putln('#endif') + h_code_main.putln('#endif') + + h_code_end.putln("") + h_code_end.putln("#endif /* !%s */" % h_guard) + + with open_new_file(result.h_file) as f: + h_code_writer.copyto(f) def generate_public_declaration(self, entry, h_code, i_code): h_code.putln("%s %s;" % ( @@ -229,8 +350,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): i_code.putln("cdef extern %s" % ( entry.type.declaration_code(entry.cname, pyrex=1))) - def api_name(self, env): - return env.qualified_name.replace(".", "__") + def api_name(self, prefix, env): + api_name = self.punycode_module_name(prefix, env.qualified_name) + return api_name.replace(".", "__") def generate_api_code(self, env, options, result): def api_entries(entries, pxd=0): @@ -239,13 +361,16 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): api_vars = api_entries(env.var_entries) api_funcs = api_entries(env.cfunc_entries) api_extension_types = api_entries(env.c_class_entries) + if api_vars or api_funcs or api_extension_types: - result.api_file = replace_suffix(result.c_file, "_api.h") + result.api_file = replace_suffix_encoded(result.c_file, "_api.h") + self.assure_safe_target(result.api_file) + h_code = Code.CCodeWriter() c_code_config = generate_c_code_config(env, options) Code.GlobalState(h_code, self, c_code_config) h_code.put_generated_by() - api_guard = Naming.api_guard_prefix + self.api_name(env) + api_guard = self.api_name(Naming.api_guard_prefix, env) h_code.put_h_guard(api_guard) # Work around https://bugs.python.org/issue4709 h_code.putln('#ifdef __MINGW64__') @@ -254,7 +379,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): h_code.putln('#include "Python.h"') if result.h_file: - h_code.putln('#include "%s"' % os.path.basename(result.h_file)) + h_filename = os.path.basename(result.h_file) + h_filename = as_encoded_filename(h_filename) + h_code.putln('#include %s' % h_filename.as_c_string_literal()) if api_extension_types: h_code.putln("") for entry in api_extension_types: @@ -274,9 +401,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): for entry in api_vars: type = CPtrType(entry.type) cname = env.mangle(Naming.varptr_prefix_api, entry.name) - h_code.putln("static %s = 0;" % type.declaration_code(cname)) + h_code.putln("static %s = 0;" % type.declaration_code(cname)) h_code.putln("#define %s (*%s)" % (entry.name, cname)) - h_code.put(UtilityCode.load_as_string("PyIdentifierFromString", "ImportExport.c")[0]) if api_vars: h_code.put(UtilityCode.load_as_string("VoidPtrImport", "ImportExport.c")[1]) if api_funcs: @@ -285,22 +411,22 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): h_code.put(UtilityCode.load_as_string("TypeImport", "ImportExport.c")[0]) h_code.put(UtilityCode.load_as_string("TypeImport", "ImportExport.c")[1]) h_code.putln("") - h_code.putln("static int import_%s(void) {" % self.api_name(env)) + h_code.putln("static int %s(void) {" % self.api_name("import", env)) h_code.putln("PyObject *module = 0;") - h_code.putln('module = PyImport_ImportModule("%s");' % env.qualified_name) + h_code.putln('module = PyImport_ImportModule(%s);' % env.qualified_name.as_c_string_literal()) h_code.putln("if (!module) goto bad;") for entry in api_funcs: cname = env.mangle(Naming.func_prefix_api, entry.name) sig = entry.type.signature_string() h_code.putln( - 'if (__Pyx_ImportFunction_%s(module, "%s", (void (**)(void))&%s, "%s") < 0) goto bad;' - % (Naming.cyversion, entry.name, cname, sig)) + 'if (__Pyx_ImportFunction_%s(module, %s, (void (**)(void))&%s, "%s") < 0) goto bad;' + % (Naming.cyversion, entry.name.as_c_string_literal(), cname, sig)) for entry in api_vars: cname = env.mangle(Naming.varptr_prefix_api, entry.name) sig = entry.type.empty_declaration_code() h_code.putln( - 'if (__Pyx_ImportVoidPtr_%s(module, "%s", (void **)&%s, "%s") < 0) goto bad;' - % (Naming.cyversion, entry.name, cname, sig)) + 'if (__Pyx_ImportVoidPtr_%s(module, %s, (void **)&%s, "%s") < 0) goto bad;' + % (Naming.cyversion, entry.name.as_c_string_literal(), cname, sig)) with ModuleImportGenerator(h_code, imported_modules={env.qualified_name: 'module'}) as import_generator: for entry in api_extension_types: self.generate_type_import_call(entry.type, h_code, import_generator, error_code="goto bad;") @@ -339,10 +465,15 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): i_code.dedent() def generate_c_code(self, env, options, result): + self.assure_safe_target(result.c_file, allow_failed=True) modules = self.referenced_modules if Options.annotate or options.annotate: - rootwriter = Annotate.AnnotationCCodeWriter() + show_entire_c_code = Options.annotate == "fullc" or options.annotate == "fullc" + rootwriter = Annotate.AnnotationCCodeWriter( + show_entire_c_code=show_entire_c_code, + source_desc=self.compilation_source.source_desc, + ) else: rootwriter = Code.CCodeWriter() @@ -364,24 +495,24 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): globalstate.use_utility_code(refnanny_utility_code) code = globalstate['before_global_var'] - code.putln('#define __Pyx_MODULE_NAME "%s"' % self.full_module_name) - module_is_main = "%s%s" % (Naming.module_is_main, self.full_module_name.replace('.', '__')) + code.putln('#define __Pyx_MODULE_NAME %s' % + self.full_module_name.as_c_string_literal()) + module_is_main = self.is_main_module_flag_cname() code.putln("extern int %s;" % module_is_main) code.putln("int %s = 0;" % module_is_main) code.putln("") - code.putln("/* Implementation of '%s' */" % env.qualified_name) + code.putln("/* Implementation of %s */" % env.qualified_name.as_c_string_literal()) code = globalstate['late_includes'] - code.putln("/* Late includes */") self.generate_includes(env, modules, code, early=False) - code = globalstate['all_the_rest'] + code = globalstate['module_code'] self.generate_cached_builtins_decls(env, code) - self.generate_lambda_definitions(env, code) + # generate normal variable and function definitions + self.generate_lambda_definitions(env, code) self.generate_variable_definitions(env, code) - self.body.generate_function_definitions(env, code) code.mark_pos(None) @@ -389,11 +520,15 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.generate_method_table(env, code) if env.has_import_star: self.generate_import_star(env, code) - self.generate_pymoduledef_struct(env, code) # initialise the macro to reduce the code size of one-time functionality code.putln(UtilityCode.load_as_string("SmallCodeConfig", "ModuleSetupCode.c")[0].strip()) + self.generate_module_state_start(env, globalstate['module_state']) + self.generate_module_state_defines(env, globalstate['module_state_defines']) + self.generate_module_state_clear(env, globalstate['module_state_clear']) + self.generate_module_state_traverse(env, globalstate['module_state_traverse']) + # init_globals is inserted before this self.generate_module_init_func(modules[:-1], env, globalstate['init_module']) self.generate_module_cleanup_func(env, globalstate['cleanup_module']) @@ -408,6 +543,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): globalstate.use_utility_code(utilcode) globalstate.finalize_main_c_code() + self.generate_module_state_end(env, modules, globalstate) + f = open_new_file(result.c_file) try: rootwriter.copyto(f) @@ -429,11 +566,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): except ImportError: import xml.etree.ElementTree as ET coverage_xml = ET.parse(coverage_xml_filename).getroot() - if hasattr(coverage_xml, 'iter'): - iterator = coverage_xml.iter() # Python 2.7 & 3.2+ - else: - iterator = coverage_xml.getiterator() - for el in iterator: + for el in coverage_xml.iter(): el.tail = None # save some memory else: coverage_xml = None @@ -452,7 +585,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if not target_file_dir.startswith(target_dir): # any other directories may not be writable => avoid trying continue - source_file = search_include_file(included_file, "", self.pos, include=True) + source_file = search_include_file(included_file, source_pos=self.pos, include=True) if not source_file: continue if target_file_dir != target_dir and not os.path.exists(target_file_dir): @@ -469,16 +602,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): markers = ccodewriter.buffer.allmarkers() d = defaultdict(list) - for c_lineno, cython_lineno in enumerate(markers): - if cython_lineno > 0: - d[cython_lineno].append(c_lineno + 1) + for c_lineno, (src_desc, src_lineno) in enumerate(markers): + if src_lineno > 0 and src_desc.filename is not None: + d[src_desc, src_lineno].append(c_lineno + 1) tb.start('LineNumberMapping') - for cython_lineno, c_linenos in sorted(d.items()): + for (src_desc, src_lineno), c_linenos in sorted(d.items()): + assert src_desc.filename is not None tb.add_entry( 'LineNumber', c_linenos=' '.join(map(str, c_linenos)), - cython_lineno=str(cython_lineno), + src_path=src_desc.filename, + src_lineno=str(src_lineno), ) tb.end('LineNumberMapping') tb.serialize() @@ -491,33 +626,36 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): module_list.append(env) def sort_types_by_inheritance(self, type_dict, type_order, getkey): - # copy the types into a list moving each parent type before - # its first child - type_list = [] - for i, key in enumerate(type_order): + subclasses = defaultdict(list) # maps type key to list of subclass keys + for key in type_order: new_entry = type_dict[key] - # collect all base classes to check for children - hierarchy = set() - base = new_entry + base = new_entry.type.base_type while base: - base_type = base.type.base_type - if not base_type: - break - base_key = getkey(base_type) - hierarchy.add(base_key) - base = type_dict.get(base_key) - new_entry.base_keys = hierarchy - - # find the first (sub-)subclass and insert before that - for j in range(i): - entry = type_list[j] - if key in entry.base_keys: - type_list.insert(j, new_entry) + base_key = getkey(base) + subclasses[base_key].append(key) + base_entry = type_dict.get(base_key) + if base_entry is None: break - else: - type_list.append(new_entry) - return type_list + base = base_entry.type.base_type + + # Simple topological sort using recursive DFS, based on + # https://en.wikipedia.org/wiki/Topological_sorting#Depth-first_search + seen = set() + result = [] + def dfs(u): + if u in seen: + return + seen.add(u) + for v in subclasses[getkey(u.type)]: + dfs(type_dict[v]) + result.append(u) + + for key in reversed(type_order): + dfs(type_dict[key]) + + result.reverse() + return result def sort_type_hierarchy(self, module_list, env): # poor developer's OrderedDict @@ -619,12 +757,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): for module in modules: defined_here = module is env modulecode.putln("") - modulecode.putln("/* Module declarations from '%s' */" % module.qualified_name) - self.generate_c_class_declarations(module, modulecode, defined_here) + modulecode.putln("/* Module declarations from %s */" % module.qualified_name.as_c_string_literal()) + self.generate_c_class_declarations(module, modulecode, defined_here, globalstate) self.generate_cvariable_declarations(module, modulecode, defined_here) self.generate_cfunction_declarations(module, modulecode, defined_here) - def _put_setup_code(self, code, name): + @staticmethod + def _put_setup_code(code, name): code.put(UtilityCode.load_as_string(name, "ModuleSetupCode.c")[1]) def generate_module_preamble(self, env, options, cimported_modules, metadata, code): @@ -638,6 +777,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#ifndef PY_SSIZE_T_CLEAN") code.putln("#define PY_SSIZE_T_CLEAN") code.putln("#endif /* PY_SSIZE_T_CLEAN */") + self._put_setup_code(code, "InitLimitedAPI") for inc in sorted(env.c_includes.values(), key=IncludeCode.sortkey): if inc.location == inc.INITIAL: @@ -645,14 +785,16 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#ifndef Py_PYTHON_H") code.putln(" #error Python headers needed to compile C extensions, " "please install development version of Python.") - code.putln("#elif PY_VERSION_HEX < 0x02060000 || " + code.putln("#elif PY_VERSION_HEX < 0x02070000 || " "(0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03030000)") - code.putln(" #error Cython requires Python 2.6+ or Python 3.3+.") + code.putln(" #error Cython requires Python 2.7+ or Python 3.3+.") code.putln("#else") code.globalstate["end"].putln("#endif /* Py_PYTHON_H */") from .. import __version__ code.putln('#define CYTHON_ABI "%s"' % __version__.replace('.', '_')) + code.putln('#define __PYX_ABI_MODULE_NAME "_cython_" CYTHON_ABI') + code.putln('#define __PYX_TYPE_MODULE_PREFIX __PYX_ABI_MODULE_NAME "."') code.putln('#define CYTHON_HEX_VERSION %s' % build_hex_version(__version__)) code.putln("#define CYTHON_FUTURE_DIVISION %d" % ( Future.division in env.context.future_directives)) @@ -680,11 +822,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln(" { __PYX_MARK_ERR_POS(f_index, lineno) goto Ln_error; }") code.putln("") - self.generate_extern_c_macro_definition(code) + self.generate_extern_c_macro_definition(code, env.is_cpp()) code.putln("") - code.putln("#define %s" % Naming.h_guard_prefix + self.api_name(env)) - code.putln("#define %s" % Naming.api_guard_prefix + self.api_name(env)) + code.putln("#define %s" % self.api_name(Naming.h_guard_prefix, env)) + code.putln("#define %s" % self.api_name(Naming.api_guard_prefix, env)) code.putln("/* Early includes */") self.generate_includes(env, cimported_modules, code, late=False) code.putln("") @@ -721,6 +863,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln('#define __Pyx_PyObject_FromString __Pyx_Py%s_FromString' % c_string_func_name) code.putln('#define __Pyx_PyObject_FromStringAndSize __Pyx_Py%s_FromStringAndSize' % c_string_func_name) code.put(UtilityCode.load_as_string("TypeConversions", "TypeConversion.c")[0]) + env.use_utility_code(UtilityCode.load_cached("FormatTypeName", "ObjectHandling.c")) # These utility functions are assumed to exist and used elsewhere. PyrexTypes.c_long_type.create_to_py_utility_code(env) @@ -730,32 +873,42 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.put(Nodes.branch_prediction_macros) code.putln('static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; }') code.putln('') + code.putln('#if !CYTHON_USE_MODULE_STATE') code.putln('static PyObject *%s = NULL;' % env.module_cname) - code.putln('static PyObject *%s;' % env.module_dict_cname) - code.putln('static PyObject *%s;' % Naming.builtins_cname) - code.putln('static PyObject *%s = NULL;' % Naming.cython_runtime_cname) - code.putln('static PyObject *%s;' % Naming.empty_tuple) - code.putln('static PyObject *%s;' % Naming.empty_bytes) - code.putln('static PyObject *%s;' % Naming.empty_unicode) if Options.pre_import is not None: code.putln('static PyObject *%s;' % Naming.preimport_cname) + code.putln('#endif') + code.putln('static int %s;' % Naming.lineno_cname) code.putln('static int %s = 0;' % Naming.clineno_cname) - code.putln('static const char * %s= %s;' % (Naming.cfilenm_cname, Naming.file_c_macro)) + code.putln('static const char * %s = %s;' % (Naming.cfilenm_cname, Naming.file_c_macro)) code.putln('static const char *%s;' % Naming.filename_cname) env.use_utility_code(UtilityCode.load_cached("FastTypeChecks", "ModuleSetupCode.c")) if has_np_pythran(env): env.use_utility_code(UtilityCode.load_cached("PythranConversion", "CppSupport.cpp")) - def generate_extern_c_macro_definition(self, code): + def generate_extern_c_macro_definition(self, code, is_cpp): name = Naming.extern_c_macro - code.putln("#ifndef %s" % name) - code.putln(" #ifdef __cplusplus") - code.putln(' #define %s extern "C"' % name) - code.putln(" #else") - code.putln(" #define %s extern" % name) - code.putln(" #endif") + code.putln("#ifdef CYTHON_EXTERN_C") + # make sure that user overrides always take precedence + code.putln(' #undef %s' % name) + code.putln(' #define %s CYTHON_EXTERN_C' % name) + code.putln("#elif defined(%s)" % name) + code.putln(" #ifdef _MSC_VER") + code.putln(" #pragma message (\"Please do not define the '%s' macro externally. Use 'CYTHON_EXTERN_C' instead.\")" % name) + code.putln(" #else") + code.putln(" #warning Please do not define the '%s' macro externally. Use 'CYTHON_EXTERN_C' instead." % name) + code.putln(" #endif") + code.putln("#else") + if is_cpp: + code.putln(' #define %s extern "C++"' % name) + else: + code.putln(" #ifdef __cplusplus") + code.putln(' #define %s extern "C"' % name) + code.putln(" #else") + code.putln(" #define %s extern" % name) + code.putln(" #endif") code.putln("#endif") def generate_dl_import_macro(self, code): @@ -764,7 +917,6 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#endif") def generate_includes(self, env, cimported_modules, code, early=True, late=True): - includes = [] for inc in sorted(env.c_includes.values(), key=IncludeCode.sortkey): if inc.location == inc.EARLY: if early: @@ -785,7 +937,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if isabs(file_path): file_path = basename(file_path) # never include absolute paths escaped_filename = file_path.replace("\\", "\\\\").replace('"', r'\"') - code.putln('"%s",' % escaped_filename) + escaped_filename = as_encoded_filename(escaped_filename) + code.putln('%s,' % escaped_filename.as_c_string_literal()) else: # Some C compilers don't like an empty array code.putln("0") @@ -802,7 +955,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if not entry.in_cinclude: #print "generate_type_header_code:", entry.name, repr(entry.type) ### type = entry.type - if type.is_typedef: # Must test this first! + if type.is_typedef: # Must test this first! pass elif type.is_struct_or_union or type.is_cpp_class: self.generate_struct_union_predeclaration(entry, code) @@ -815,9 +968,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if not entry.in_cinclude: #print "generate_type_header_code:", entry.name, repr(entry.type) ### type = entry.type - if type.is_typedef: # Must test this first! + if type.is_typedef: # Must test this first! self.generate_typedef(entry, code) - elif type.is_enum: + elif type.is_enum or type.is_cpp_enum: self.generate_enum_definition(entry, code) elif type.is_struct_or_union: self.generate_struct_union_definition(entry, code) @@ -844,7 +997,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): def generate_typedef(self, entry, code): base_type = entry.type.typedef_base_type - if base_type.is_numeric: + enclosing_scope = entry.scope + if base_type.is_numeric and not enclosing_scope.is_cpp_class_scope: try: writer = code.globalstate['numeric_typedefs'] except KeyError: @@ -894,8 +1048,6 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#endif") code.putln(header) var_entries = scope.var_entries - if not var_entries: - error(entry.pos, "Empty struct or union definition not allowed outside a 'cdef extern from' block") for attr in var_entries: code.putln( "%s;" % attr.type.declaration_code(attr.cname)) @@ -922,6 +1074,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): [base_class.empty_declaration_code() for base_class in type.base_classes]) code.put(" : public %s" % base_class_decl) code.putln(" {") + self.generate_type_header_code(scope.type_entries, code) py_attrs = [e for e in scope.entries.values() if e.type.is_pyobject and not e.is_inherited] has_virtual_methods = False @@ -956,67 +1109,69 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): arg_decls = ["void"] arg_names = [] if is_implementing: - code.putln("%s(%s) {" % (type.cname, ", ".join(arg_decls))) - if py_attrs: - code.put_ensure_gil() - for attr in py_attrs: - code.put_init_var_to_py_none(attr, nanny=False); - if constructor: - code.putln("%s(%s);" % (constructor.cname, ", ".join(arg_names))) - if py_attrs: - code.put_release_ensured_gil() - code.putln("}") + code.putln("%s(%s) {" % (type.cname, ", ".join(arg_decls))) + if py_attrs: + code.put_ensure_gil() + for attr in py_attrs: + code.put_init_var_to_py_none(attr, nanny=False) + if constructor: + code.putln("%s(%s);" % (constructor.cname, ", ".join(arg_names))) + if py_attrs: + code.put_release_ensured_gil() + code.putln("}") else: - code.putln("%s(%s);" % (type.cname, ", ".join(arg_decls))) + code.putln("%s(%s);" % (type.cname, ", ".join(arg_decls))) if destructor or py_attrs or has_virtual_methods: if has_virtual_methods: code.put("virtual ") if is_implementing: - code.putln("~%s() {" % type.cname) - if py_attrs: - code.put_ensure_gil() - if destructor: - code.putln("%s();" % destructor.cname) - if py_attrs: - for attr in py_attrs: - code.put_var_xdecref(attr, nanny=False); - code.put_release_ensured_gil() - code.putln("}") + code.putln("~%s() {" % type.cname) + if py_attrs: + code.put_ensure_gil() + if destructor: + code.putln("%s();" % destructor.cname) + if py_attrs: + for attr in py_attrs: + code.put_var_xdecref(attr, nanny=False) + code.put_release_ensured_gil() + code.putln("}") else: - code.putln("~%s();" % type.cname) + code.putln("~%s();" % type.cname) if py_attrs: # Also need copy constructor and assignment operators. if is_implementing: - code.putln("%s(const %s& __Pyx_other) {" % (type.cname, type.cname)) - code.put_ensure_gil() - for attr in scope.var_entries: - if not attr.type.is_cfunction: - code.putln("%s = __Pyx_other.%s;" % (attr.cname, attr.cname)) - code.put_var_incref(attr, nanny=False) - code.put_release_ensured_gil() - code.putln("}") - code.putln("%s& operator=(const %s& __Pyx_other) {" % (type.cname, type.cname)) - code.putln("if (this != &__Pyx_other) {") - code.put_ensure_gil() - for attr in scope.var_entries: - if not attr.type.is_cfunction: - code.put_var_xdecref(attr, nanny=False); - code.putln("%s = __Pyx_other.%s;" % (attr.cname, attr.cname)) - code.put_var_incref(attr, nanny=False) - code.put_release_ensured_gil() - code.putln("}") - code.putln("return *this;") - code.putln("}") + code.putln("%s(const %s& __Pyx_other) {" % (type.cname, type.cname)) + code.put_ensure_gil() + for attr in scope.var_entries: + if not attr.type.is_cfunction: + code.putln("%s = __Pyx_other.%s;" % (attr.cname, attr.cname)) + code.put_var_incref(attr, nanny=False) + code.put_release_ensured_gil() + code.putln("}") + code.putln("%s& operator=(const %s& __Pyx_other) {" % (type.cname, type.cname)) + code.putln("if (this != &__Pyx_other) {") + code.put_ensure_gil() + for attr in scope.var_entries: + if not attr.type.is_cfunction: + code.put_var_xdecref(attr, nanny=False) + code.putln("%s = __Pyx_other.%s;" % (attr.cname, attr.cname)) + code.put_var_incref(attr, nanny=False) + code.put_release_ensured_gil() + code.putln("}") + code.putln("return *this;") + code.putln("}") else: - code.putln("%s(const %s& __Pyx_other);" % (type.cname, type.cname)) - code.putln("%s& operator=(const %s& __Pyx_other);" % (type.cname, type.cname)) + code.putln("%s(const %s& __Pyx_other);" % (type.cname, type.cname)) + code.putln("%s& operator=(const %s& __Pyx_other);" % (type.cname, type.cname)) code.putln("};") def generate_enum_definition(self, entry, code): code.mark_pos(entry.pos) type = entry.type name = entry.cname or entry.name or "" - header, footer = self.sue_header_footer(type, "enum", name) + + kind = "enum class" if entry.type.is_cpp_enum else "enum" + header, footer = self.sue_header_footer(type, kind, name) code.putln(header) enum_values = entry.enum_values if not enum_values: @@ -1030,18 +1185,20 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): for value_entry in enum_values: if value_entry.value_node is None: - value_code = value_entry.cname + value_code = value_entry.cname.split("::")[-1] else: value_code = ("%s = %s" % ( - value_entry.cname, + value_entry.cname.split("::")[-1], value_entry.value_node.result())) if value_entry is not last_entry: value_code += "," code.putln(value_code) code.putln(footer) - if entry.type.typedef_flag: - # Not pre-declared. - code.putln("typedef enum %s %s;" % (name, name)) + + if entry.type.is_enum: + if entry.type.typedef_flag: + # Not pre-declared. + code.putln("typedef enum %s %s;" % (name, name)) def generate_typeobj_predeclaration(self, entry, code): code.putln("") @@ -1120,7 +1277,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # Generate object struct definition for an # extension type. if not type.scope: - return # Forward declared but never defined + return # Forward declared but never defined header, footer = \ self.sue_header_footer(type, "struct", type.objstruct_cname) code.putln(header) @@ -1148,18 +1305,53 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): attr_type = py_object_type else: attr_type = attr.type - code.putln( - "%s;" % attr_type.declaration_code(attr.cname)) + if attr.is_cpp_optional: + decl = attr_type.cpp_optional_declaration_code(attr.cname) + else: + decl = attr_type.declaration_code(attr.cname) + type.scope.use_entry_utility_code(attr) + code.putln("%s;" % decl) code.putln(footer) if type.objtypedef_cname is not None: # Only for exposing public typedef name. code.putln("typedef struct %s %s;" % (type.objstruct_cname, type.objtypedef_cname)) - def generate_c_class_declarations(self, env, code, definition): + def generate_c_class_declarations(self, env, code, definition, globalstate): + module_state = globalstate['module_state'] + module_state_defines = globalstate['module_state_defines'] + module_state_clear = globalstate['module_state_clear'] + module_state_traverse = globalstate['module_state_traverse'] + module_state_typeobj = module_state.insertion_point() + module_state_defines_typeobj = module_state_defines.insertion_point() + for writer in [module_state_typeobj, module_state_defines_typeobj]: + writer.putln("#if CYTHON_USE_MODULE_STATE") for entry in env.c_class_entries: if definition or entry.defined_in_pxd: - code.putln("static PyTypeObject *%s = 0;" % ( + module_state.putln("PyTypeObject *%s;" % entry.type.typeptr_cname) + module_state_defines.putln("#define %s %s->%s" % ( + entry.type.typeptr_cname, + Naming.modulestateglobal_cname, entry.type.typeptr_cname)) + module_state_clear.putln( + "Py_CLEAR(clear_module_state->%s);" % + entry.type.typeptr_cname) + module_state_traverse.putln( + "Py_VISIT(traverse_module_state->%s);" % + entry.type.typeptr_cname) + if entry.type.typeobj_cname is not None: + module_state_typeobj.putln("PyObject *%s;" % entry.type.typeobj_cname) + module_state_defines_typeobj.putln("#define %s %s->%s" % ( + entry.type.typeobj_cname, + Naming.modulestateglobal_cname, + entry.type.typeobj_cname)) + module_state_clear.putln( + "Py_CLEAR(clear_module_state->%s);" % ( + entry.type.typeobj_cname)) + module_state_traverse.putln( + "Py_VISIT(traverse_module_state->%s);" % ( + entry.type.typeobj_cname)) + for writer in [module_state_typeobj, module_state_defines_typeobj]: + writer.putln("#endif") def generate_cvariable_declarations(self, env, code, definition): if env.is_cython_builtin: @@ -1199,17 +1391,26 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if storage_class: code.put("%s " % storage_class) - code.put(type.declaration_code( - cname, dll_linkage=dll_linkage)) + if entry.is_cpp_optional: + code.put(type.cpp_optional_declaration_code( + cname, dll_linkage=dll_linkage)) + else: + code.put(type.declaration_code( + cname, dll_linkage=dll_linkage)) if init is not None: code.put_safe(" = %s" % init) code.putln(";") if entry.cname != cname: code.putln("#define %s (*%s)" % (entry.cname, cname)) + env.use_entry_utility_code(entry) def generate_cfunction_declarations(self, env, code, definition): for entry in env.cfunc_entries: - if entry.used or (entry.visibility == 'public' or entry.api): + from_pyx = Options.cimport_from_pyx and not entry.visibility == 'extern' + if (entry.used + or entry.visibility == 'public' + or entry.api + or from_pyx): generate_cfunction_declaration(entry, env, code, definition) def generate_variable_definitions(self, env, code): @@ -1229,14 +1430,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if entry.visibility != 'extern': type = entry.type scope = type.scope - if scope: # could be None if there was an error - if not scope.directives['c_api_binop_methods']: - error(self.pos, - "The 'c_api_binop_methods' directive is only supported for forward compatibility" - " and must be True.") + if scope: # could be None if there was an error self.generate_exttype_vtable(scope, code) self.generate_new_function(scope, code, entry) + self.generate_del_function(scope, code) self.generate_dealloc_function(scope, code) + if scope.needs_gc(): self.generate_traverse_function(scope, code, entry) if scope.needs_tp_clear(): @@ -1264,12 +1463,26 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.generate_descr_set_function(scope, code) if not scope.is_closure_class_scope and scope.defines_any(["__dict__"]): self.generate_dict_getter_function(scope, code) + if scope.defines_any_special(TypeSlots.richcmp_special_methods): self.generate_richcmp_function(scope, code) + elif 'total_ordering' in scope.directives: + # Warn if this is used when it can't have any effect. + warning(scope.parent_type.pos, + "total_ordering directive used, but no comparison and equality methods defined") + + for slot in TypeSlots.get_slot_table(code.globalstate.directives).PyNumberMethods: + if slot.is_binop and scope.defines_any_special(slot.user_methods): + self.generate_binop_function(scope, slot, code, entry.pos) + self.generate_property_accessors(scope, code) self.generate_method_table(scope, code) self.generate_getset_table(scope, code) + code.putln("#if CYTHON_USE_TYPE_SPECS") + self.generate_typeobj_spec(entry, code) + code.putln("#else") self.generate_typeobj_definition(full_module_name, entry, code) + code.putln("#endif") def generate_exttype_vtable(self, scope, code): # Generate the definition of an extension type's vtable. @@ -1287,8 +1500,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): type.empty_declaration_code())) def generate_new_function(self, scope, code, cclass_entry): - tp_slot = TypeSlots.ConstructorSlot("tp_new", '__new__') + tp_slot = TypeSlots.ConstructorSlot("tp_new", "__cinit__") slot_func = scope.mangle_internal("tp_new") + if tp_slot.slot_code(scope) != slot_func: + return # never used + type = scope.parent_type base_type = type.base_type @@ -1298,12 +1514,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if scope.is_internal: # internal classes (should) never need None inits, normal zeroing will do py_attrs = [] - cpp_class_attrs = [entry for entry in scope.var_entries - if entry.type.is_cpp_class] + cpp_constructable_attrs = [entry for entry in scope.var_entries if entry.type.needs_cpp_construction] + + cinit_func_entry = scope.lookup_here("__cinit__") + if cinit_func_entry and not cinit_func_entry.is_special: + cinit_func_entry = None - new_func_entry = scope.lookup_here("__new__") - if base_type or (new_func_entry and new_func_entry.is_special - and not new_func_entry.trivial_signature): + if base_type or (cinit_func_entry and not cinit_func_entry.trivial_signature): unused_marker = '' else: unused_marker = 'CYTHON_UNUSED ' @@ -1331,26 +1548,30 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): need_self_cast = (type.vtabslot_cname or (py_buffers or memoryview_slices or py_attrs) or - cpp_class_attrs) + cpp_constructable_attrs) if need_self_cast: code.putln("%s;" % scope.parent_type.declaration_code("p")) if base_type: tp_new = TypeSlots.get_base_slot_function(scope, tp_slot) if tp_new is None: - tp_new = "%s->tp_new" % base_type.typeptr_cname + tp_new = "__Pyx_PyType_GetSlot(%s, tp_new, newfunc)" % base_type.typeptr_cname code.putln("PyObject *o = %s(t, a, k);" % tp_new) else: code.putln("PyObject *o;") + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln("allocfunc alloc_func = (allocfunc)PyType_GetSlot(t, Py_tp_alloc);") + code.putln("o = alloc_func(t, 0);") + code.putln("#else") if freelist_size: code.globalstate.use_utility_code( UtilityCode.load_cached("IncludeStringH", "StringTools.c")) if is_final_type: type_safety_check = '' else: - type_safety_check = ' & ((t->tp_flags & (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)) == 0)' + type_safety_check = ' & (int)(!__Pyx_PyType_HasFeature(t, (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))' obj_struct = type.declaration_code("", deref=True) code.putln( - "if (CYTHON_COMPILING_IN_CPYTHON && likely((%s > 0) & (t->tp_basicsize == sizeof(%s))%s)) {" % ( + "if (CYTHON_COMPILING_IN_CPYTHON && likely((int)(%s > 0) & (int)(t->tp_basicsize == sizeof(%s))%s)) {" % ( freecount_name, obj_struct, type_safety_check)) code.putln("o = (PyObject*)%s[--%s];" % ( freelist_name, freecount_name)) @@ -1360,7 +1581,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("PyObject_GC_Track(o);") code.putln("} else {") if not is_final_type: - code.putln("if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) {") + code.putln("if (likely(!__Pyx_PyType_HasFeature(t, Py_TPFLAGS_IS_ABSTRACT))) {") code.putln("o = (*t->tp_alloc)(t, 0);") if not is_final_type: code.putln("} else {") @@ -1369,6 +1590,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("if (unlikely(!o)) return 0;") if freelist_size and not base_type: code.putln('}') + if not base_type: + code.putln("#endif") if need_self_cast: code.putln("p = %s;" % type.cast_code("o")) #if need_self_cast: @@ -1389,9 +1612,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): type.vtabslot_cname, struct_type_cast, type.vtabptr_cname)) - for entry in cpp_class_attrs: + for entry in cpp_constructable_attrs: + if entry.is_cpp_optional: + decl_code = entry.type.cpp_optional_declaration_code("") + else: + decl_code = entry.type.empty_declaration_code() code.putln("new((void*)&(p->%s)) %s();" % ( - entry.cname, entry.type.empty_declaration_code())) + entry.cname, decl_code)) for entry in py_attrs: if entry.name == "__dict__": @@ -1411,14 +1638,14 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if cclass_entry.cname == '__pyx_memoryviewslice': code.putln("p->from_slice.memview = NULL;") - if new_func_entry and new_func_entry.is_special: - if new_func_entry.trivial_signature: + if cinit_func_entry: + if cinit_func_entry.trivial_signature: cinit_args = "o, %s, NULL" % Naming.empty_tuple else: cinit_args = "o, a, k" needs_error_cleanup = True code.putln("if (unlikely(%s(%s) < 0)) goto bad;" % ( - new_func_entry.func_cname, cinit_args)) + cinit_func_entry.func_cname, cinit_args)) code.putln( "return o;") @@ -1429,6 +1656,28 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln( "}") + def generate_del_function(self, scope, code): + tp_slot = TypeSlots.get_slot_by_name("tp_finalize", scope.directives) + slot_func_cname = scope.mangle_internal("tp_finalize") + if tp_slot.slot_code(scope) != slot_func_cname: + return # never used + + entry = scope.lookup_here("__del__") + if entry is None or not entry.is_special: + return # nothing to wrap + code.putln("") + + if tp_slot.used_ifdef: + code.putln("#if %s" % tp_slot.used_ifdef) + code.putln("static void %s(PyObject *o) {" % slot_func_cname) + code.putln("PyObject *etype, *eval, *etb;") + code.putln("PyErr_Fetch(&etype, &eval, &etb);") + code.putln("%s(o);" % entry.func_cname) + code.putln("PyErr_Restore(etype, eval, etb);") + code.putln("}") + if tp_slot.used_ifdef: + code.putln("#endif") + def generate_dealloc_function(self, scope, code): tp_slot = TypeSlots.ConstructorSlot("tp_dealloc", '__dealloc__') slot_func = scope.mangle_internal("tp_dealloc") @@ -1443,6 +1692,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): is_final_type = scope.parent_type.is_final_type needs_gc = scope.needs_gc() + needs_trashcan = scope.needs_trashcan() weakref_slot = scope.lookup_here("__weakref__") if not scope.is_closure_class_scope else None if weakref_slot not in scope.var_entries: @@ -1453,13 +1703,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): dict_slot = None _, (py_attrs, _, memoryview_slices) = scope.get_refcounted_entries() - cpp_class_attrs = [entry for entry in scope.var_entries - if entry.type.is_cpp_class] + cpp_destructable_attrs = [entry for entry in scope.var_entries + if entry.type.needs_cpp_construction] - if py_attrs or cpp_class_attrs or memoryview_slices or weakref_slot or dict_slot: + if py_attrs or cpp_destructable_attrs or memoryview_slices or weakref_slot or dict_slot: self.generate_self_cast(scope, code) - if not is_final_type: + if not is_final_type or scope.may_have_finalize(): # in Py3.4+, call tp_finalize() as early as possible code.putln("#if CYTHON_USE_TP_FINALIZE") if needs_gc: @@ -1468,11 +1718,15 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): finalised_check = ( '(!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))') code.putln( - "if (unlikely(PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE)" - " && Py_TYPE(o)->tp_finalize) && %s) {" % finalised_check) + "if (unlikely(" + "(PY_VERSION_HEX >= 0x03080000 || __Pyx_PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE))" + " && __Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && %s) {" % finalised_check) + + code.putln("if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == %s) {" % slot_func_cname) # if instance was resurrected by finaliser, return code.putln("if (PyObject_CallFinalizerFromDealloc(o)) return;") code.putln("}") + code.putln("}") code.putln("#endif") if needs_gc: @@ -1481,33 +1735,40 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # running this destructor. code.putln("PyObject_GC_UnTrack(o);") - # call the user's __dealloc__ - self.generate_usr_dealloc_call(scope, code) + if needs_trashcan: + code.globalstate.use_utility_code( + UtilityCode.load_cached("PyTrashcan", "ExtensionTypes.c")) + code.putln("__Pyx_TRASHCAN_BEGIN(o, %s)" % slot_func_cname) if weakref_slot: + # We must clean the weakreferences before calling the user's __dealloc__ + # because if the __dealloc__ releases the GIL, a weakref can be + # dereferenced accessing the object in an inconsistent state or + # resurrecting it. code.putln("if (p->__weakref__) PyObject_ClearWeakRefs(o);") + # call the user's __dealloc__ + self.generate_usr_dealloc_call(scope, code) + if dict_slot: code.putln("if (p->__dict__) PyDict_Clear(p->__dict__);") - for entry in cpp_class_attrs: + for entry in cpp_destructable_attrs: code.putln("__Pyx_call_destructor(p->%s);" % entry.cname) - for entry in py_attrs: + for entry in (py_attrs + memoryview_slices): code.put_xdecref_clear("p->%s" % entry.cname, entry.type, nanny=False, - clear_before_decref=True) - - for entry in memoryview_slices: - code.put_xdecref_memoryviewslice("p->%s" % entry.cname, - have_gil=True) + clear_before_decref=True, have_gil=True) if base_type: base_cname = base_type.typeptr_cname if needs_gc: # The base class deallocator probably expects this to be tracked, # so undo the untracking above. - if base_type.scope and base_type.scope.needs_gc(): - code.putln("PyObject_GC_Track(o);") + if base_type.scope: + # Assume that we know whether the base class uses GC or not. + if base_type.scope.needs_gc(): + code.putln("PyObject_GC_Track(o);") else: code.putln("if (PyType_IS_GC(%s)) PyObject_GC_Track(o);" % base_cname) @@ -1515,13 +1776,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if tp_dealloc is not None: code.putln("%s(o);" % tp_dealloc) elif base_type.is_builtin_type: - code.putln("%s->tp_dealloc(o);" % base_cname) + code.putln("__Pyx_PyType_GetSlot(%s, tp_dealloc, destructor)(o);" % base_cname) else: # This is an externally defined type. Calling through the # cimported base type pointer directly interacts badly with # the module cleanup, which may already have cleared it. # In that case, fall back to traversing the type hierarchy. - code.putln("if (likely(%s)) %s->tp_dealloc(o); " + code.putln("if (likely(%s)) __Pyx_PyType_GetSlot(%s, tp_dealloc, destructor)(o); " "else __Pyx_call_next_tp_dealloc(o, %s);" % ( base_cname, base_cname, slot_func_cname)) code.globalstate.use_utility_code( @@ -1536,11 +1797,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): type_safety_check = '' else: type_safety_check = ( - ' & ((Py_TYPE(o)->tp_flags & (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)) == 0)') + ' & (int)(!__Pyx_PyType_HasFeature(Py_TYPE(o), (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))') type = scope.parent_type code.putln( - "if (CYTHON_COMPILING_IN_CPYTHON && ((%s < %d) & (Py_TYPE(o)->tp_basicsize == sizeof(%s))%s)) {" % ( + "if (CYTHON_COMPILING_IN_CPYTHON && ((int)(%s < %d) & (int)(Py_TYPE(o)->tp_basicsize == sizeof(%s))%s)) {" % ( freecount_name, freelist_size, type.declaration_code("", deref=True), @@ -1551,12 +1812,16 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("(*Py_TYPE(o)->tp_free)(o);") if freelist_size: code.putln("}") + + if needs_trashcan: + code.putln("__Pyx_TRASHCAN_END") + code.putln( "}") def generate_usr_dealloc_call(self, scope, code): entry = scope.lookup_here("__dealloc__") - if not entry: + if not entry or not entry.is_special: return code.putln("{") @@ -1632,11 +1897,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("}") def generate_clear_function(self, scope, code, cclass_entry): - tp_slot = TypeSlots.get_slot_by_name("tp_clear") + tp_slot = TypeSlots.get_slot_by_name("tp_clear", scope.directives) slot_func = scope.mangle_internal("tp_clear") base_type = scope.parent_type.base_type if tp_slot.slot_code(scope) != slot_func: - return # never used + return # never used have_entries, (py_attrs, py_buffers, memoryview_slices) = ( scope.get_refcounted_entries(include_gc_simple=False)) @@ -1694,7 +1959,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("Py_CLEAR(p->%s.obj);" % entry.cname) if cclass_entry.cname == '__pyx_memoryviewslice': - code.putln("__PYX_XDEC_MEMVIEW(&p->from_slice, 1);") + code.putln("__PYX_XCLEAR_MEMVIEW(&p->from_slice, 1);") code.putln("return 0;") code.putln("}") @@ -1735,12 +2000,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if set_entry: code.putln("return %s(o, i, v);" % set_entry.func_cname) else: + code.putln( + "__Pyx_TypeName o_type_name;") self.generate_guarded_basetype_call( base_type, "tp_as_mapping", "mp_ass_subscript", "o, i, v", code) code.putln( + "o_type_name = __Pyx_PyType_GetName(Py_TYPE(o));") + code.putln( "PyErr_Format(PyExc_NotImplementedError,") code.putln( - ' "Subscript assignment not supported by %.200s", Py_TYPE(o)->tp_name);') + ' "Subscript assignment not supported by " __Pyx_FMT_TYPENAME, o_type_name);') + code.putln( + "__Pyx_DECREF_TypeName(o_type_name);") code.putln( "return -1;") code.putln( @@ -1752,12 +2023,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): "return %s(o, i);" % ( del_entry.func_cname)) else: + code.putln( + "__Pyx_TypeName o_type_name;") self.generate_guarded_basetype_call( base_type, "tp_as_mapping", "mp_ass_subscript", "o, i, v", code) code.putln( + "o_type_name = __Pyx_PyType_GetName(Py_TYPE(o));") + code.putln( "PyErr_Format(PyExc_NotImplementedError,") code.putln( - ' "Subscript deletion not supported by %.200s", Py_TYPE(o)->tp_name);') + ' "Subscript deletion not supported by " __Pyx_FMT_TYPENAME, o_type_name);') + code.putln( + "__Pyx_DECREF_TypeName(o_type_name);") code.putln( "return -1;") code.putln( @@ -1802,12 +2079,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): "return %s(o, i, j, v);" % ( set_entry.func_cname)) else: + code.putln( + "__Pyx_TypeName o_type_name;") self.generate_guarded_basetype_call( base_type, "tp_as_sequence", "sq_ass_slice", "o, i, j, v", code) code.putln( + "o_type_name = __Pyx_PyType_GetName(Py_TYPE(o));") + code.putln( "PyErr_Format(PyExc_NotImplementedError,") code.putln( - ' "2-element slice assignment not supported by %.200s", Py_TYPE(o)->tp_name);') + ' "2-element slice assignment not supported by " __Pyx_FMT_TYPENAME, o_type_name);') + code.putln( + "__Pyx_DECREF_TypeName(o_type_name);") code.putln( "return -1;") code.putln( @@ -1819,12 +2102,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): "return %s(o, i, j);" % ( del_entry.func_cname)) else: + code.putln( + "__Pyx_TypeName o_type_name;") self.generate_guarded_basetype_call( base_type, "tp_as_sequence", "sq_ass_slice", "o, i, j, v", code) code.putln( + "o_type_name = __Pyx_PyType_GetName(Py_TYPE(o));") + code.putln( "PyErr_Format(PyExc_NotImplementedError,") code.putln( - ' "2-element slice deletion not supported by %.200s", Py_TYPE(o)->tp_name);') + ' "2-element slice deletion not supported by " __Pyx_FMT_TYPENAME, o_type_name);') + code.putln( + "__Pyx_DECREF_TypeName(o_type_name);") code.putln( "return -1;") code.putln( @@ -1854,37 +2143,112 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # need to call up into base classes as we may not know all implemented comparison methods extern_parent = cls if cls.typeptr_cname else scope.parent_type.base_type - eq_entry = None - has_ne = False + total_ordering = 'total_ordering' in scope.directives + + comp_entry = {} + for cmp_method in TypeSlots.richcmp_special_methods: for class_scope in class_scopes: entry = class_scope.lookup_here(cmp_method) if entry is not None: + comp_entry[cmp_method] = entry break + + if total_ordering: + # Check this is valid - we must have at least 1 operation defined. + comp_names = [from_name for from_name, to_name in TOTAL_ORDERING if from_name in comp_entry] + if not comp_names: + if '__eq__' not in comp_entry and '__ne__' not in comp_entry: + warning(scope.parent_type.pos, + "total_ordering directive used, but no comparison and equality methods defined") + else: + warning(scope.parent_type.pos, + "total_ordering directive used, but no comparison methods defined") + total_ordering = False else: - continue + if '__eq__' not in comp_entry and '__ne__' not in comp_entry: + warning(scope.parent_type.pos, "total_ordering directive used, but no equality method defined") + total_ordering = False + + # Same priority as functools, prefers + # __lt__ to __le__ to __gt__ to __ge__ + ordering_source = max(comp_names) + for cmp_method in TypeSlots.richcmp_special_methods: cmp_type = cmp_method.strip('_').upper() # e.g. "__eq__" -> EQ + entry = comp_entry.get(cmp_method) + if entry is None and (not total_ordering or cmp_type in ('NE', 'EQ')): + # No definition, fall back to superclasses. + # eq/ne methods shouldn't use the total_ordering code. + continue + code.putln("case Py_%s: {" % cmp_type) - if cmp_method == '__eq__': - eq_entry = entry - # Python itself does not do this optimisation, it seems... - #code.putln("if (o1 == o2) return __Pyx_NewRef(Py_True);") - elif cmp_method == '__ne__': - has_ne = True - # Python itself does not do this optimisation, it seems... - #code.putln("if (o1 == o2) return __Pyx_NewRef(Py_False);") - code.putln("return %s(o1, o2);" % entry.func_cname) - code.putln("}") + if entry is None: + assert total_ordering + # We need to generate this from the other methods. + invert_comp, comp_op, invert_equals = TOTAL_ORDERING[ordering_source, cmp_method] + + # First we always do the comparison. + code.putln("PyObject *ret;") + code.putln("ret = %s(o1, o2);" % comp_entry[ordering_source].func_cname) + code.putln("if (likely(ret && ret != Py_NotImplemented)) {") + code.putln("int order_res = __Pyx_PyObject_IsTrue(ret);") + code.putln("Py_DECREF(ret);") + code.putln("if (unlikely(order_res < 0)) return NULL;") + # We may need to check equality too. For some combos it's never required. + if invert_equals is not None: + # Implement the and/or check with an if. + if comp_op == '&&': + code.putln("if (%s order_res) {" % ('!!' if invert_comp else '!')) + code.putln("ret = __Pyx_NewRef(Py_False);") + code.putln("} else {") + elif comp_op == '||': + code.putln("if (%s order_res) {" % ('!' if invert_comp else '')) + code.putln("ret = __Pyx_NewRef(Py_True);") + code.putln("} else {") + else: + raise AssertionError('Unknown op %s' % (comp_op, )) + if '__eq__' in comp_entry: + eq_func = '__eq__' + else: + # Fall back to NE, which is defined here. + eq_func = '__ne__' + invert_equals = not invert_equals + + code.putln("ret = %s(o1, o2);" % comp_entry[eq_func].func_cname) + code.putln("if (likely(ret && ret != Py_NotImplemented)) {") + code.putln("int eq_res = __Pyx_PyObject_IsTrue(ret);") + code.putln("Py_DECREF(ret);") + code.putln("if (unlikely(eq_res < 0)) return NULL;") + if invert_equals: + code.putln("ret = eq_res ? Py_False : Py_True;") + else: + code.putln("ret = eq_res ? Py_True : Py_False;") + code.putln("Py_INCREF(ret);") + code.putln("}") # equals success + code.putln("}") # Needs to try equals + else: + # Convert direct to a boolean. + if invert_comp: + code.putln("ret = order_res ? Py_False : Py_True;") + else: + code.putln("ret = order_res ? Py_True : Py_False;") + code.putln("Py_INCREF(ret);") + code.putln("}") # comp_op + code.putln("return ret;") + else: + code.putln("return %s(o1, o2);" % entry.func_cname) + code.putln("}") # Case - if eq_entry and not has_ne and not extern_parent: + if '__eq__' in comp_entry and '__ne__' not in comp_entry and not extern_parent: code.putln("case Py_NE: {") code.putln("PyObject *ret;") # Python itself does not do this optimisation, it seems... #code.putln("if (o1 == o2) return __Pyx_NewRef(Py_False);") - code.putln("ret = %s(o1, o2);" % eq_entry.func_cname) + code.putln("ret = %s(o1, o2);" % comp_entry['__eq__'].func_cname) code.putln("if (likely(ret && ret != Py_NotImplemented)) {") - code.putln("int b = __Pyx_PyObject_IsTrue(ret); Py_DECREF(ret);") + code.putln("int b = __Pyx_PyObject_IsTrue(ret);") + code.putln("Py_DECREF(ret);") code.putln("if (unlikely(b < 0)) return NULL;") code.putln("ret = (b) ? Py_False : Py_True;") code.putln("Py_INCREF(ret);") @@ -1902,6 +2266,73 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("}") # switch code.putln("}") + def generate_binop_function(self, scope, slot, code, pos): + func_name = scope.mangle_internal(slot.slot_name) + if scope.directives['c_api_binop_methods']: + code.putln('#define %s %s' % (func_name, slot.left_slot.slot_code(scope))) + return + + code.putln() + preprocessor_guard = slot.preprocessor_guard_code() + if preprocessor_guard: + code.putln(preprocessor_guard) + + if slot.left_slot.signature in (TypeSlots.binaryfunc, TypeSlots.ibinaryfunc): + slot_type = 'binaryfunc' + extra_arg = extra_arg_decl = '' + elif slot.left_slot.signature in (TypeSlots.powternaryfunc, TypeSlots.ipowternaryfunc): + slot_type = 'ternaryfunc' + extra_arg = ', extra_arg' + extra_arg_decl = ', PyObject* extra_arg' + else: + error(pos, "Unexpected type slot signature: %s" % slot) + return + + def get_slot_method_cname(method_name): + entry = scope.lookup(method_name) + return entry.func_cname if entry and entry.is_special else None + + def call_slot_method(method_name, reverse): + func_cname = get_slot_method_cname(method_name) + if func_cname: + return "%s(%s%s)" % ( + func_cname, + "right, left" if reverse else "left, right", + extra_arg) + else: + return '%s_maybe_call_slot(__Pyx_PyType_GetSlot(%s, tp_base, PyTypeObject*), left, right %s)' % ( + func_name, + scope.parent_type.typeptr_cname, + extra_arg) + + if get_slot_method_cname(slot.left_slot.method_name) and not get_slot_method_cname(slot.right_slot.method_name): + warning(pos, "Extension type implements %s() but not %s(). " + "The behaviour has changed from previous Cython versions to match Python semantics. " + "You can implement both special methods in a backwards compatible way." % ( + slot.left_slot.method_name, + slot.right_slot.method_name, + )) + + overloads_left = int(bool(get_slot_method_cname(slot.left_slot.method_name))) + overloads_right = int(bool(get_slot_method_cname(slot.right_slot.method_name))) + code.putln( + TempitaUtilityCode.load_as_string( + "BinopSlot", "ExtensionTypes.c", + context={ + "func_name": func_name, + "slot_name": slot.slot_name, + "overloads_left": overloads_left, + "overloads_right": overloads_right, + "call_left": call_slot_method(slot.left_slot.method_name, reverse=False), + "call_right": call_slot_method(slot.right_slot.method_name, reverse=True), + "type_cname": scope.parent_type.typeptr_cname, + "slot_type": slot_type, + "extra_arg": extra_arg, + "extra_arg_decl": extra_arg_decl, + })[1]) + if preprocessor_guard: + code.putln("#endif") + def generate_getattro_function(self, scope, code): # First try to get the attribute using __getattribute__, if defined, or # PyObject_GenericGetAttr. @@ -2136,10 +2567,42 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln( "}") + def generate_typeobj_spec(self, entry, code): + ext_type = entry.type + scope = ext_type.scope + + members_slot = TypeSlots.get_slot_by_name("tp_members", code.globalstate.directives) + members_slot.generate_substructure_spec(scope, code) + + buffer_slot = TypeSlots.get_slot_by_name("tp_as_buffer", code.globalstate.directives) + if not buffer_slot.is_empty(scope): + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") + buffer_slot.generate_substructure(scope, code) + code.putln("#endif") + + code.putln("static PyType_Slot %s_slots[] = {" % ext_type.typeobj_cname) + for slot in TypeSlots.get_slot_table(code.globalstate.directives): + slot.generate_spec(scope, code) + code.putln("{0, 0},") + code.putln("};") + + if ext_type.typedef_flag: + objstruct = ext_type.objstruct_cname + else: + objstruct = "struct %s" % ext_type.objstruct_cname + classname = scope.class_name.as_c_string_literal() + code.putln("static PyType_Spec %s_spec = {" % ext_type.typeobj_cname) + code.putln('"%s.%s",' % (self.full_module_name, classname.replace('"', ''))) + code.putln("sizeof(%s)," % objstruct) + code.putln("0,") + code.putln("%s," % TypeSlots.get_slot_by_name("tp_flags", scope.directives).slot_code(scope)) + code.putln("%s_slots," % ext_type.typeobj_cname) + code.putln("};") + def generate_typeobj_definition(self, modname, entry, code): type = entry.type scope = type.scope - for suite in TypeSlots.substructures: + for suite in TypeSlots.get_slot_table(code.globalstate.directives).substructures: suite.generate_substructure(scope, code) code.putln("") if entry.visibility == 'public': @@ -2150,9 +2613,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln(header % type.typeobj_cname) code.putln( "PyVarObject_HEAD_INIT(0, 0)") + classname = scope.class_name.as_c_string_literal() code.putln( - '"%s.%s", /*tp_name*/' % ( - self.full_module_name, scope.class_name)) + '"%s."%s, /*tp_name*/' % ( + self.full_module_name, + classname)) if type.typedef_flag: objstruct = type.objstruct_cname else: @@ -2161,7 +2626,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): "sizeof(%s), /*tp_basicsize*/" % objstruct) code.putln( "0, /*tp_itemsize*/") - for slot in TypeSlots.slot_table: + for slot in TypeSlots.get_slot_table(code.globalstate.directives): slot.generate(scope, code) code.putln( "};") @@ -2215,12 +2680,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if doc: if doc.is_unicode: doc = doc.as_utf8_string() - doc_code = doc.as_c_string_literal() + doc_code = "PyDoc_STR(%s)" % doc.as_c_string_literal() else: doc_code = "0" code.putln( - '{(char *)"%s", %s, %s, (char *)%s, 0},' % ( - entry.name, + '{(char *)%s, %s, %s, (char *)%s, 0},' % ( + entry.name.as_c_string_literal(), entry.getter_cname or "0", entry.setter_cname or "0", doc_code)) @@ -2296,7 +2761,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if code.label_used(code.error_label): code.put_label(code.error_label) # This helps locate the offending name. - code.put_add_traceback(self.full_module_name) + code.put_add_traceback(EncodedString(self.full_module_name)) code.error_label = old_error_label code.putln("bad:") code.putln("return -1;") @@ -2305,20 +2770,213 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln(UtilityCode.load_as_string("ImportStar", "ImportExport.c")[1]) code.exit_cfunc_scope() # done with labels + def generate_module_state_start(self, env, code): + # TODO: Refactor to move module state struct decl closer to the static decl + code.putln('typedef struct {') + code.putln('PyObject *%s;' % env.module_dict_cname) + code.putln('PyObject *%s;' % Naming.builtins_cname) + code.putln('PyObject *%s;' % Naming.cython_runtime_cname) + code.putln('PyObject *%s;' % Naming.empty_tuple) + code.putln('PyObject *%s;' % Naming.empty_bytes) + code.putln('PyObject *%s;' % Naming.empty_unicode) + if Options.pre_import is not None: + code.putln('PyObject *%s;' % Naming.preimport_cname) + for type_cname, used_name in Naming.used_types_and_macros: + code.putln('#ifdef %s' % used_name) + code.putln('PyTypeObject *%s;' % type_cname) + code.putln('#endif') + + def generate_module_state_end(self, env, modules, globalstate): + module_state = globalstate['module_state'] + module_state_defines = globalstate['module_state_defines'] + module_state_clear = globalstate['module_state_clear'] + module_state_traverse = globalstate['module_state_traverse'] + module_state.putln('} %s;' % Naming.modulestate_cname) + module_state.putln('') + module_state.putln("#if CYTHON_USE_MODULE_STATE") + module_state.putln('#ifdef __cplusplus') + module_state.putln('namespace {') + module_state.putln('extern struct PyModuleDef %s;' % Naming.pymoduledef_cname) + module_state.putln('} /* anonymous namespace */') + module_state.putln('#else') + module_state.putln('static struct PyModuleDef %s;' % Naming.pymoduledef_cname) + module_state.putln('#endif') + module_state.putln('') + module_state.putln('#define %s(o) ((%s *)__Pyx_PyModule_GetState(o))' % ( + Naming.modulestate_cname, + Naming.modulestate_cname)) + module_state.putln('') + module_state.putln('#define %s (%s(PyState_FindModule(&%s)))' % ( + Naming.modulestateglobal_cname, + Naming.modulestate_cname, + Naming.pymoduledef_cname)) + module_state.putln('') + module_state.putln('#define %s (PyState_FindModule(&%s))' % ( + env.module_cname, + Naming.pymoduledef_cname)) + module_state.putln("#else") + module_state.putln('static %s %s_static =' % ( + Naming.modulestate_cname, + Naming.modulestateglobal_cname + )) + module_state.putln('#ifdef __cplusplus') + # C++ likes to be initialized with {} to avoid "missing initializer" warnings + # but it isn't valid C + module_state.putln(' {};') + module_state.putln('#else') + module_state.putln(' {0};') + module_state.putln('#endif') + module_state.putln('static %s *%s = &%s_static;' % ( + Naming.modulestate_cname, + Naming.modulestateglobal_cname, + Naming.modulestateglobal_cname + )) + module_state.putln("#endif") + module_state_clear.putln("return 0;") + module_state_clear.putln("}") + module_state_clear.putln("#endif") + module_state_traverse.putln("return 0;") + module_state_traverse.putln("}") + module_state_traverse.putln("#endif") + + def generate_module_state_defines(self, env, code): + code.putln('#define %s %s->%s' % ( + env.module_dict_cname, + Naming.modulestateglobal_cname, + env.module_dict_cname)) + code.putln('#define %s %s->%s' % ( + Naming.builtins_cname, + Naming.modulestateglobal_cname, + Naming.builtins_cname)) + code.putln('#define %s %s->%s' % ( + Naming.cython_runtime_cname, + Naming.modulestateglobal_cname, + Naming.cython_runtime_cname)) + code.putln('#define %s %s->%s' % ( + Naming.empty_tuple, + Naming.modulestateglobal_cname, + Naming.empty_tuple)) + code.putln('#define %s %s->%s' % ( + Naming.empty_bytes, + Naming.modulestateglobal_cname, + Naming.empty_bytes)) + code.putln('#define %s %s->%s' % ( + Naming.empty_unicode, + Naming.modulestateglobal_cname, + Naming.empty_unicode)) + if Options.pre_import is not None: + code.putln('#define %s %s->%s' % ( + Naming.preimport_cname, + Naming.modulestateglobal_cname, + Naming.preimport_cname)) + for cname, used_name in Naming.used_types_and_macros: + code.putln('#ifdef %s' % used_name) + code.putln('#define %s %s->%s' % ( + cname, + Naming.modulestateglobal_cname, + cname)) + code.putln('#endif') + + def generate_module_state_clear(self, env, code): + code.putln("#if CYTHON_USE_MODULE_STATE") + code.putln("static int %s_clear(PyObject *m) {" % Naming.module_cname) + code.putln("%s *clear_module_state = %s(m);" % ( + Naming.modulestate_cname, + Naming.modulestate_cname)) + code.putln("if (!clear_module_state) return 0;") + code.putln('Py_CLEAR(clear_module_state->%s);' % + env.module_dict_cname) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.builtins_cname) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.cython_runtime_cname) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.empty_tuple) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.empty_bytes) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.empty_unicode) + code.putln('#ifdef __Pyx_CyFunction_USED') + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.cyfunction_type_cname) + code.putln('#endif') + code.putln('#ifdef __Pyx_FusedFunction_USED') + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.fusedfunction_type_cname) + code.putln('#endif') + + def generate_module_state_traverse(self, env, code): + code.putln("#if CYTHON_USE_MODULE_STATE") + code.putln("static int %s_traverse(PyObject *m, visitproc visit, void *arg) {" % Naming.module_cname) + code.putln("%s *traverse_module_state = %s(m);" % ( + Naming.modulestate_cname, + Naming.modulestate_cname)) + code.putln("if (!traverse_module_state) return 0;") + code.putln('Py_VISIT(traverse_module_state->%s);' % + env.module_dict_cname) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.builtins_cname) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.cython_runtime_cname) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.empty_tuple) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.empty_bytes) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.empty_unicode) + code.putln('#ifdef __Pyx_CyFunction_USED') + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.cyfunction_type_cname) + code.putln('#endif') + code.putln('#ifdef __Pyx_FusedFunction_USED') + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.fusedfunction_type_cname) + code.putln('#endif') + def generate_module_init_func(self, imported_modules, env, code): subfunction = self.mod_init_subfunction(self.pos, self.scope, code) + self.generate_pymoduledef_struct(env, code) + code.enter_cfunc_scope(self.scope) code.putln("") code.putln(UtilityCode.load_as_string("PyModInitFuncType", "ModuleSetupCode.c")[0]) - header2 = "__Pyx_PyMODINIT_FUNC init%s(void)" % env.module_name + if env.module_name.isascii(): + py2_mod_name = env.module_name + fail_compilation_in_py2 = False + else: + fail_compilation_in_py2 = True + # at this point py2_mod_name is largely a placeholder and the value doesn't matter + py2_mod_name = env.module_name.encode("ascii", errors="ignore").decode("utf8") + + header2 = "__Pyx_PyMODINIT_FUNC init%s(void)" % py2_mod_name header3 = "__Pyx_PyMODINIT_FUNC %s(void)" % self.mod_init_func_cname('PyInit', env) + header3 = EncodedString(header3) code.putln("#if PY_MAJOR_VERSION < 3") # Optimise for small code size as the module init function is only executed once. code.putln("%s CYTHON_SMALL_CODE; /*proto*/" % header2) + if fail_compilation_in_py2: + code.putln('#error "Unicode module names are not supported in Python 2";') + if self.scope.is_package: + code.putln("#if !defined(CYTHON_NO_PYINIT_EXPORT) && (defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS))") + code.putln("__Pyx_PyMODINIT_FUNC init__init__(void) { init%s(); }" % py2_mod_name) + code.putln("#endif") code.putln(header2) code.putln("#else") code.putln("%s CYTHON_SMALL_CODE; /*proto*/" % header3) + if self.scope.is_package: + code.putln("#if !defined(CYTHON_NO_PYINIT_EXPORT) && (defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS))") + code.putln("__Pyx_PyMODINIT_FUNC PyInit___init__(void) { return %s(); }" % ( + self.mod_init_func_cname('PyInit', env))) + code.putln("#endif") + # Hack for a distutils bug - https://bugs.python.org/issue39432 + # distutils attempts to make visible a slightly wrong PyInitU module name. Just create a dummy + # function to keep it quiet + wrong_punycode_module_name = self.wrong_punycode_module_name(env.module_name) + if wrong_punycode_module_name: + code.putln("#if !defined(CYTHON_NO_PYINIT_EXPORT) && (defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS))") + code.putln("void %s(void) {} /* workaround for https://bugs.python.org/issue39432 */" % wrong_punycode_module_name) + code.putln("#endif") code.putln(header3) # CPython 3.5+ supports multi-phase module initialisation (gives access to __spec__, __file__, etc.) @@ -2333,7 +2991,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("") # main module init code lives in Py_mod_exec function, not in PyInit function code.putln("static CYTHON_SMALL_CODE int %s(PyObject *%s)" % ( - self.mod_init_func_cname(Naming.pymodule_exec_func_cname, env), + self.module_init_func_cname(), Naming.pymodinit_module_arg)) code.putln("#endif") # PEP489 @@ -2341,12 +2999,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # start of module init/exec function (pre/post PEP 489) code.putln("{") + code.putln('int stringtab_initialized = 0;') + code.putln("#if CYTHON_USE_MODULE_STATE") + code.putln('int pystate_addmodule_run = 0;') + code.putln("#endif") tempdecl_code = code.insertion_point() profile = code.globalstate.directives['profile'] linetrace = code.globalstate.directives['linetrace'] if profile or linetrace: + if linetrace: + code.use_fast_gil_utility_code() code.globalstate.use_utility_code(UtilityCode.load_cached("Profile", "Profile.c")) code.put_declare_refcount_context() @@ -2361,7 +3025,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): )) code.putln('PyErr_SetString(PyExc_RuntimeError,' ' "Module \'%s\' has already been imported. Re-initialisation is not supported.");' % - env.module_name) + env.module_name.as_c_string_literal()[1:-1]) code.putln("return -1;") code.putln("}") code.putln("#elif PY_MAJOR_VERSION >= 3") @@ -2372,6 +3036,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): )) code.putln("#endif") + code.putln("/*--- Module creation code ---*/") + self.generate_module_creation_code(env, code) + if profile or linetrace: tempdecl_code.put_trace_declarations() code.put_trace_frame_init() @@ -2395,7 +3062,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): for ext_type in ('CyFunction', 'FusedFunction', 'Coroutine', 'Generator', 'AsyncGen', 'StopAsyncIteration'): code.putln("#ifdef __Pyx_%s_USED" % ext_type) - code.put_error_if_neg(self.pos, "__pyx_%s_init()" % ext_type) + code.put_error_if_neg(self.pos, "__pyx_%s_init(%s)" % (ext_type, env.module_cname)) code.putln("#endif") code.putln("/*--- Library function declarations ---*/") @@ -2408,18 +3075,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("PyEval_InitThreads();") code.putln("#endif") - code.putln("/*--- Module creation code ---*/") - self.generate_module_creation_code(env, code) - code.putln("/*--- Initialize various global constants etc. ---*/") - code.put_error_if_neg(self.pos, "__Pyx_InitGlobals()") + code.put_error_if_neg(self.pos, "__Pyx_InitConstants()") + code.putln("stringtab_initialized = 1;") + code.put_error_if_neg(self.pos, "__Pyx_InitGlobals()") # calls any utility code + code.putln("#if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || " "__PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT)") code.put_error_if_neg(self.pos, "__Pyx_init_sys_getdefaultencoding_params()") code.putln("#endif") - code.putln("if (%s%s) {" % (Naming.module_is_main, self.full_module_name.replace('.', '__'))) + code.putln("if (%s) {" % self.is_main_module_flag_cname()) code.put_error_if_neg(self.pos, 'PyObject_SetAttr(%s, %s, %s)' % ( env.module_cname, code.intern_identifier(EncodedString("__name__")), @@ -2474,7 +3141,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.put_trace_call(header3, self.pos, nogil=not code.funcstate.gil_owned) code.funcstate.can_trace = True + code.mark_pos(None) self.body.generate_execution_code(code) + code.mark_pos(None) if profile or linetrace: code.funcstate.can_trace = False @@ -2495,8 +3164,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): for cname, type in code.funcstate.all_managed_temps(): code.put_xdecref(cname, type) code.putln('if (%s) {' % env.module_cname) - code.putln('if (%s) {' % env.module_dict_cname) - code.put_add_traceback("init %s" % env.qualified_name) + code.putln('if (%s && stringtab_initialized) {' % env.module_dict_cname) + # We can run into errors before the module or stringtab are initialized. + # In this case it is not safe to add a traceback (because it uses the stringtab) + code.put_add_traceback(EncodedString("init %s" % env.qualified_name)) code.globalstate.use_utility_code(Nodes.traceback_utility_code) # Module reference and module dict are in global variables which might still be needed # for cleanup, atexit code, etc., so leaking is better than crashing. @@ -2504,9 +3175,24 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # user code in atexit or other global registries. ##code.put_decref_clear(env.module_dict_cname, py_object_type, nanny=False) code.putln('}') + code.putln("#if !CYTHON_USE_MODULE_STATE") code.put_decref_clear(env.module_cname, py_object_type, nanny=False, clear_before_decref=True) + code.putln("#else") + # This section is mainly for the limited API. env.module_cname still owns a reference so + # decrement that + code.put_decref(env.module_cname, py_object_type, nanny=False) + # Also remove the failed module from the module state lookup + # fetch/restore the error indicator because PyState_RemvoeModule might fail itself + code.putln("if (pystate_addmodule_run) {") + code.putln("PyObject *tp, *value, *tb;") + code.putln("PyErr_Fetch(&tp, &value, &tb);") + code.putln("PyState_RemoveModule(&%s);" % Naming.pymoduledef_cname) + code.putln("PyErr_Restore(tp, value, tb);") + code.putln("}") + code.putln("#endif") code.putln('} else if (!PyErr_Occurred()) {') - code.putln('PyErr_SetString(PyExc_ImportError, "init %s");' % env.qualified_name) + code.putln('PyErr_SetString(PyExc_ImportError, "init %s");' % + env.qualified_name.as_c_string_literal()[1:-1]) code.putln('}') code.put_label(code.return_label) @@ -2556,7 +3242,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("static int %s(void) {" % self.cfunc_name) code.put_declare_refcount_context() self.tempdecl_code = code.insertion_point() - code.put_setup_refcount_context(self.cfunc_name) + code.put_setup_refcount_context(EncodedString(self.cfunc_name)) # Leave a grepable marker that makes it easy to find the generator source. code.putln("/*--- %s ---*/" % self.description) return code @@ -2613,7 +3299,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): EncodedString(decode_filename( os.path.dirname(module_path)))).cname, code.error_goto_if_null(temp, self.pos))) - code.put_gotref(temp) + code.put_gotref(temp, py_object_type) code.putln( 'if (PyObject_SetAttrString(%s, "__path__", %s) < 0) %s;' % ( env.module_cname, temp, code.error_goto(self.pos))) @@ -2637,14 +3323,15 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # CPython may not have put us into sys.modules yet, but relative imports and reimports require it fq_module_name = self.full_module_name if fq_module_name.endswith('.__init__'): - fq_module_name = fq_module_name[:-len('.__init__')] + fq_module_name = EncodedString(fq_module_name[:-len('.__init__')]) + fq_module_name_cstring = fq_module_name.as_c_string_literal() code.putln("#if PY_MAJOR_VERSION >= 3") code.putln("{") code.putln("PyObject *modules = PyImport_GetModuleDict(); %s" % code.error_goto_if_null("modules", self.pos)) - code.putln('if (!PyDict_GetItemString(modules, "%s")) {' % fq_module_name) - code.putln(code.error_goto_if_neg('PyDict_SetItemString(modules, "%s", %s)' % ( - fq_module_name, env.module_cname), self.pos)) + code.putln('if (!PyDict_GetItemString(modules, %s)) {' % fq_module_name_cstring) + code.putln(code.error_goto_if_neg('PyDict_SetItemString(modules, %s, %s)' % ( + fq_module_name_cstring, env.module_cname), self.pos)) code.putln("}") code.putln("}") code.putln("#endif") @@ -2717,11 +3404,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if Options.pre_import is not None: code.put_decref_clear(Naming.preimport_cname, py_object_type, nanny=False, clear_before_decref=True) - for cname in [env.module_dict_cname, Naming.cython_runtime_cname, Naming.builtins_cname]: + for cname in [Naming.cython_runtime_cname, Naming.builtins_cname]: code.put_decref_clear(cname, py_object_type, nanny=False, clear_before_decref=True) + code.put_decref_clear(env.module_dict_cname, py_object_type, nanny=False, clear_before_decref=True) def generate_main_method(self, env, code): - module_is_main = "%s%s" % (Naming.module_is_main, self.full_module_name.replace('.', '__')) + module_is_main = self.is_main_module_flag_cname() if Options.embed == "main": wmain = "wmain" else: @@ -2734,8 +3422,33 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): main_method=Options.embed, wmain_method=wmain)) + def punycode_module_name(self, prefix, name): + # adapted from PEP483 + try: + name = '_' + name.encode('ascii').decode('ascii') + except UnicodeEncodeError: + name = 'U_' + name.encode('punycode').replace(b'-', b'_').decode('ascii') + return "%s%s" % (prefix, name) + + def wrong_punycode_module_name(self, name): + # to work around a distutils bug by also generating an incorrect symbol... + try: + name.encode("ascii") + return None # workaround is not needed + except UnicodeEncodeError: + return "PyInitU" + (u"_"+name).encode('punycode').replace(b'-', b'_').decode('ascii') + def mod_init_func_cname(self, prefix, env): - return '%s_%s' % (prefix, env.module_name) + # from PEP483 + return self.punycode_module_name(prefix, env.module_name) + + # Returns the name of the C-function that corresponds to the module initialisation. + # (module initialisation == the cython code outside of functions) + # Note that this should never be the name of a wrapper and always the name of the + # function containing the actual code. Otherwise, cygdb will experience problems. + def module_init_func_cname(self): + env = self.scope + return self.mod_init_func_cname(Naming.pymodule_exec_func_cname, env) def generate_pymoduledef_struct(self, env, code): if env.doc: @@ -2750,7 +3463,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("") code.putln("#if PY_MAJOR_VERSION >= 3") code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT") - exec_func_cname = self.mod_init_func_cname(Naming.pymodule_exec_func_cname, env) + exec_func_cname = self.module_init_func_cname() code.putln("static PyObject* %s(PyObject *spec, PyModuleDef *def); /*proto*/" % Naming.pymodule_create_func_cname) code.putln("static int %s(PyObject* module); /*proto*/" % exec_func_cname) @@ -2760,15 +3473,27 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("{Py_mod_exec, (void*)%s}," % exec_func_cname) code.putln("{0, NULL}") code.putln("};") + if not env.module_name.isascii(): + code.putln("#else /* CYTHON_PEP489_MULTI_PHASE_INIT */") + code.putln('#error "Unicode module names are only supported with multi-phase init' + ' as per PEP489"') code.putln("#endif") code.putln("") - code.putln("static struct PyModuleDef %s = {" % Naming.pymoduledef_cname) + code.putln('#ifdef __cplusplus') + code.putln('namespace {') + code.putln("struct PyModuleDef %s =" % Naming.pymoduledef_cname) + code.putln('#else') + code.putln("static struct PyModuleDef %s =" % Naming.pymoduledef_cname) + code.putln('#endif') + code.putln('{') code.putln(" PyModuleDef_HEAD_INIT,") - code.putln(' "%s",' % env.module_name) + code.putln(' %s,' % env.module_name.as_c_string_literal()) code.putln(" %s, /* m_doc */" % doc) code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT") code.putln(" 0, /* m_size */") + code.putln("#elif CYTHON_USE_MODULE_STATE") # FIXME: should allow combination with PEP-489 + code.putln(" sizeof(%s), /* m_size */" % Naming.modulestate_cname) code.putln("#else") code.putln(" -1, /* m_size */") code.putln("#endif") @@ -2778,10 +3503,19 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#else") code.putln(" NULL, /* m_reload */") code.putln("#endif") + code.putln("#if CYTHON_USE_MODULE_STATE") + code.putln(" %s_traverse, /* m_traverse */" % Naming.module_cname) + code.putln(" %s_clear, /* m_clear */" % Naming.module_cname) + code.putln(" %s /* m_free */" % cleanup_func) + code.putln("#else") code.putln(" NULL, /* m_traverse */") code.putln(" NULL, /* m_clear */") code.putln(" %s /* m_free */" % cleanup_func) + code.putln("#endif") code.putln("};") + code.putln('#ifdef __cplusplus') + code.putln('} /* anonymous namespace */') + code.putln('#endif') code.putln("#endif") def generate_module_creation_code(self, env, code): @@ -2800,20 +3534,44 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#else") code.putln("#if PY_MAJOR_VERSION < 3") code.putln( - '%s = Py_InitModule4("%s", %s, %s, 0, PYTHON_API_VERSION); Py_XINCREF(%s);' % ( + '%s = Py_InitModule4(%s, %s, %s, 0, PYTHON_API_VERSION); Py_XINCREF(%s);' % ( env.module_cname, - env.module_name, + env.module_name.as_c_string_literal(), env.method_table_cname, doc, env.module_cname)) - code.putln("#else") + code.putln(code.error_goto_if_null(env.module_cname, self.pos)) + code.putln("#elif CYTHON_USE_MODULE_STATE") + # manage_ref is False (and refnanny calls are omitted) because refnanny isn't yet initialized + module_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=False) + code.putln( + "%s = PyModule_Create(&%s); %s" % ( + module_temp, + Naming.pymoduledef_cname, + code.error_goto_if_null(module_temp, self.pos))) + code.putln("{") + # So that PyState_FindModule works in the init function: + code.putln("int add_module_result = PyState_AddModule(%s, &%s);" % ( + module_temp, Naming.pymoduledef_cname)) + code.putln("%s = 0; /* transfer ownership from %s to %s pseudovariable */" % ( + module_temp, module_temp, env.module_name + )) + # At this stage the module likely has a refcount of 2 - one owned by the list + # inside PyState_AddModule and one owned by "__pyx_m" (and returned from this + # function as a new reference). + code.putln(code.error_goto_if_neg("add_module_result", self.pos)) + code.putln("pystate_addmodule_run = 1;") + code.putln("}") + code.funcstate.release_temp(module_temp) + code.putln('#else') code.putln( "%s = PyModule_Create(&%s);" % ( env.module_cname, Naming.pymoduledef_cname)) - code.putln("#endif") code.putln(code.error_goto_if_null(env.module_cname, self.pos)) + code.putln("#endif") code.putln("#endif") # CYTHON_PEP489_MULTI_PHASE_INIT + code.putln("CYTHON_UNUSED_VAR(%s);" % module_temp) # only used in limited API code.putln( "%s = PyModule_GetDict(%s); %s" % ( @@ -2903,8 +3661,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # investigation shows that the resulting binary is smaller with repeated functions calls. for entry in entries: signature = entry.type.signature_string() - code.putln('if (__Pyx_ExportFunction("%s", (void (*)(void))%s, "%s") < 0) %s' % ( - entry.name, + code.putln('if (__Pyx_ExportFunction(%s, (void (*)(void))%s, "%s") < 0) %s' % ( + entry.name.as_c_string_literal(), entry.cname, signature, code.error_goto(self.pos))) @@ -2946,7 +3704,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): module.qualified_name, temp, code.error_goto(self.pos))) - code.put_gotref(temp) + code.put_gotref(temp, py_object_type) for entry in entries: if env is module: cname = entry.cname @@ -2977,13 +3735,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): module.qualified_name, temp, code.error_goto(self.pos))) - code.put_gotref(temp) + code.put_gotref(temp, py_object_type) for entry in entries: code.putln( - 'if (__Pyx_ImportFunction_%s(%s, "%s", (void (**)(void))&%s, "%s") < 0) %s' % ( + 'if (__Pyx_ImportFunction_%s(%s, %s, (void (**)(void))&%s, "%s") < 0) %s' % ( Naming.cyversion, temp, - entry.name, + entry.name.as_c_string_literal(), entry.cname, entry.type.signature_string(), code.error_goto(self.pos))) @@ -3006,7 +3764,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): def generate_base_type_import_code(self, env, entry, code, import_generator): base_type = entry.type.base_type if (base_type and base_type.module_name != env.qualified_name and not - base_type.is_builtin_type and not entry.utility_code_definition): + (base_type.is_builtin_type or base_type.is_cython_builtin_type) + and not entry.utility_code_definition): self.generate_type_import_code(env, base_type, self.pos, code, import_generator) def generate_type_import_code(self, env, type, pos, code, import_generator): @@ -3023,7 +3782,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if type.vtabptr_cname: code.globalstate.use_utility_code( UtilityCode.load_cached('GetVTable', 'ImportExport.c')) - code.putln("%s = (struct %s*)__Pyx_GetVtable(%s->tp_dict); %s" % ( + code.putln("%s = (struct %s*)__Pyx_GetVtable(%s); %s" % ( type.vtabptr_cname, type.vtabstruct_cname, type.typeptr_cname, @@ -3064,15 +3823,17 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): module, module_name)) + type_name = type.name.as_c_string_literal() + if condition and replacement: code.putln("") # start in new line code.putln("#if %s" % condition) code.putln('"%s",' % replacement) code.putln("#else") - code.putln('"%s",' % type.name) + code.putln('%s,' % type_name) code.putln("#endif") else: - code.put(' "%s", ' % type.name) + code.put(' %s, ' % type_name) if sizeof_objstruct != objstruct: if not condition: @@ -3080,6 +3841,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000") code.putln('sizeof(%s), __PYX_GET_STRUCT_ALIGNMENT_%s(%s),' % ( objstruct, Naming.cyversion, objstruct)) + code.putln("#elif CYTHON_COMPILING_IN_LIMITED_API") + code.putln('sizeof(%s), __PYX_GET_STRUCT_ALIGNMENT_%s(%s),' % ( + objstruct, Naming.cyversion, objstruct)) code.putln("#else") code.putln('sizeof(%s), __PYX_GET_STRUCT_ALIGNMENT_%s(%s),' % ( sizeof_objstruct, Naming.cyversion, sizeof_objstruct)) @@ -3104,6 +3868,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): def generate_type_ready_code(self, entry, code): Nodes.CClassDefNode.generate_type_ready_code(entry, code) + def is_main_module_flag_cname(self): + full_module_name = self.full_module_name.replace('.', '__') + return self.punycode_module_name(Naming.module_is_main, full_module_name) + def generate_exttype_vtable_init_code(self, entry, code): # Generate code to initialise the C method table of an # extension type. @@ -3156,7 +3924,7 @@ class ModuleImportGenerator(object): self.temps.append(temp) code.putln('%s = PyImport_ImportModule(%s); if (unlikely(!%s)) %s' % ( temp, module_name_string, temp, error_code)) - code.put_gotref(temp) + code.put_gotref(temp, py_object_type) self.imported[module_name_string] = temp return temp @@ -3216,5 +3984,3 @@ packed_struct_utility_code = UtilityCode(proto=""" #define __Pyx_PACKED #endif """, impl="", proto_block='utility_code_proto_before_types') - -capsule_utility_code = UtilityCode.load("Capsule") diff --git a/Cython/Compiler/Naming.py b/Cython/Compiler/Naming.py index 4dd6cbbd5..140fe0435 100644 --- a/Cython/Compiler/Naming.py +++ b/Cython/Compiler/Naming.py @@ -15,8 +15,11 @@ codewriter_temp_prefix = pyrex_prefix + "t_" temp_prefix = u"__cyt_" +pyunicode_identifier_prefix = pyrex_prefix + 'U' + builtin_prefix = pyrex_prefix + "builtin_" arg_prefix = pyrex_prefix + "arg_" +genexpr_arg_prefix = pyrex_prefix + "genexpr_arg_" funcdoc_prefix = pyrex_prefix + "doc_" enum_prefix = pyrex_prefix + "e_" func_prefix = pyrex_prefix + "f_" @@ -47,12 +50,18 @@ pybufferstruct_prefix = pyrex_prefix + "pybuffer_" vtable_prefix = pyrex_prefix + "vtable_" vtabptr_prefix = pyrex_prefix + "vtabptr_" vtabstruct_prefix = pyrex_prefix + "vtabstruct_" +unicode_vtabentry_prefix = pyrex_prefix + "Uvtabentry_" +# vtab entries aren't normally mangled, +# but punycode names sometimes start with numbers leading to a C syntax error +unicode_structmember_prefix = pyrex_prefix + "Umember_" +# as above - +# not normally mangled but punycode names cause specific problems opt_arg_prefix = pyrex_prefix + "opt_args_" convert_func_prefix = pyrex_prefix + "convert_" closure_scope_prefix = pyrex_prefix + "scope_" closure_class_prefix = pyrex_prefix + "scope_struct_" lambda_func_prefix = pyrex_prefix + "lambda_" -module_is_main = pyrex_prefix + "module_is_main_" +module_is_main = pyrex_prefix + "module_is_main" defaults_struct_prefix = pyrex_prefix + "defaults" dynamic_args_cname = pyrex_prefix + "dynamic_args" @@ -69,6 +78,8 @@ interned_prefixes = { ctuple_type_prefix = pyrex_prefix + "ctuple_" args_cname = pyrex_prefix + "args" +nargs_cname = pyrex_prefix + "nargs" +kwvalues_cname = pyrex_prefix + "kwvalues" generator_cname = pyrex_prefix + "generator" sent_value_cname = pyrex_prefix + "sent_value" pykwdlist_cname = pyrex_prefix + "pyargnames" @@ -87,6 +98,8 @@ clineno_cname = pyrex_prefix + "clineno" cfilenm_cname = pyrex_prefix + "cfilenm" local_tstate_cname = pyrex_prefix + "tstate" module_cname = pyrex_prefix + "m" +modulestate_cname = pyrex_prefix + "mstate" +modulestateglobal_cname = pyrex_prefix + "mstate_global" moddoc_cname = pyrex_prefix + "mdoc" methtable_cname = pyrex_prefix + "methods" retval_cname = pyrex_prefix + "r" @@ -99,7 +112,7 @@ gilstate_cname = pyrex_prefix + "state" skip_dispatch_cname = pyrex_prefix + "skip_dispatch" empty_tuple = pyrex_prefix + "empty_tuple" empty_bytes = pyrex_prefix + "empty_bytes" -empty_unicode = pyrex_prefix + "empty_unicode" +empty_unicode = pyrex_prefix + "empty_unicode" print_function = pyrex_prefix + "print" print_function_kwargs = pyrex_prefix + "print_kwargs" cleanup_cname = pyrex_prefix + "module_cleanup" @@ -116,13 +129,20 @@ cur_scope_cname = pyrex_prefix + "cur_scope" enc_scope_cname = pyrex_prefix + "enc_scope" frame_cname = pyrex_prefix + "frame" frame_code_cname = pyrex_prefix + "frame_code" +error_without_exception_cname = pyrex_prefix + "error_without_exception" binding_cfunc = pyrex_prefix + "binding_PyCFunctionType" fused_func_prefix = pyrex_prefix + 'fuse_' -quick_temp_cname = pyrex_prefix + "temp" # temp variable for quick'n'dirty temping +quick_temp_cname = pyrex_prefix + "temp" # temp variable for quick'n'dirty temping tp_dict_version_temp = pyrex_prefix + "tp_dict_version" obj_dict_version_temp = pyrex_prefix + "obj_dict_version" -type_dict_guard_temp = pyrex_prefix + "type_dict_guard" +type_dict_guard_temp = pyrex_prefix + "typedict_guard" cython_runtime_cname = pyrex_prefix + "cython_runtime" +cyfunction_type_cname = pyrex_prefix + "CyFunctionType" +fusedfunction_type_cname = pyrex_prefix + "FusedFunctionType" +# the name "dflt" was picked by analogy with the CPython dataclass module which stores +# the default values in variables named f"_dflt_{field.name}" in a hidden scope that's +# passed to the __init__ function. (The name is unimportant to the exact workings though) +dataclass_field_default_cname = pyrex_prefix + "dataclass_dflt" global_code_object_cache_find = pyrex_prefix + 'find_code_object' global_code_object_cache_insert = pyrex_prefix + 'insert_code_object' @@ -154,11 +174,24 @@ exc_vars = (exc_type_name, exc_value_name, exc_tb_name) api_name = pyrex_prefix + "capi__" -h_guard_prefix = "__PYX_HAVE__" -api_guard_prefix = "__PYX_HAVE_API__" +# the h and api guards get changed to: +# __PYX_HAVE__FILENAME (for ascii filenames) +# __PYX_HAVE_U_PUNYCODEFILENAME (for non-ascii filenames) +h_guard_prefix = "__PYX_HAVE_" +api_guard_prefix = "__PYX_HAVE_API_" api_func_guard = "__PYX_HAVE_API_FUNC_" PYX_NAN = "__PYX_NAN()" def py_version_hex(major, minor=0, micro=0, release_level=0, release_serial=0): return (major << 24) | (minor << 16) | (micro << 8) | (release_level << 4) | (release_serial) + +# there's a few places where it's useful to iterate over all of these +used_types_and_macros = [ + (cyfunction_type_cname, '__Pyx_CyFunction_USED'), + (fusedfunction_type_cname, '__Pyx_FusedFunction_USED'), + ('__pyx_GeneratorType', '__Pyx_Generator_USED'), + ('__pyx_IterableCoroutineType', '__Pyx_IterableCoroutine_USED'), + ('__pyx_CoroutineAwaitType', '__Pyx_Coroutine_USED'), + ('__pyx_CoroutineType', '__Pyx_Coroutine_USED'), +] diff --git a/Cython/Compiler/Nodes.py b/Cython/Compiler/Nodes.py index 457ae94ad..230226298 100644 --- a/Cython/Compiler/Nodes.py +++ b/Cython/Compiler/Nodes.py @@ -5,6 +5,7 @@ from __future__ import absolute_import import cython + cython.declare(sys=object, os=object, copy=object, Builtin=object, error=object, warning=object, Naming=object, PyrexTypes=object, py_object_type=object, ModuleScope=object, LocalScope=object, ClosureScope=object, @@ -12,17 +13,18 @@ cython.declare(sys=object, os=object, copy=object, CppClassScope=object, UtilityCode=object, EncodedString=object, error_type=object, _py_int_types=object) -import sys, os, copy +import sys, copy from itertools import chain from . import Builtin -from .Errors import error, warning, InternalError, CompileError +from .Errors import error, warning, InternalError, CompileError, CannotSpecialize from . import Naming from . import PyrexTypes from . import TypeSlots from .PyrexTypes import py_object_type, error_type -from .Symtab import (ModuleScope, LocalScope, ClosureScope, - StructOrUnionScope, PyClassScope, CppClassScope, TemplateScope) +from .Symtab import (ModuleScope, LocalScope, ClosureScope, PropertyScope, + StructOrUnionScope, PyClassScope, CppClassScope, TemplateScope, GeneratorExpressionScope, + CppScopedEnumScope, punycodify_name) from .Code import UtilityCode from .StringEncoding import EncodedString from . import Future @@ -38,6 +40,9 @@ else: _py_int_types = (int, long) +IMPLICIT_CLASSMETHODS = {"__init_subclass__", "__class_getitem__"} + + def relative_position(pos): return (pos[0].get_filenametable_entry(), pos[1]) @@ -68,53 +73,6 @@ def embed_position(pos, docstring): return doc -def analyse_type_annotation(annotation, env, assigned_value=None): - base_type = None - is_ambiguous = False - explicit_pytype = explicit_ctype = False - if annotation.is_dict_literal: - warning(annotation.pos, - "Dicts should no longer be used as type annotations. Use 'cython.int' etc. directly.") - for name, value in annotation.key_value_pairs: - if not name.is_string_literal: - continue - if name.value in ('type', b'type'): - explicit_pytype = True - if not explicit_ctype: - annotation = value - elif name.value in ('ctype', b'ctype'): - explicit_ctype = True - annotation = value - if explicit_pytype and explicit_ctype: - warning(annotation.pos, "Duplicate type declarations found in signature annotation") - arg_type = annotation.analyse_as_type(env) - if annotation.is_name and not annotation.cython_attribute and annotation.name in ('int', 'long', 'float'): - # Map builtin numeric Python types to C types in safe cases. - if assigned_value is not None and arg_type is not None and not arg_type.is_pyobject: - assigned_type = assigned_value.infer_type(env) - if assigned_type and assigned_type.is_pyobject: - # C type seems unsafe, e.g. due to 'None' default value => ignore annotation type - is_ambiguous = True - arg_type = None - # ignore 'int' and require 'cython.int' to avoid unsafe integer declarations - if arg_type in (PyrexTypes.c_long_type, PyrexTypes.c_int_type, PyrexTypes.c_float_type): - arg_type = PyrexTypes.c_double_type if annotation.name == 'float' else py_object_type - elif arg_type is not None and annotation.is_string_literal: - warning(annotation.pos, - "Strings should no longer be used for type declarations. Use 'cython.int' etc. directly.") - if arg_type is not None: - if explicit_pytype and not explicit_ctype and not arg_type.is_pyobject: - warning(annotation.pos, - "Python type declaration in signature annotation does not refer to a Python type") - base_type = CAnalysedBaseTypeNode( - annotation.pos, type=arg_type, is_arg=True) - elif is_ambiguous: - warning(annotation.pos, "Ambiguous types in annotation, ignoring") - else: - warning(annotation.pos, "Unknown type declaration in annotation, ignoring") - return base_type, arg_type - - def write_func_call(func, codewriter_class): def f(*args, **kwds): if len(args) > 1 and isinstance(args[1], codewriter_class): @@ -125,19 +83,16 @@ def write_func_call(func, codewriter_class): ' ' * code.call_level, node.__class__.__name__, func.__name__, - node.pos[1:]) - pristine = code.buffer.stream.tell() - code.putln(marker) + node.pos[1:], + ) + insertion_point = code.insertion_point() start = code.buffer.stream.tell() code.call_level += 4 res = func(*args, **kwds) code.call_level -= 4 - if start == code.buffer.stream.tell(): - # no code written => undo writing marker - code.buffer.stream.truncate(pristine) - else: - marker = marker.replace('->', '<-', 1) - code.putln(marker) + if start != code.buffer.stream.tell(): + code.putln(marker.replace('->', '<-', 1)) + insertion_point.putln(marker) return res else: return func(*args, **kwds) @@ -160,9 +115,11 @@ class VerboseCodeWriter(type): class CheckAnalysers(type): """Metaclass to check that type analysis functions return a node. """ - methods = set(['analyse_types', - 'analyse_expressions', - 'analyse_target_types']) + methods = frozenset({ + 'analyse_types', + 'analyse_expressions', + 'analyse_target_types', + }) def __new__(cls, name, bases, attrs): from types import FunctionType @@ -200,6 +157,8 @@ class Node(object): is_literal = 0 is_terminator = 0 is_wrapper = False # is a DefNode wrapper for a C function + is_cproperty = False + is_templated_type_node = False temps = None # All descendants should set child_attrs to a list of the attributes @@ -254,7 +213,7 @@ class Node(object): # - # There are 3 phases of parse tree processing, applied in order to + # There are 3 main phases of parse tree processing, applied in order to # all the statements in a given scope-block: # # (0) analyse_declarations @@ -266,25 +225,25 @@ class Node(object): # Determine the result types of expressions and fill in the # 'type' attribute of each ExprNode. Insert coercion nodes into the # tree where needed to convert to and from Python objects. - # Allocate temporary locals for intermediate results. Fill - # in the 'result_code' attribute of each ExprNode with a C code - # fragment. + # Replace tree nodes with more appropriate implementations found by + # the type analysis. # # (2) generate_code # Emit C code for all declarations, statements and expressions. - # Recursively applies the 3 processing phases to the bodies of - # functions. + # + # These phases are triggered by tree transformations. + # See the full pipeline in Pipeline.py. # def analyse_declarations(self, env): pass def analyse_expressions(self, env): - raise InternalError("analyse_expressions not implemented for %s" % \ + raise InternalError("analyse_expressions not implemented for %s" % self.__class__.__name__) def generate_code(self, code): - raise InternalError("generate_code not implemented for %s" % \ + raise InternalError("generate_code not implemented for %s" % self.__class__.__name__) def annotate(self, code): @@ -360,6 +319,7 @@ class Node(object): return u'"%s":%d:%d\n%s\n' % ( source_desc.get_escaped_description(), line, col, u''.join(lines)) + class CompilerDirectivesNode(Node): """ Sets compiler directives for the children nodes @@ -402,6 +362,7 @@ class CompilerDirectivesNode(Node): self.body.annotate(code) code.globalstate.directives = old + class BlockNode(object): # Mixin class for nodes representing a declaration block. @@ -415,14 +376,15 @@ class BlockNode(object): for node in env.lambda_defs: node.generate_function_definitions(env, code) + class StatListNode(Node): # stats a list of StatNode child_attrs = ["stats"] @staticmethod - def create_analysed(pos, env, *args, **kw): - node = StatListNode(pos, *args, **kw) + def create_analysed(pos, env, **kw): + node = StatListNode(pos, **kw) return node # No node-specific analysis needed def analyse_declarations(self, env): @@ -469,7 +431,7 @@ class StatNode(Node): pass def generate_execution_code(self, code): - raise InternalError("generate_execution_code not implemented for %s" % \ + raise InternalError("generate_execution_code not implemented for %s" % self.__class__.__name__) @@ -499,8 +461,13 @@ class CDefExternNode(StatNode): env.add_include_file(self.include_file, self.verbatim_include, late) def analyse_expressions(self, env): + # Allow C properties, inline methods, etc. also in external types. + self.body = self.body.analyse_expressions(env) return self + def generate_function_definitions(self, env, code): + self.body.generate_function_definitions(env, code) + def generate_execution_code(self, code): pass @@ -525,6 +492,9 @@ class CDeclaratorNode(Node): calling_convention = "" + def declared_name(self): + return None + def analyse_templates(self): # Only C++ functions have templates. return None @@ -539,6 +509,9 @@ class CNameDeclaratorNode(CDeclaratorNode): default = None + def declared_name(self): + return self.name + def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False): if nonempty and self.name == '': # May have mistaken the name for the type. @@ -551,7 +524,12 @@ class CNameDeclaratorNode(CDeclaratorNode): base_type = py_object_type if base_type.is_fused and env.fused_to_specific: - base_type = base_type.specialize(env.fused_to_specific) + try: + base_type = base_type.specialize(env.fused_to_specific) + except CannotSpecialize: + error(self.pos, + "'%s' cannot be specialized since its type is not a fused argument to this function" % + self.name) self.type = base_type return self, base_type @@ -562,6 +540,9 @@ class CPtrDeclaratorNode(CDeclaratorNode): child_attrs = ["base"] + def declared_name(self): + return self.base.declared_name() + def analyse_templates(self): return self.base.analyse_templates() @@ -572,14 +553,17 @@ class CPtrDeclaratorNode(CDeclaratorNode): return self.base.analyse(ptr_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd) -class CReferenceDeclaratorNode(CDeclaratorNode): - # base CDeclaratorNode - +class _CReferenceDeclaratorBaseNode(CDeclaratorNode): child_attrs = ["base"] + def declared_name(self): + return self.base.declared_name() + def analyse_templates(self): return self.base.analyse_templates() + +class CReferenceDeclaratorNode(_CReferenceDeclaratorBaseNode): def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False): if base_type.is_pyobject: error(self.pos, "Reference base type cannot be a Python object") @@ -587,6 +571,14 @@ class CReferenceDeclaratorNode(CDeclaratorNode): return self.base.analyse(ref_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd) +class CppRvalueReferenceDeclaratorNode(_CReferenceDeclaratorBaseNode): + def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False): + if base_type.is_pyobject: + error(self.pos, "Rvalue-reference base type cannot be a Python object") + ref_type = PyrexTypes.cpp_rvalue_ref_type(base_type) + return self.base.analyse(ref_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd) + + class CArrayDeclaratorNode(CDeclaratorNode): # base CDeclaratorNode # dimension ExprNode @@ -594,7 +586,9 @@ class CArrayDeclaratorNode(CDeclaratorNode): child_attrs = ["base", "dimension"] def analyse(self, base_type, env, nonempty=0, visibility=None, in_pxd=False): - if (base_type.is_cpp_class and base_type.is_template_type()) or base_type.is_cfunction: + if ((base_type.is_cpp_class and base_type.is_template_type()) or + base_type.is_cfunction or + base_type.python_type_constructor_name): from .ExprNodes import TupleNode if isinstance(self.dimension, TupleNode): args = self.dimension.args @@ -606,7 +600,7 @@ class CArrayDeclaratorNode(CDeclaratorNode): error(args[ix].pos, "Template parameter not a type") base_type = error_type else: - base_type = base_type.specialize_here(self.pos, values) + base_type = base_type.specialize_here(self.pos, env, values) return self.base.analyse(base_type, env, nonempty=nonempty, visibility=visibility, in_pxd=in_pxd) if self.dimension: self.dimension = self.dimension.analyse_const_expression(env) @@ -636,8 +630,8 @@ class CFuncDeclaratorNode(CDeclaratorNode): # args [CArgDeclNode] # templates [TemplatePlaceholderType] # has_varargs boolean - # exception_value ConstNode - # exception_check boolean True if PyErr_Occurred check needed + # exception_value ConstNode or NameNode NameNode when the name of a c++ exception conversion function + # exception_check boolean or "+" True if PyErr_Occurred check needed, "+" for a c++ check # nogil boolean Can be called without gil # with_gil boolean Acquire gil around function body # is_const_method boolean Whether this is a const method @@ -649,6 +643,9 @@ class CFuncDeclaratorNode(CDeclaratorNode): is_const_method = 0 templates = None + def declared_name(self): + return self.base.declared_name() + def analyse_templates(self): if isinstance(self.base, CArrayDeclaratorNode): from .ExprNodes import TupleNode, NameNode @@ -718,6 +715,12 @@ class CFuncDeclaratorNode(CDeclaratorNode): env.add_include_file('new') # for std::bad_alloc env.add_include_file('stdexcept') env.add_include_file('typeinfo') # for std::bad_cast + elif return_type.is_pyobject and self.exception_check: + # Functions in pure Python mode default to always check return values for exceptions + # (equivalent to the "except*" declaration). In this case, the exception clause + # is silently ignored for functions returning a Python object. + self.exception_check = False + if (return_type.is_pyobject and (self.exception_value or self.exception_check) and self.exception_check != '+'): @@ -727,15 +730,21 @@ class CFuncDeclaratorNode(CDeclaratorNode): # Use an explicit exception return value to speed up exception checks. # Even if it is not declared, we can use the default exception value of the return type, # unless the function is some kind of external function that we do not control. - if return_type.exception_value is not None and (visibility != 'extern' and not in_pxd): - # Extension types are more difficult because the signature must match the base type signature. - if not env.is_c_class_scope: + if (return_type.exception_value is not None and (visibility != 'extern' and not in_pxd)): + # - We skip this optimization for extension types; they are more difficult because + # the signature must match the base type signature. + # - Same for function pointers, as we want them to be able to match functions + # with any exception value. + # - Ideally the function-pointer test would be better after self.base is analysed + # however that is hard to do with the current implementation so it lives here + # for now. + if not env.is_c_class_scope and not isinstance(self.base, CPtrDeclaratorNode): from .ExprNodes import ConstNode self.exception_value = ConstNode( self.pos, value=return_type.exception_value, type=return_type) if self.exception_value: - self.exception_value = self.exception_value.analyse_const_expression(env) if self.exception_check == '+': + self.exception_value = self.exception_value.analyse_const_expression(env) exc_val_type = self.exception_value.type if (not exc_val_type.is_error and not exc_val_type.is_pyobject @@ -745,19 +754,28 @@ class CFuncDeclaratorNode(CDeclaratorNode): and not (exc_val_type == PyrexTypes.c_char_type and self.exception_value.value == '*')): error(self.exception_value.pos, - "Exception value must be a Python exception or cdef function with no arguments or *.") + "Exception value must be a Python exception, or C++ function with no arguments, or *.") exc_val = self.exception_value else: - self.exception_value = self.exception_value.coerce_to( + self.exception_value = self.exception_value.analyse_types(env).coerce_to( return_type, env).analyse_const_expression(env) exc_val = self.exception_value.get_constant_c_result_code() if exc_val is None: - raise InternalError( - "get_constant_c_result_code not implemented for %s" % - self.exception_value.__class__.__name__) + error(self.exception_value.pos, "Exception value must be constant") if not return_type.assignable_from(self.exception_value.type): error(self.exception_value.pos, "Exception value incompatible with function return type") + if (visibility != 'extern' + and (return_type.is_int or return_type.is_float) + and self.exception_value.has_constant_result()): + try: + type_default_value = float(return_type.default_value) + except ValueError: + pass + else: + if self.exception_value.constant_result == type_default_value: + warning(self.pos, "Ambiguous exception value, same as default return value: %r" % + self.exception_value.constant_result) exc_check = self.exception_check if return_type.is_cfunction: error(self.pos, "Function cannot return a function") @@ -789,6 +807,13 @@ class CFuncDeclaratorNode(CDeclaratorNode): error(self.pos, "cannot have both '%s' and '%s' " "calling conventions" % (current, callspec)) func_type.calling_convention = callspec + + if func_type.return_type.is_rvalue_reference: + warning(self.pos, "Rvalue-reference as function return type not supported", 1) + for arg in func_type.args: + if arg.type.is_rvalue_reference and not arg.is_forwarding_reference(): + warning(self.pos, "Rvalue-reference as function argument not supported", 1) + return self.base.analyse(func_type, env, visibility=visibility, in_pxd=in_pxd) def declare_optional_arg_struct(self, func_type, env, fused_cname=None): @@ -850,8 +875,12 @@ class CArgDeclNode(Node): # annotation ExprNode or None Py3 function arg annotation # is_self_arg boolean Is the "self" arg of an extension type method # is_type_arg boolean Is the "class" arg of an extension type classmethod - # is_kw_only boolean Is a keyword-only argument + # kw_only boolean Is a keyword-only argument # is_dynamic boolean Non-literal arg stored inside CyFunction + # pos_only boolean Is a positional-only argument + # + # name_cstring property that converts the name to a cstring taking care of unicode + # and quoting it child_attrs = ["base_type", "declarator", "default", "annotation"] outer_attrs = ["default", "annotation"] @@ -859,7 +888,9 @@ class CArgDeclNode(Node): is_self_arg = 0 is_type_arg = 0 is_generic = 1 + is_special_method_optional = False kw_only = 0 + pos_only = 0 not_none = 0 or_none = 0 type = None @@ -868,59 +899,106 @@ class CArgDeclNode(Node): annotation = None is_dynamic = 0 + def declared_name(self): + return self.declarator.declared_name() + + @property + def name_cstring(self): + return self.name.as_c_string_literal() + + @property + def hdr_cname(self): + # done lazily - needs self.entry to be set to get the class-mangled + # name, which means it has to be generated relatively late + if self.needs_conversion: + return punycodify_name(Naming.arg_prefix + self.entry.name) + else: + return punycodify_name(Naming.var_prefix + self.entry.name) + + def analyse(self, env, nonempty=0, is_self_arg=False): if is_self_arg: - self.base_type.is_self_arg = self.is_self_arg = True - if self.type is None: - # The parser may misinterpret names as types. We fix that here. - if isinstance(self.declarator, CNameDeclaratorNode) and self.declarator.name == '': - if nonempty: - if self.base_type.is_basic_c_type: - # char, short, long called "int" - type = self.base_type.analyse(env, could_be_name=True) - arg_name = type.empty_declaration_code() - else: - arg_name = self.base_type.name - self.declarator.name = EncodedString(arg_name) - self.base_type.name = None - self.base_type.is_basic_c_type = False - could_be_name = True - else: - could_be_name = False - self.base_type.is_arg = True - base_type = self.base_type.analyse(env, could_be_name=could_be_name) - if hasattr(self.base_type, 'arg_name') and self.base_type.arg_name: - self.declarator.name = self.base_type.arg_name - - # The parser is unable to resolve the ambiguity of [] as part of the - # type (e.g. in buffers) or empty declarator (as with arrays). - # This is only arises for empty multi-dimensional arrays. - if (base_type.is_array - and isinstance(self.base_type, TemplatedTypeNode) - and isinstance(self.declarator, CArrayDeclaratorNode)): - declarator = self.declarator - while isinstance(declarator.base, CArrayDeclaratorNode): - declarator = declarator.base - declarator.base = self.base_type.array_declarator - base_type = base_type.base_type + self.base_type.is_self_arg = self.is_self_arg = is_self_arg + if self.type is not None: + return self.name_declarator, self.type - # inject type declaration from annotations - # this is called without 'env' by AdjustDefByDirectives transform before declaration analysis - if self.annotation and env and env.directives['annotation_typing'] and self.base_type.name is None: - arg_type = self.inject_type_from_annotations(env) - if arg_type is not None: - base_type = arg_type - return self.declarator.analyse(base_type, env, nonempty=nonempty) + # The parser may misinterpret names as types. We fix that here. + if isinstance(self.declarator, CNameDeclaratorNode) and self.declarator.name == '': + if nonempty: + if self.base_type.is_basic_c_type: + # char, short, long called "int" + type = self.base_type.analyse(env, could_be_name=True) + arg_name = type.empty_declaration_code() + else: + arg_name = self.base_type.name + self.declarator.name = EncodedString(arg_name) + self.base_type.name = None + self.base_type.is_basic_c_type = False + could_be_name = True else: - return self.name_declarator, self.type + could_be_name = False + self.base_type.is_arg = True + base_type = self.base_type.analyse(env, could_be_name=could_be_name) + base_arg_name = getattr(self.base_type, 'arg_name', None) + if base_arg_name: + self.declarator.name = base_arg_name + + # The parser is unable to resolve the ambiguity of [] as part of the + # type (e.g. in buffers) or empty declarator (as with arrays). + # This is only arises for empty multi-dimensional arrays. + if (base_type.is_array + and isinstance(self.base_type, TemplatedTypeNode) + and isinstance(self.declarator, CArrayDeclaratorNode)): + declarator = self.declarator + while isinstance(declarator.base, CArrayDeclaratorNode): + declarator = declarator.base + declarator.base = self.base_type.array_declarator + base_type = base_type.base_type + + # inject type declaration from annotations + # this is called without 'env' by AdjustDefByDirectives transform before declaration analysis + if (self.annotation and env and env.directives['annotation_typing'] + # CSimpleBaseTypeNode has a name attribute; CAnalysedBaseTypeNode + # (and maybe other options) doesn't + and getattr(self.base_type, "name", None) is None): + arg_type = self.inject_type_from_annotations(env) + if arg_type is not None: + base_type = arg_type + return self.declarator.analyse(base_type, env, nonempty=nonempty) def inject_type_from_annotations(self, env): annotation = self.annotation if not annotation: return None - base_type, arg_type = analyse_type_annotation(annotation, env, assigned_value=self.default) - if base_type is not None: - self.base_type = base_type + + modifiers, arg_type = annotation.analyse_type_annotation(env, assigned_value=self.default) + if arg_type is not None: + self.base_type = CAnalysedBaseTypeNode( + annotation.pos, type=arg_type, is_arg=True) + + if arg_type: + if "typing.Optional" in modifiers: + # "x: Optional[...]" => explicitly allow 'None' + arg_type = arg_type.resolve() + if arg_type and not arg_type.is_pyobject: + # We probably already reported this as "cannot be applied to non-Python type". + # error(annotation.pos, "Only Python type arguments can use typing.Optional[...]") + pass + else: + self.or_none = True + elif arg_type is py_object_type: + # exclude ": object" from the None check - None is a generic object. + self.or_none = True + elif self.default and self.default.is_none and (arg_type.is_pyobject or arg_type.equivalent_type): + # "x: ... = None" => implicitly allow 'None' + if not arg_type.is_pyobject: + arg_type = arg_type.equivalent_type + if not self.or_none: + warning(self.pos, "PEP-484 recommends 'typing.Optional[...]' for arguments that can be None.") + self.or_none = True + elif arg_type.is_pyobject and not self.or_none: + self.not_none = True + return arg_type def calculate_default_value_code(self, code): @@ -947,8 +1025,7 @@ class CArgDeclNode(Node): default.make_owned_reference(code) result = default.result() if overloaded_assignment else default.result_as(self.type) code.putln("%s = %s;" % (target, result)) - if self.type.is_pyobject: - code.put_giveref(default.result()) + code.put_giveref(default.result(), self.type) default.generate_post_assignment_code(code) default.free_temps(code) @@ -989,6 +1066,7 @@ class CSimpleBaseTypeNode(CBaseTypeNode): module_path = [] is_basic_c_type = False complex = False + is_self_arg = False def analyse(self, env, could_be_name=False): # Return type descriptor. @@ -1009,22 +1087,31 @@ class CSimpleBaseTypeNode(CBaseTypeNode): else: type = py_object_type else: + scope = env if self.module_path: # Maybe it's a nested C++ class. - scope = env for item in self.module_path: entry = scope.lookup(item) - if entry is not None and entry.is_cpp_class: + if entry is not None and ( + entry.is_cpp_class or + entry.is_type and entry.type.is_cpp_class + ): scope = entry.type.scope + elif entry and entry.as_module: + scope = entry.as_module else: scope = None break - + if scope is None and len(self.module_path) == 1: + # (may be possible to handle longer module paths?) + # TODO: probably not the best place to declare it? + from .Builtin import get_known_standard_library_module_scope + found_entry = env.lookup(self.module_path[0]) + if found_entry and found_entry.known_standard_library_import: + scope = get_known_standard_library_module_scope(found_entry.known_standard_library_import) if scope is None: # Maybe it's a cimport. scope = env.find_imported_module(self.module_path, self.pos) - else: - scope = env if scope: if scope.is_c_class_scope: @@ -1043,7 +1130,7 @@ class CSimpleBaseTypeNode(CBaseTypeNode): self.arg_name = EncodedString(self.name) else: if self.templates: - if not self.name in self.templates: + if self.name not in self.templates: error(self.pos, "'%s' is not a type identifier" % self.name) type = PyrexTypes.TemplatePlaceholderType(self.name) else: @@ -1063,10 +1150,9 @@ class CSimpleBaseTypeNode(CBaseTypeNode): type = PyrexTypes.c_double_complex_type type.create_declaration_utility_code(env) self.complex = True - if type: - return type - else: - return PyrexTypes.error_type + if not type: + type = PyrexTypes.error_type + return type class MemoryViewSliceTypeNode(CBaseTypeNode): @@ -1135,29 +1221,56 @@ class TemplatedTypeNode(CBaseTypeNode): child_attrs = ["base_type_node", "positional_args", "keyword_args", "dtype_node"] + is_templated_type_node = True dtype_node = None - name = None + def _analyse_template_types(self, env, base_type): + require_python_types = base_type.python_type_constructor_name in ( + 'typing.Optional', + 'dataclasses.ClassVar', + ) + in_c_type_context = env.in_c_type_context and not require_python_types + + template_types = [] + for template_node in self.positional_args: + # CBaseTypeNode -> allow C type declarations in a 'cdef' context again + with env.new_c_type_context(in_c_type_context or isinstance(template_node, CBaseTypeNode)): + ttype = template_node.analyse_as_type(env) + if ttype is None: + if base_type.is_cpp_class: + error(template_node.pos, "unknown type in template argument") + ttype = error_type + # For Python generics we can be a bit more flexible and allow None. + elif require_python_types and not ttype.is_pyobject: + if ttype.equivalent_type and not template_node.as_cython_attribute(): + ttype = ttype.equivalent_type + else: + error(template_node.pos, "%s[...] cannot be applied to non-Python type %s" % ( + base_type.python_type_constructor_name, + ttype, + )) + ttype = error_type + template_types.append(ttype) + + return template_types + def analyse(self, env, could_be_name=False, base_type=None): if base_type is None: base_type = self.base_type_node.analyse(env) if base_type.is_error: return base_type - if base_type.is_cpp_class and base_type.is_template_type(): - # Templated class + if ((base_type.is_cpp_class and base_type.is_template_type()) or + base_type.python_type_constructor_name): + # Templated class, Python generics, etc. if self.keyword_args and self.keyword_args.key_value_pairs: - error(self.pos, "c++ templates cannot take keyword arguments") + tp = "c++ templates" if base_type.is_cpp_class else "indexed types" + error(self.pos, "%s cannot take keyword arguments" % tp) self.type = PyrexTypes.error_type - else: - template_types = [] - for template_node in self.positional_args: - type = template_node.analyse_as_type(env) - if type is None: - error(template_node.pos, "unknown type in template argument") - type = error_type - template_types.append(type) - self.type = base_type.specialize_here(self.pos, template_types) + return self.type + + template_types = self._analyse_template_types(env, base_type) + self.type = base_type.specialize_here(self.pos, env, template_types) elif base_type.is_pyobject: # Buffer @@ -1198,11 +1311,29 @@ class TemplatedTypeNode(CBaseTypeNode): dimension=dimension) self.type = self.array_declarator.analyse(base_type, env)[1] - if self.type.is_fused and env.fused_to_specific: - self.type = self.type.specialize(env.fused_to_specific) + if self.type and self.type.is_fused and env.fused_to_specific: + try: + self.type = self.type.specialize(env.fused_to_specific) + except CannotSpecialize: + error(self.pos, + "'%s' cannot be specialized since its type is not a fused argument to this function" % + self.name) return self.type + def analyse_pytyping_modifiers(self, env): + # Check for declaration modifiers, e.g. "typing.Optional[...]" or "dataclasses.InitVar[...]" + # TODO: somehow bring this together with IndexNode.analyse_pytyping_modifiers() + modifiers = [] + modifier_node = self + while modifier_node.is_templated_type_node and modifier_node.base_type_node and len(modifier_node.positional_args) == 1: + modifier_type = self.base_type_node.analyse_as_type(env) + if modifier_type.python_type_constructor_name and modifier_type.modifier_name: + modifiers.append(modifier_type.modifier_name) + modifier_node = modifier_node.positional_args[0] + + return modifiers + class CComplexBaseTypeNode(CBaseTypeNode): # base_type CBaseTypeNode @@ -1273,8 +1404,10 @@ class FusedTypeNode(CBaseTypeNode): return PyrexTypes.FusedType(types, name=self.name) -class CConstTypeNode(CBaseTypeNode): +class CConstOrVolatileTypeNode(CBaseTypeNode): # base_type CBaseTypeNode + # is_const boolean + # is_volatile boolean child_attrs = ["base_type"] @@ -1282,8 +1415,8 @@ class CConstTypeNode(CBaseTypeNode): base = self.base_type.analyse(env, could_be_name) if base.is_pyobject: error(self.pos, - "Const base type cannot be a Python object") - return PyrexTypes.c_const_type(base) + "Const/volatile base type cannot be a Python object") + return PyrexTypes.c_const_or_volatile_type(base, self.is_const, self.is_volatile) class CVarDefNode(StatNode): @@ -1328,6 +1461,11 @@ class CVarDefNode(StatNode): base_type = self.base_type.analyse(env) + # Check for declaration modifiers, e.g. "typing.Optional[...]" or "dataclasses.InitVar[...]" + modifiers = None + if self.base_type.is_templated_type_node: + modifiers = self.base_type.analyse_pytyping_modifiers(env) + if base_type.is_fused and not self.in_pxd and (env.is_c_class_scope or env.is_module_scope): error(self.pos, "Fused types not allowed here") @@ -1369,6 +1507,8 @@ class CVarDefNode(StatNode): return if type.is_reference and self.visibility != 'extern': error(declarator.pos, "C++ references cannot be declared; use a pointer instead") + if type.is_rvalue_reference and self.visibility != 'extern': + error(declarator.pos, "C++ rvalue-references cannot be declared") if type.is_cfunction: if 'staticmethod' in env.directives: type.is_static_method = True @@ -1383,14 +1523,13 @@ class CVarDefNode(StatNode): self.entry.create_wrapper = True else: if self.overridable: - warning(self.pos, "cpdef variables will not be supported in Cython 3; " - "currently they are no different from cdef variables", 2) + error(self.pos, "Variables cannot be declared with 'cpdef'. Use 'cdef' instead.") if self.directive_locals: error(self.pos, "Decorators can only be followed by functions") self.entry = dest_scope.declare_var( name, type, declarator.pos, cname=cname, visibility=visibility, in_pxd=self.in_pxd, - api=self.api, is_cdef=1) + api=self.api, is_cdef=True, pytyping_modifiers=modifiers) if Options.docstrings: self.entry.doc = embed_position(self.pos, self.doc) @@ -1499,6 +1638,9 @@ class CppClassNode(CStructOrUnionDefNode, BlockNode): elif isinstance(attr, CompilerDirectivesNode): for sub_attr in func_attributes(attr.body.stats): yield sub_attr + elif isinstance(attr, CppClassNode) and attr.attributes is not None: + for sub_attr in func_attributes(attr.attributes): + yield sub_attr if self.attributes is not None: if self.in_pxd and not env.in_cinclude: self.entry.defined_in_pxd = 1 @@ -1529,36 +1671,62 @@ class CppClassNode(CStructOrUnionDefNode, BlockNode): class CEnumDefNode(StatNode): - # name string or None - # cname string or None - # items [CEnumDefItemNode] - # typedef_flag boolean - # visibility "public" or "private" or "extern" - # api boolean - # in_pxd boolean - # create_wrapper boolean - # entry Entry - - child_attrs = ["items"] + # name string or None + # cname string or None + # scoped boolean Is a C++ scoped enum + # underlying_type CSimpleBaseTypeNode The underlying value type (int or C++ type) + # items [CEnumDefItemNode] + # typedef_flag boolean + # visibility "public" or "private" or "extern" + # api boolean + # in_pxd boolean + # create_wrapper boolean + # entry Entry + # doc EncodedString or None Doc string + + child_attrs = ["items", "underlying_type"] + doc = None def declare(self, env): - self.entry = env.declare_enum( - self.name, self.pos, - cname=self.cname, typedef_flag=self.typedef_flag, - visibility=self.visibility, api=self.api, - create_wrapper=self.create_wrapper) + doc = None + if Options.docstrings: + doc = embed_position(self.pos, self.doc) + + self.entry = env.declare_enum( + self.name, self.pos, + cname=self.cname, + scoped=self.scoped, + typedef_flag=self.typedef_flag, + visibility=self.visibility, api=self.api, + create_wrapper=self.create_wrapper, doc=doc) def analyse_declarations(self, env): + scope = None + underlying_type = self.underlying_type.analyse(env) + + if not underlying_type.is_int: + error(self.underlying_type.pos, "underlying type is not an integral type") + + self.entry.type.underlying_type = underlying_type + + if self.scoped and self.items is not None: + scope = CppScopedEnumScope(self.name, env) + scope.type = self.entry.type + else: + scope = env + if self.items is not None: if self.in_pxd and not env.in_cinclude: self.entry.defined_in_pxd = 1 for item in self.items: - item.analyse_declarations(env, self.entry) + item.analyse_declarations(scope, self.entry) def analyse_expressions(self, env): return self def generate_execution_code(self, code): + if self.scoped: + return # nothing to do here for C++ enums if self.visibility == 'public' or self.api: code.mark_pos(self.pos) temp = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True) @@ -1567,7 +1735,7 @@ class CEnumDefNode(StatNode): temp, item.cname, code.error_goto_if_null(temp, item.pos))) - code.put_gotref(temp) + code.put_gotref(temp, PyrexTypes.py_object_type) code.putln('if (PyDict_SetItemString(%s, "%s", %s) < 0) %s' % ( Naming.moddict_cname, item.name, @@ -1590,9 +1758,15 @@ class CEnumDefItemNode(StatNode): if not self.value.type.is_int: self.value = self.value.coerce_to(PyrexTypes.c_int_type, env) self.value = self.value.analyse_const_expression(env) + + if enum_entry.type.is_cpp_enum: + cname = "%s::%s" % (enum_entry.cname, self.name) + else: + cname = self.cname + entry = env.declare_const( self.name, enum_entry.type, - self.value, self.pos, cname=self.cname, + self.value, self.pos, cname=cname, visibility=enum_entry.visibility, api=enum_entry.api, create_wrapper=enum_entry.create_wrapper and enum_entry.name is None) enum_entry.enum_values.append(entry) @@ -1659,6 +1833,9 @@ class FuncDefNode(StatNode, BlockNode): needs_outer_scope = False pymethdef_required = False is_generator = False + is_generator_expression = False # this can be True alongside is_generator + is_coroutine = False + is_asyncgen = False is_generator_body = False is_async_def = False modifiers = [] @@ -1667,6 +1844,9 @@ class FuncDefNode(StatNode, BlockNode): starstar_arg = None is_cyfunction = False code_object = None + return_type_annotation = None + + outer_attrs = None # overridden by some derived classes - to be visited outside the node's scope def analyse_default_values(self, env): default_seen = 0 @@ -1676,6 +1856,10 @@ class FuncDefNode(StatNode, BlockNode): if arg.is_generic: arg.default = arg.default.analyse_types(env) arg.default = arg.default.coerce_to(arg.type, env) + elif arg.is_special_method_optional: + if not arg.default.is_none: + error(arg.pos, "This argument cannot have a non-None default value") + arg.default = None else: error(arg.pos, "This argument cannot have a default value") arg.default = None @@ -1684,18 +1868,12 @@ class FuncDefNode(StatNode, BlockNode): elif default_seen: error(arg.pos, "Non-default argument following default argument") - def analyse_annotation(self, env, annotation): - # Annotations can not only contain valid Python expressions but arbitrary type references. - if annotation is None: - return None - if not env.directives['annotation_typing'] or annotation.analyse_as_type(env) is None: - annotation = annotation.analyse_types(env) - return annotation - def analyse_annotations(self, env): for arg in self.args: if arg.annotation: - arg.annotation = self.analyse_annotation(env, arg.annotation) + arg.annotation = arg.annotation.analyse_types(env) + if self.return_type_annotation: + self.return_type_annotation = self.return_type_annotation.analyse_types(env) def align_argument_type(self, env, arg): # @cython.locals() @@ -1718,6 +1896,9 @@ class FuncDefNode(StatNode, BlockNode): error(type_node.pos, "Previous declaration here") else: arg.type = other_type + if arg.type.is_complex: + # utility code for complex types is special-cased and also important to ensure that it's run + arg.type.create_declaration_utility_code(env) return arg def need_gil_acquisition(self, lenv): @@ -1728,7 +1909,8 @@ class FuncDefNode(StatNode, BlockNode): while genv.is_py_class_scope or genv.is_c_class_scope: genv = genv.outer_scope if self.needs_closure: - lenv = ClosureScope(name=self.entry.name, + cls = GeneratorExpressionScope if self.is_generator_expression else ClosureScope + lenv = cls(name=self.entry.name, outer_scope=genv, parent_scope=env, scope_name=self.entry.cname) @@ -1749,8 +1931,6 @@ class FuncDefNode(StatNode, BlockNode): def generate_function_definitions(self, env, code): from . import Buffer - if self.return_type.is_memoryviewslice: - from . import MemoryView lenv = self.local_scope if lenv.is_closure_scope and not lenv.is_passthrough: @@ -1778,6 +1958,8 @@ class FuncDefNode(StatNode, BlockNode): profile = code.globalstate.directives['profile'] linetrace = code.globalstate.directives['linetrace'] if profile or linetrace: + if linetrace: + code.use_fast_gil_utility_code() code.globalstate.use_utility_code( UtilityCode.load_cached("Profile", "Profile.c")) @@ -1823,14 +2005,20 @@ class FuncDefNode(StatNode, BlockNode): # Initialize the return variable __pyx_r init = "" - if not self.return_type.is_void: - if self.return_type.is_pyobject: + return_type = self.return_type + if return_type.is_cv_qualified and return_type.is_const: + # Within this function body, we want to be able to set this + # variable, even though the function itself needs to return + # a const version + return_type = return_type.cv_base_type + if not return_type.is_void: + if return_type.is_pyobject: init = " = NULL" - elif self.return_type.is_memoryviewslice: - init = ' = ' + MemoryView.memslice_entry_init + elif return_type.is_memoryviewslice: + init = ' = ' + return_type.literal_code(return_type.default_value) code.putln("%s%s;" % ( - self.return_type.declaration_code(Naming.retval_cname), + return_type.declaration_code(Naming.retval_cname), init)) tempvardecl_code = code.insertion_point() @@ -1862,11 +2050,12 @@ class FuncDefNode(StatNode, BlockNode): use_refnanny = not lenv.nogil or lenv.has_with_gil_block + gilstate_decl = None if acquire_gil or acquire_gil_for_var_decls_only: code.put_ensure_gil() code.funcstate.gil_owned = True - elif lenv.nogil and lenv.has_with_gil_block: - code.declare_gilstate() + else: + gilstate_decl = code.insertion_point() if profile or linetrace: if not self.is_generator: @@ -1908,7 +2097,7 @@ class FuncDefNode(StatNode, BlockNode): code.put_incref("Py_None", py_object_type) code.putln(code.error_goto(self.pos)) code.putln("} else {") - code.put_gotref(Naming.cur_scope_cname) + code.put_gotref(Naming.cur_scope_cname, lenv.scope_class.type) code.putln("}") # Note that it is unsafe to decref the scope at this point. if self.needs_outer_scope: @@ -1927,7 +2116,7 @@ class FuncDefNode(StatNode, BlockNode): elif self.needs_closure: # inner closures own a reference to their outer parent code.put_incref(outer_scope_cname, cenv.scope_class.type) - code.put_giveref(outer_scope_cname) + code.put_giveref(outer_scope_cname, cenv.scope_class.type) # ----- Trace function call if profile or linetrace: # this looks a bit late, but if we don't get here due to a @@ -1945,20 +2134,19 @@ class FuncDefNode(StatNode, BlockNode): self.generate_argument_parsing_code(env, code) # If an argument is assigned to in the body, we must # incref it to properly keep track of refcounts. - is_cdef = isinstance(self, CFuncDefNode) for entry in lenv.arg_entries: - if entry.type.is_pyobject: - if (acquire_gil or len(entry.cf_assignments) > 1) and not entry.in_closure: + if not entry.type.is_memoryviewslice: + if (acquire_gil or entry.cf_is_reassigned) and not entry.in_closure: code.put_var_incref(entry) - # Note: defaults are always incref-ed. For def functions, we # we acquire arguments from object conversion, so we have # new references. If we are a cdef function, we need to # incref our arguments - elif is_cdef and entry.type.is_memoryviewslice and len(entry.cf_assignments) > 1: - code.put_incref_memoryviewslice(entry.cname, have_gil=code.funcstate.gil_owned) + elif entry.cf_is_reassigned and not entry.in_closure: + code.put_var_incref_memoryviewslice(entry, + have_gil=code.funcstate.gil_owned) for entry in lenv.var_entries: - if entry.is_arg and len(entry.cf_assignments) > 1 and not entry.in_closure: + if entry.is_arg and entry.cf_is_reassigned and not entry.in_closure: if entry.xdecref_cleanup: code.put_var_xincref(entry) else: @@ -1989,27 +2177,45 @@ class FuncDefNode(StatNode, BlockNode): code.putln("") code.putln("/* function exit code */") + gil_owned = { + 'success': code.funcstate.gil_owned, + 'error': code.funcstate.gil_owned, + 'gil_state_declared': gilstate_decl is None, + } + def assure_gil(code_path, code=code): + if not gil_owned[code_path]: + if not gil_owned['gil_state_declared']: + gilstate_decl.declare_gilstate() + gil_owned['gil_state_declared'] = True + code.put_ensure_gil(declare_gilstate=False) + gil_owned[code_path] = True + # ----- Default return value + return_type = self.return_type if not self.body.is_terminator: - if self.return_type.is_pyobject: - #if self.return_type.is_extension_type: + if return_type.is_pyobject: + #if return_type.is_extension_type: # lhs = "(PyObject *)%s" % Naming.retval_cname #else: lhs = Naming.retval_cname - code.put_init_to_py_none(lhs, self.return_type) - else: - val = self.return_type.default_value + assure_gil('success') + code.put_init_to_py_none(lhs, return_type) + elif not return_type.is_memoryviewslice: + # memory view structs receive their default value on initialisation + val = return_type.default_value if val: code.putln("%s = %s;" % (Naming.retval_cname, val)) - elif not self.return_type.is_void: + elif not return_type.is_void: code.putln("__Pyx_pretend_to_initialize(&%s);" % Naming.retval_cname) + # ----- Error cleanup - if code.error_label in code.labels_used: + if code.label_used(code.error_label): if not self.body.is_terminator: code.put_goto(code.return_label) code.put_label(code.error_label) for cname, type in code.funcstate.all_managed_temps(): - code.put_xdecref(cname, type, have_gil=not lenv.nogil) + assure_gil('error') + code.put_xdecref(cname, type, have_gil=gil_owned['error']) # Clean up buffers -- this calls a Python function # so need to save and restore error state @@ -2019,6 +2225,7 @@ class FuncDefNode(StatNode, BlockNode): code.globalstate.use_utility_code(restore_exception_utility_code) code.putln("{ PyObject *__pyx_type, *__pyx_value, *__pyx_tb;") code.putln("__Pyx_PyThreadState_declare") + assure_gil('error') code.putln("__Pyx_PyThreadState_assign") code.putln("__Pyx_ErrFetch(&__pyx_type, &__pyx_value, &__pyx_tb);") for entry in used_buffer_entries: @@ -2026,7 +2233,8 @@ class FuncDefNode(StatNode, BlockNode): #code.putln("%s = 0;" % entry.cname) code.putln("__Pyx_ErrRestore(__pyx_type, __pyx_value, __pyx_tb);}") - if self.return_type.is_memoryviewslice: + if return_type.is_memoryviewslice: + from . import MemoryView MemoryView.put_init_entry(Naming.retval_cname, code) err_val = Naming.retval_cname else: @@ -2038,104 +2246,136 @@ class FuncDefNode(StatNode, BlockNode): # code.globalstate.use_utility_code(get_exception_tuple_utility_code) # code.put_trace_exception() - if lenv.nogil and not lenv.has_with_gil_block: - code.putln("{") - code.put_ensure_gil() - + assure_gil('error') + if code.funcstate.error_without_exception: + tempvardecl_code.putln( + "int %s = 0; /* StopIteration */" % Naming.error_without_exception_cname + ) + code.putln("if (!%s) {" % Naming.error_without_exception_cname) code.put_add_traceback(self.entry.qualified_name) - - if lenv.nogil and not lenv.has_with_gil_block: - code.put_release_ensured_gil() + if code.funcstate.error_without_exception: code.putln("}") else: warning(self.entry.pos, "Unraisable exception in function '%s'." % self.entry.qualified_name, 0) - code.put_unraisable(self.entry.qualified_name, lenv.nogil) - default_retval = self.return_type.default_value + assure_gil('error') + code.put_unraisable(self.entry.qualified_name) + default_retval = return_type.default_value if err_val is None and default_retval: err_val = default_retval if err_val is not None: if err_val != Naming.retval_cname: code.putln("%s = %s;" % (Naming.retval_cname, err_val)) - elif not self.return_type.is_void: + elif not return_type.is_void: code.putln("__Pyx_pretend_to_initialize(&%s);" % Naming.retval_cname) if is_getbuffer_slot: + assure_gil('error') self.getbuffer_error_cleanup(code) + def align_error_path_gil_to_success_path(code=code.insertion_point()): + # align error and success GIL state when both join + if gil_owned['success']: + assure_gil('error', code=code) + elif gil_owned['error']: + code.put_release_ensured_gil() + gil_owned['error'] = False + assert gil_owned['error'] == gil_owned['success'], "%s: error path %s != success path %s" % ( + self.pos, gil_owned['error'], gil_owned['success']) + # If we are using the non-error cleanup section we should # jump past it if we have an error. The if-test below determine # whether this section is used. - if buffers_present or is_getbuffer_slot or self.return_type.is_memoryviewslice: + if buffers_present or is_getbuffer_slot or return_type.is_memoryviewslice: + # In the buffer cases, we already called assure_gil('error') and own the GIL. + assert gil_owned['error'] or return_type.is_memoryviewslice code.put_goto(code.return_from_error_cleanup_label) + else: + # Adapt the GIL state to the success path right now. + align_error_path_gil_to_success_path() + else: + # No error path, no need to adapt the GIL state. + def align_error_path_gil_to_success_path(): pass # ----- Non-error return cleanup - code.put_label(code.return_label) - for entry in used_buffer_entries: - Buffer.put_release_buffer_code(code, entry) - if is_getbuffer_slot: - self.getbuffer_normal_cleanup(code) + if code.label_used(code.return_label) or not code.label_used(code.error_label): + code.put_label(code.return_label) - if self.return_type.is_memoryviewslice: - # See if our return value is uninitialized on non-error return - # from . import MemoryView - # MemoryView.err_if_nogil_initialized_check(self.pos, env) - cond = code.unlikely(self.return_type.error_condition(Naming.retval_cname)) - code.putln( - 'if (%s) {' % cond) - if env.nogil: - code.put_ensure_gil() - code.putln( - 'PyErr_SetString(PyExc_TypeError, "Memoryview return value is not initialized");') - if env.nogil: - code.put_release_ensured_gil() - code.putln( - '}') + for entry in used_buffer_entries: + assure_gil('success') + Buffer.put_release_buffer_code(code, entry) + if is_getbuffer_slot: + assure_gil('success') + self.getbuffer_normal_cleanup(code) + + if return_type.is_memoryviewslice: + # See if our return value is uninitialized on non-error return + # from . import MemoryView + # MemoryView.err_if_nogil_initialized_check(self.pos, env) + cond = code.unlikely(return_type.error_condition(Naming.retval_cname)) + code.putln( + 'if (%s) {' % cond) + if not gil_owned['success']: + code.put_ensure_gil() + code.putln( + 'PyErr_SetString(PyExc_TypeError, "Memoryview return value is not initialized");') + if not gil_owned['success']: + code.put_release_ensured_gil() + code.putln( + '}') # ----- Return cleanup for both error and no-error return - code.put_label(code.return_from_error_cleanup_label) + if code.label_used(code.return_from_error_cleanup_label): + align_error_path_gil_to_success_path() + code.put_label(code.return_from_error_cleanup_label) for entry in lenv.var_entries: if not entry.used or entry.in_closure: continue - if entry.type.is_memoryviewslice: - code.put_xdecref_memoryviewslice(entry.cname, have_gil=not lenv.nogil) - elif entry.type.is_pyobject: - if not entry.is_arg or len(entry.cf_assignments) > 1: - if entry.xdecref_cleanup: - code.put_var_xdecref(entry) - else: - code.put_var_decref(entry) + if entry.type.is_pyobject: + if entry.is_arg and not entry.cf_is_reassigned: + continue + if entry.type.needs_refcounting: + assure_gil('success') + # FIXME ideally use entry.xdecref_cleanup but this currently isn't reliable + code.put_var_xdecref(entry, have_gil=gil_owned['success']) # Decref any increfed args for entry in lenv.arg_entries: - if entry.type.is_pyobject: - if (acquire_gil or len(entry.cf_assignments) > 1) and not entry.in_closure: - code.put_var_decref(entry) - elif (entry.type.is_memoryviewslice and - (not is_cdef or len(entry.cf_assignments) > 1)): + if entry.in_closure: + continue + if entry.type.is_memoryviewslice: # decref slices of def functions and acquired slices from cdef # functions, but not borrowed slices from cdef functions. - code.put_xdecref_memoryviewslice(entry.cname, - have_gil=not lenv.nogil) + if not entry.cf_is_reassigned: + continue + else: + if not acquire_gil and not entry.cf_is_reassigned: + continue + if entry.type.needs_refcounting: + assure_gil('success') + + # FIXME use entry.xdecref_cleanup - del arg seems to be the problem + code.put_var_xdecref(entry, have_gil=gil_owned['success']) if self.needs_closure: + assure_gil('success') code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type) # ----- Return # This code is duplicated in ModuleNode.generate_module_init_func if not lenv.nogil: - default_retval = self.return_type.default_value + default_retval = return_type.default_value err_val = self.error_value() if err_val is None and default_retval: err_val = default_retval # FIXME: why is err_val not used? - if self.return_type.is_pyobject: - code.put_xgiveref(self.return_type.as_pyobject(Naming.retval_cname)) + code.put_xgiveref(Naming.retval_cname, return_type) if self.entry.is_special and self.entry.name == "__hash__": # Returning -1 for __hash__ is supposed to signal an error # We do as Python instances and coerce -1 into -2. + assure_gil('success') # in special methods, the GIL is owned anyway code.putln("if (unlikely(%s == -1) && !PyErr_Occurred()) %s = -2;" % ( Naming.retval_cname, Naming.retval_cname)) @@ -2143,23 +2383,22 @@ class FuncDefNode(StatNode, BlockNode): code.funcstate.can_trace = False if not self.is_generator: # generators are traced when iterated, not at creation - if self.return_type.is_pyobject: + if return_type.is_pyobject: code.put_trace_return( - Naming.retval_cname, nogil=not code.funcstate.gil_owned) + Naming.retval_cname, nogil=not gil_owned['success']) else: code.put_trace_return( - "Py_None", nogil=not code.funcstate.gil_owned) + "Py_None", nogil=not gil_owned['success']) - if not lenv.nogil: - # GIL holding function - code.put_finish_refcount_context() + if use_refnanny: + code.put_finish_refcount_context(nogil=not gil_owned['success']) - if acquire_gil or (lenv.nogil and lenv.has_with_gil_block): + if acquire_gil or (lenv.nogil and gil_owned['success']): # release the GIL (note that with-gil blocks acquire it on exit in their EnsureGILNode) code.put_release_ensured_gil() code.funcstate.gil_owned = False - if not self.return_type.is_void: + if not return_type.is_void: code.putln("return %s;" % Naming.retval_cname) code.putln("}") @@ -2194,11 +2433,11 @@ class FuncDefNode(StatNode, BlockNode): typeptr_cname = arg.type.typeptr_cname arg_code = "((PyObject *)%s)" % arg.entry.cname code.putln( - 'if (unlikely(!__Pyx_ArgTypeTest(%s, %s, %d, "%s", %s))) %s' % ( + 'if (unlikely(!__Pyx_ArgTypeTest(%s, %s, %d, %s, %s))) %s' % ( arg_code, typeptr_cname, arg.accept_none, - arg.name, + arg.name_cstring, arg.type.is_builtin_type and arg.type.require_exact, code.error_goto(arg.pos))) else: @@ -2212,8 +2451,8 @@ class FuncDefNode(StatNode, BlockNode): cname = arg.entry.cname code.putln('if (unlikely(((PyObject *)%s) == Py_None)) {' % cname) - code.putln('''PyErr_Format(PyExc_TypeError, "Argument '%%.%ds' must not be None", "%s"); %s''' % ( - max(200, len(arg.name)), arg.name, + code.putln('''PyErr_Format(PyExc_TypeError, "Argument '%%.%ds' must not be None", %s); %s''' % ( + max(200, len(arg.name_cstring)), arg.name_cstring, code.error_goto(arg.pos))) code.putln('}') @@ -2223,9 +2462,11 @@ class FuncDefNode(StatNode, BlockNode): def generate_execution_code(self, code): code.mark_pos(self.pos) # Evaluate and store argument default values - for arg in self.args: - if not arg.is_dynamic: - arg.generate_assignment_code(code) + # skip this for wrappers since it's done by wrapped function + if not self.is_wrapper: + for arg in self.args: + if not arg.is_dynamic: + arg.generate_assignment_code(code) # # Special code for the __getbuffer__ function @@ -2249,7 +2490,7 @@ class FuncDefNode(StatNode, BlockNode): def getbuffer_check(self, code): py_buffer, _ = self._get_py_buffer_info() view = py_buffer.cname - code.putln("if (%s == NULL) {" % view) + code.putln("if (unlikely(%s == NULL)) {" % view) code.putln("PyErr_SetString(PyExc_BufferError, " "\"PyObject_GetBuffer: view==NULL argument is obsolete\");") code.putln("return -1;") @@ -2260,7 +2501,7 @@ class FuncDefNode(StatNode, BlockNode): view = py_buffer.cname if obj_type and obj_type.is_pyobject: code.put_init_to_py_none("%s->obj" % view, obj_type) - code.put_giveref("%s->obj" % view) # Do not refnanny object within structs + code.put_giveref("%s->obj" % view, obj_type) # Do not refnanny object within structs else: code.putln("%s->obj = NULL;" % view) @@ -2269,7 +2510,7 @@ class FuncDefNode(StatNode, BlockNode): view = py_buffer.cname if obj_type and obj_type.is_pyobject: code.putln("if (%s->obj != NULL) {" % view) - code.put_gotref("%s->obj" % view) + code.put_gotref("%s->obj" % view, obj_type) code.put_decref_clear("%s->obj" % view, obj_type) code.putln("}") else: @@ -2280,7 +2521,7 @@ class FuncDefNode(StatNode, BlockNode): view = py_buffer.cname if obj_type and obj_type.is_pyobject: code.putln("if (%s->obj == Py_None) {" % view) - code.put_gotref("%s->obj" % view) + code.put_gotref("%s->obj" % view, obj_type) code.put_decref_clear("%s->obj" % view, obj_type) code.putln("}") @@ -2288,7 +2529,7 @@ class FuncDefNode(StatNode, BlockNode): if not self.entry.is_special: return None name = self.entry.name - slot = TypeSlots.method_name_to_slot.get(name) + slot = TypeSlots.get_slot_table(self.local_scope.directives).get_slot_by_method_name(name) if not slot: return None if name == '__long__' and not self.entry.scope.lookup_here('__int__'): @@ -2322,7 +2563,8 @@ class CFuncDefNode(FuncDefNode): # is_static_method whether this is a static method # is_c_class_method whether this is a cclass method - child_attrs = ["base_type", "declarator", "body", "py_func_stat"] + child_attrs = ["base_type", "declarator", "body", "decorators", "py_func_stat"] + outer_attrs = ["decorators", "py_func_stat"] inline_in_pxd = False decorators = None @@ -2336,6 +2578,9 @@ class CFuncDefNode(FuncDefNode): def unqualified_name(self): return self.entry.name + def declared_name(self): + return self.declarator.declared_name() + @property def code_object(self): # share the CodeObject with the cpdef wrapper (if available) @@ -2356,20 +2601,20 @@ class CFuncDefNode(FuncDefNode): self.is_static_method = 'staticmethod' in env.directives and not env.lookup_here('staticmethod') # The 2 here is because we need both function and argument names. if isinstance(self.declarator, CFuncDeclaratorNode): - name_declarator, type = self.declarator.analyse( + name_declarator, typ = self.declarator.analyse( base_type, env, nonempty=2 * (self.body is not None), directive_locals=self.directive_locals, visibility=self.visibility) else: - name_declarator, type = self.declarator.analyse( + name_declarator, typ = self.declarator.analyse( base_type, env, nonempty=2 * (self.body is not None), visibility=self.visibility) - if not type.is_cfunction: + if not typ.is_cfunction: error(self.pos, "Suite attached to non-function declaration") # Remember the actual type according to the function header # written here, because the type in the symbol table entry # may be different if we're overriding a C method inherited # from the base type of an extension type. - self.type = type - type.is_overridable = self.overridable + self.type = typ + typ.is_overridable = self.overridable declarator = self.declarator while not hasattr(declarator, 'args'): declarator = declarator.base @@ -2382,11 +2627,18 @@ class CFuncDefNode(FuncDefNode): error(self.cfunc_declarator.pos, "Function with optional arguments may not be declared public or api") - if type.exception_check == '+' and self.visibility != 'extern': - warning(self.cfunc_declarator.pos, + if typ.exception_check == '+' and self.visibility != 'extern': + if typ.exception_value and typ.exception_value.is_name: + # it really is impossible to reason about what the user wants to happens + # if they've specified a C++ exception translation function. Therefore, + # raise an error. + error(self.pos, "Only extern functions can throw C++ exceptions.") + else: + warning(self.pos, + "Only extern functions can throw C++ exceptions.", 2) - for formal_arg, type_arg in zip(self.args, type.args): + for formal_arg, type_arg in zip(self.args, typ.args): self.align_argument_type(env, type_arg) formal_arg.type = type_arg.type formal_arg.name = type_arg.name @@ -2407,20 +2659,21 @@ class CFuncDefNode(FuncDefNode): elif 'inline' in self.modifiers: warning(formal_arg.pos, "Buffer unpacking not optimized away.", 1) - self._validate_type_visibility(type.return_type, self.pos, env) + self._validate_type_visibility(typ.return_type, self.pos, env) name = name_declarator.name cname = name_declarator.cname - type.is_const_method = self.is_const_method - type.is_static_method = self.is_static_method + typ.is_const_method = self.is_const_method + typ.is_static_method = self.is_static_method + self.entry = env.declare_cfunction( - name, type, self.pos, + name, typ, self.pos, cname=cname, visibility=self.visibility, api=self.api, defining=self.body is not None, modifiers=self.modifiers, overridable=self.overridable) self.entry.inline_func_in_pxd = self.inline_in_pxd - self.return_type = type.return_type + self.return_type = typ.return_type if self.return_type.is_array and self.visibility != 'extern': error(self.pos, "Function cannot return an array") if self.return_type.is_cpp_class: @@ -2435,38 +2688,45 @@ class CFuncDefNode(FuncDefNode): self.create_local_scope(env) def declare_cpdef_wrapper(self, env): - if self.overridable: - if self.is_static_method: - # TODO(robertwb): Finish this up, perhaps via more function refactoring. - error(self.pos, "static cpdef methods not yet supported") - name = self.entry.name - py_func_body = self.call_self_node(is_module_scope=env.is_module_scope) - if self.is_static_method: - from .ExprNodes import NameNode - decorators = [DecoratorNode(self.pos, decorator=NameNode(self.pos, name='staticmethod'))] - decorators[0].decorator.analyse_types(env) + if not self.overridable: + return + if self.is_static_method: + # TODO(robertwb): Finish this up, perhaps via more function refactoring. + error(self.pos, "static cpdef methods not yet supported") + + name = self.entry.name + py_func_body = self.call_self_node(is_module_scope=env.is_module_scope) + if self.is_static_method: + from .ExprNodes import NameNode + decorators = [DecoratorNode(self.pos, decorator=NameNode(self.pos, name=EncodedString('staticmethod')))] + decorators[0].decorator.analyse_types(env) + else: + decorators = [] + self.py_func = DefNode(pos=self.pos, + name=self.entry.name, + args=self.args, + star_arg=None, + starstar_arg=None, + doc=self.doc, + body=py_func_body, + decorators=decorators, + is_wrapper=1) + self.py_func.is_module_scope = env.is_module_scope + self.py_func.analyse_declarations(env) + self.py_func.entry.is_overridable = True + self.py_func_stat = StatListNode(self.pos, stats=[self.py_func]) + self.py_func.type = PyrexTypes.py_object_type + self.entry.as_variable = self.py_func.entry + self.entry.used = self.entry.as_variable.used = True + # Reset scope entry the above cfunction + env.entries[name] = self.entry + if (not self.entry.is_final_cmethod and + (not env.is_module_scope or Options.lookup_module_cpdef)): + if self.override: + # This is a hack: we shouldn't create the wrapper twice, but we do for fused functions. + assert self.entry.is_fused_specialized # should not happen for non-fused cpdef functions + self.override.py_func = self.py_func else: - decorators = [] - self.py_func = DefNode(pos=self.pos, - name=self.entry.name, - args=self.args, - star_arg=None, - starstar_arg=None, - doc=self.doc, - body=py_func_body, - decorators=decorators, - is_wrapper=1) - self.py_func.is_module_scope = env.is_module_scope - self.py_func.analyse_declarations(env) - self.py_func.entry.is_overridable = True - self.py_func_stat = StatListNode(self.pos, stats=[self.py_func]) - self.py_func.type = PyrexTypes.py_object_type - self.entry.as_variable = self.py_func.entry - self.entry.used = self.entry.as_variable.used = True - # Reset scope entry the above cfunction - env.entries[name] = self.entry - if (not self.entry.is_final_cmethod and - (not env.is_module_scope or Options.lookup_module_cpdef)): self.override = OverrideCheckNode(self.pos, py_func=self.py_func) self.body = StatListNode(self.pos, stats=[self.override, self.body]) @@ -2585,7 +2845,7 @@ class CFuncDefNode(FuncDefNode): header = self.return_type.declaration_code(entity, dll_linkage=dll_linkage) #print (storage_class, modifiers, header) - needs_proto = self.is_c_class_method + needs_proto = self.is_c_class_method or self.entry.is_cproperty if self.template_declaration: if needs_proto: code.globalstate.parts['module_declarations'].putln(self.template_declaration) @@ -2638,7 +2898,7 @@ class CFuncDefNode(FuncDefNode): if entry.in_closure and not arg.default: code.putln('%s = %s;' % (entry.cname, entry.original_cname)) if entry.type.is_memoryviewslice: - code.put_incref_memoryviewslice(entry.cname, have_gil=True) + entry.type.generate_incref_memoryviewslice(code, entry.cname, True) else: code.put_var_incref(entry) code.put_var_giveref(entry) @@ -2670,7 +2930,6 @@ class CFuncDefNode(FuncDefNode): if self.return_type.is_pyobject: return "0" else: - #return None return self.entry.type.exception_value def caller_will_check_exceptions(self): @@ -2769,7 +3028,7 @@ class DefNode(FuncDefNode): self_in_stararg = 0 py_cfunc_node = None requires_classobj = False - defaults_struct = None # Dynamic kwrds structure name + defaults_struct = None # Dynamic kwrds structure name doc = None fused_py_func = False @@ -2782,14 +3041,17 @@ class DefNode(FuncDefNode): def __init__(self, pos, **kwds): FuncDefNode.__init__(self, pos, **kwds) - k = rk = r = 0 + p = k = rk = r = 0 for arg in self.args: + if arg.pos_only: + p += 1 if arg.kw_only: k += 1 if not arg.default: rk += 1 if not arg.default: r += 1 + self.num_posonly_args = p self.num_kwonly_args = k self.num_required_kw_args = rk self.num_required_args = r @@ -2887,8 +3149,18 @@ class DefNode(FuncDefNode): # staticmethod() was overridden - not much we can do here ... self.is_staticmethod = False - if self.name == '__new__' and env.is_py_class_scope: - self.is_staticmethod = 1 + if env.is_py_class_scope or env.is_c_class_scope: + if self.name == '__new__' and env.is_py_class_scope: + self.is_staticmethod = True + elif self.name == '__init_subclass__' and env.is_c_class_scope: + error(self.pos, "'__init_subclass__' is not supported by extension class") + elif self.name in IMPLICIT_CLASSMETHODS and not self.is_classmethod: + self.is_classmethod = True + # TODO: remove the need to generate a real decorator here, is_classmethod=True should suffice. + from .ExprNodes import NameNode + self.decorators = self.decorators or [] + self.decorators.insert(0, DecoratorNode( + self.pos, decorator=NameNode(self.pos, name=EncodedString('classmethod')))) self.analyse_argument_types(env) if self.name == '<lambda>': @@ -2901,7 +3173,7 @@ class DefNode(FuncDefNode): # if a signature annotation provides a more specific return object type, use it if self.return_type is py_object_type and self.return_type_annotation: if env.directives['annotation_typing'] and not self.entry.is_special: - _, return_type = analyse_type_annotation(self.return_type_annotation, env) + _, return_type = self.return_type_annotation.analyse_type_annotation(env) if return_type and return_type.is_pyobject: self.return_type = return_type @@ -2941,9 +3213,6 @@ class DefNode(FuncDefNode): arg.name = name_declarator.name arg.type = type - if type.is_fused: - self.has_fused_arguments = True - self.align_argument_type(env, arg) if name_declarator and name_declarator.cname: error(self.pos, "Python function argument cannot have C name specification") @@ -2968,12 +3237,15 @@ class DefNode(FuncDefNode): else: # probably just a plain 'object' arg.accept_none = True - else: - arg.accept_none = True # won't be used, but must be there + elif not arg.type.is_error: + arg.accept_none = True # won't be used, but must be there if arg.not_none: error(arg.pos, "Only Python type arguments can have 'not None'") if arg.or_none: error(arg.pos, "Only Python type arguments can have 'or None'") + + if arg.type.is_fused: + self.has_fused_arguments = True env.fused_to_specific = f2s if has_np_pythran(env): @@ -2986,8 +3258,10 @@ class DefNode(FuncDefNode): if self.decorators: error(self.pos, "special functions of cdef classes cannot have decorators") self.entry.trivial_signature = len(self.args) == 1 and not (self.star_arg or self.starstar_arg) - elif not env.directives['always_allow_keywords'] and not (self.star_arg or self.starstar_arg): - # Use the simpler calling signature for zero- and one-argument functions. + elif not (self.star_arg or self.starstar_arg) and ( + not env.directives['always_allow_keywords'] + or all([arg.pos_only for arg in self.args])): + # Use the simpler calling signature for zero- and one-argument pos-only functions. if self.entry.signature is TypeSlots.pyfunction_signature: if len(self.args) == 0: self.entry.signature = TypeSlots.pyfunction_noargs @@ -3002,18 +3276,19 @@ class DefNode(FuncDefNode): self.entry.signature = TypeSlots.ibinaryfunc sig = self.entry.signature - nfixed = sig.num_fixed_args() + nfixed = sig.max_num_fixed_args() + min_nfixed = sig.min_num_fixed_args() if (sig is TypeSlots.pymethod_signature and nfixed == 1 and len(self.args) == 0 and self.star_arg): # this is the only case where a diverging number of # arguments is not an error - when we have no explicit # 'self' parameter as in method(*args) - sig = self.entry.signature = TypeSlots.pyfunction_signature # self is not 'really' used + sig = self.entry.signature = TypeSlots.pyfunction_signature # self is not 'really' used self.self_in_stararg = 1 - nfixed = 0 + nfixed = min_nfixed = 0 if self.is_staticmethod and env.is_c_class_scope: - nfixed = 0 + nfixed = min_nfixed = 0 self.self_in_stararg = True # FIXME: why for staticmethods? self.entry.signature = sig = copy.copy(sig) @@ -3028,6 +3303,8 @@ class DefNode(FuncDefNode): for i in range(min(nfixed, len(self.args))): arg = self.args[i] arg.is_generic = 0 + if i >= min_nfixed: + arg.is_special_method_optional = True if sig.is_self_arg(i) and not self.is_staticmethod: if self.is_classmethod: arg.is_type_arg = 1 @@ -3043,12 +3320,8 @@ class DefNode(FuncDefNode): arg.needs_type_test = 1 else: arg.needs_conversion = 1 - if arg.needs_conversion: - arg.hdr_cname = Naming.arg_prefix + arg.name - else: - arg.hdr_cname = Naming.var_prefix + arg.name - if nfixed > len(self.args): + if min_nfixed > len(self.args): self.bad_signature() return elif nfixed < len(self.args): @@ -3058,11 +3331,38 @@ class DefNode(FuncDefNode): if arg.is_generic and (arg.type.is_extension_type or arg.type.is_builtin_type): arg.needs_type_test = 1 + # Decide whether to use METH_FASTCALL + # 1. If we use METH_NOARGS or METH_O, keep that. We can only change + # METH_VARARGS to METH_FASTCALL + # 2. Special methods like __call__ always use the METH_VARGARGS + # calling convention + mf = sig.method_flags() + if mf and TypeSlots.method_varargs in mf and not self.entry.is_special: + # 3. If the function uses the full args tuple, it's more + # efficient to use METH_VARARGS. This happens when the function + # takes *args but no other positional arguments (apart from + # possibly self). We don't do the analogous check for keyword + # arguments since the kwargs dict is copied anyway. + if self.star_arg: + uses_args_tuple = True + for arg in self.args: + if (arg.is_generic and not arg.kw_only and + not arg.is_self_arg and not arg.is_type_arg): + # Other positional argument + uses_args_tuple = False + else: + uses_args_tuple = False + + if not uses_args_tuple: + sig = self.entry.signature = sig.with_fastcall() + def bad_signature(self): sig = self.entry.signature - expected_str = "%d" % sig.num_fixed_args() + expected_str = "%d" % sig.min_num_fixed_args() if sig.has_generic_args: expected_str += " or more" + elif sig.optional_object_arg_count: + expected_str += " to %d" % sig.max_num_fixed_args() name = self.name if name.startswith("__") and name.endswith("__"): desc = "Special method" @@ -3083,16 +3383,16 @@ class DefNode(FuncDefNode): entry = env.declare_pyfunction(name, self.pos, allow_redefine=not self.is_wrapper) self.entry = entry prefix = env.next_id(env.scope_prefix) - self.entry.pyfunc_cname = Naming.pyfunc_prefix + prefix + name + self.entry.pyfunc_cname = punycodify_name(Naming.pyfunc_prefix + prefix + name) if Options.docstrings: entry.doc = embed_position(self.pos, self.doc) - entry.doc_cname = Naming.funcdoc_prefix + prefix + name + entry.doc_cname = punycodify_name(Naming.funcdoc_prefix + prefix + name) if entry.is_special: if entry.name in TypeSlots.invisible or not entry.doc or ( entry.name in '__getattr__' and env.directives['fast_getattr']): entry.wrapperbase_cname = None else: - entry.wrapperbase_cname = Naming.wrapperbase_prefix + prefix + name + entry.wrapperbase_cname = punycodify_name(Naming.wrapperbase_prefix + prefix + name) else: entry.doc = None @@ -3135,8 +3435,6 @@ class DefNode(FuncDefNode): self.local_scope.directives = env.directives self.analyse_default_values(env) self.analyse_annotations(env) - if self.return_type_annotation: - self.return_type_annotation = self.analyse_annotation(env, self.return_type_annotation) if not self.needs_assignment_synthesis(env) and self.decorators: for decorator in self.decorators[::-1]: @@ -3237,8 +3535,20 @@ class DefNode(FuncDefNode): # Move arguments into closure if required def put_into_closure(entry): if entry.in_closure: - code.putln('%s = %s;' % (entry.cname, entry.original_cname)) - if entry.xdecref_cleanup: + if entry.type.is_array: + # This applies to generator expressions that iterate over C arrays (and need to + # capture them by value), under most other circumstances C array arguments are dropped to + # pointers so this copy isn't used + assert entry.type.size is not None + code.globalstate.use_utility_code(UtilityCode.load_cached("IncludeStringH", "StringTools.c")) + code.putln("memcpy({0}, {1}, sizeof({0}));".format(entry.cname, entry.original_cname)) + else: + code.putln('%s = %s;' % (entry.cname, entry.original_cname)) + if entry.type.is_memoryviewslice: + # TODO - at some point reference count of memoryviews should + # genuinely be unified with PyObjects + entry.type.generate_incref_memoryviewslice(code, entry.cname, True) + elif entry.xdecref_cleanup: # mostly applies to the starstar arg - this can sometimes be NULL # so must be xincrefed instead code.put_var_xincref(entry) @@ -3260,10 +3570,11 @@ class DefNodeWrapper(FuncDefNode): # DefNode python wrapper code generator defnode = None - target = None # Target DefNode + target = None # Target DefNode def __init__(self, *args, **kwargs): FuncDefNode.__init__(self, *args, **kwargs) + self.num_posonly_args = self.target.num_posonly_args self.num_kwonly_args = self.target.num_kwonly_args self.num_required_kw_args = self.target.num_required_kw_args self.num_required_args = self.target.num_required_args @@ -3274,8 +3585,8 @@ class DefNodeWrapper(FuncDefNode): target_entry = self.target.entry name = self.name prefix = env.next_id(env.scope_prefix) - target_entry.func_cname = Naming.pywrap_prefix + prefix + name - target_entry.pymethdef_cname = Naming.pymethdef_prefix + prefix + name + target_entry.func_cname = punycodify_name(Naming.pywrap_prefix + prefix + name) + target_entry.pymethdef_cname = punycodify_name(Naming.pymethdef_prefix + prefix + name) self.signature = target_entry.signature @@ -3289,10 +3600,10 @@ class DefNodeWrapper(FuncDefNode): for arg in self.args: if not arg.type.is_pyobject: if not arg.type.create_from_py_utility_code(env): - pass # will fail later + pass # will fail later elif arg.hdr_type and not arg.hdr_type.is_pyobject: if not arg.hdr_type.create_to_py_utility_code(env): - pass # will fail later + pass # will fail later if self.starstar_arg and not self.starstar_arg.entry.cf_used: # we will set the kwargs argument to NULL instead of a new dict @@ -3319,7 +3630,13 @@ class DefNodeWrapper(FuncDefNode): if self.signature.has_dummy_arg: args.append(Naming.self_cname) for arg in self.args: - if arg.hdr_type and not (arg.type.is_memoryviewslice or + if arg.type.is_cpp_class: + # it's safe to move converted C++ types because they aren't + # used again afterwards + code.globalstate.use_utility_code( + UtilityCode.load_cached("MoveIfSupported", "CppSupport.cpp")) + args.append("__PYX_STD_MOVE_IF_SUPPORTED(%s)" % arg.entry.cname) + elif arg.hdr_type and not (arg.type.is_memoryviewslice or arg.type.is_struct or arg.type.is_complex): args.append(arg.type.cast_code(arg.entry.cname)) @@ -3363,7 +3680,7 @@ class DefNodeWrapper(FuncDefNode): self.return_type.declaration_code(Naming.retval_cname), retval_init)) code.put_declare_refcount_context() - code.put_setup_refcount_context('%s (wrapper)' % self.name) + code.put_setup_refcount_context(EncodedString('%s (wrapper)' % self.name)) self.generate_argument_parsing_code(lenv, code) self.generate_argument_type_tests(code) @@ -3389,8 +3706,20 @@ class DefNodeWrapper(FuncDefNode): # ----- Non-error return cleanup code.put_label(code.return_label) for entry in lenv.var_entries: - if entry.is_arg and entry.type.is_pyobject: - code.put_var_decref(entry) + if entry.is_arg: + # mainly captures the star/starstar args + if entry.xdecref_cleanup: + code.put_var_xdecref(entry) + else: + code.put_var_decref(entry) + for arg in self.args: + if not arg.type.is_pyobject: + # This captures anything that's been converted from a PyObject. + # Primarily memoryviews at the moment + if arg.entry.xdecref_cleanup: + code.put_var_xdecref(arg.entry) + else: + code.put_var_decref(arg.entry) code.put_finish_refcount_context() if not self.return_type.is_void: @@ -3420,12 +3749,20 @@ class DefNodeWrapper(FuncDefNode): entry = self.target.entry if not entry.is_special and sig.method_flags() == [TypeSlots.method_noargs]: arg_code_list.append("CYTHON_UNUSED PyObject *unused") - if entry.scope.is_c_class_scope and entry.name == "__ipow__": - arg_code_list.append("CYTHON_UNUSED PyObject *unused") if sig.has_generic_args: - arg_code_list.append( - "PyObject *%s, PyObject *%s" % ( - Naming.args_cname, Naming.kwds_cname)) + varargs_args = "PyObject *%s, PyObject *%s" % ( + Naming.args_cname, Naming.kwds_cname) + if sig.use_fastcall: + fastcall_args = "PyObject *const *%s, Py_ssize_t %s, PyObject *%s" % ( + Naming.args_cname, Naming.nargs_cname, Naming.kwds_cname) + arg_code_list.append( + "\n#if CYTHON_METH_FASTCALL\n%s\n#else\n%s\n#endif\n" % ( + fastcall_args, varargs_args)) + else: + arg_code_list.append(varargs_args) + if entry.is_special: + for n in range(len(self.args), sig.max_num_fixed_args()): + arg_code_list.append("CYTHON_UNUSED PyObject *unused_arg_%s" % n) arg_code = ", ".join(arg_code_list) # Prevent warning: unused function '__pyx_pw_5numpy_7ndarray_1__getbuffer__' @@ -3436,7 +3773,7 @@ class DefNodeWrapper(FuncDefNode): with_pymethdef = False dc = self.return_type.declaration_code(entry.func_cname) - header = "static %s%s(%s)" % (mf, dc, arg_code) + header = "%sstatic %s(%s)" % (mf, dc, arg_code) code.putln("%s; /*proto*/" % header) if proto_only: @@ -3459,7 +3796,7 @@ class DefNodeWrapper(FuncDefNode): docstr = docstr.as_utf8_string() if not (entry.is_special and entry.name in ('__getbuffer__', '__releasebuffer__')): - code.putln('static char %s[] = %s;' % ( + code.putln('PyDoc_STRVAR(%s, %s);' % ( entry.doc_cname, docstr.as_c_string_literal())) @@ -3486,6 +3823,23 @@ class DefNodeWrapper(FuncDefNode): if entry.is_arg: code.put_var_declaration(entry) + # Assign nargs variable as len(args), but avoid an "unused" warning in the few cases where we don't need it. + if self.signature_has_generic_args(): + nargs_code = "CYTHON_UNUSED const Py_ssize_t %s = PyTuple_GET_SIZE(%s);" % ( + Naming.nargs_cname, Naming.args_cname) + if self.signature.use_fastcall: + code.putln("#if !CYTHON_METH_FASTCALL") + code.putln(nargs_code) + code.putln("#endif") + else: + code.putln(nargs_code) + + # Array containing the values of keyword arguments when using METH_FASTCALL. + code.globalstate.use_utility_code( + UtilityCode.load_cached("fastcall", "FunctionArguments.c")) + code.putln('CYTHON_UNUSED PyObject *const *%s = __Pyx_KwValues_%s(%s, %s);' % ( + Naming.kwvalues_cname, self.signature.fastvar, Naming.args_cname, Naming.nargs_cname)) + def generate_argument_parsing_code(self, env, code): # Generate fast equivalent of PyArg_ParseTuple call for # generic arguments, if any, including args/kwargs @@ -3509,6 +3863,8 @@ class DefNodeWrapper(FuncDefNode): elif not self.signature_has_nongeneric_args(): # func(*args) or func(**kw) or func(*args, **kw) + # possibly with a "self" argument but no other non-star + # arguments self.generate_stararg_copy_code(code) else: @@ -3526,6 +3882,11 @@ class DefNodeWrapper(FuncDefNode): code.put_var_xdecref_clear(self.starstar_arg.entry) else: code.put_var_decref_clear(self.starstar_arg.entry) + for arg in self.args: + if not arg.type.is_pyobject and arg.type.needs_refcounting: + # at the moment this just catches memoryviewslices, but in future + # other non-PyObject reference counted types might need cleanup + code.put_var_xdecref(arg.entry) code.put_add_traceback(self.target.entry.qualified_name) code.put_finish_refcount_context() code.putln("return %s;" % self.error_value()) @@ -3544,10 +3905,9 @@ class DefNodeWrapper(FuncDefNode): if not self.star_arg: code.globalstate.use_utility_code( UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c")) - code.putln("if (unlikely(PyTuple_GET_SIZE(%s) > 0)) {" % - Naming.args_cname) - code.put('__Pyx_RaiseArgtupleInvalid("%s", 1, 0, 0, PyTuple_GET_SIZE(%s)); return %s;' % ( - self.name, Naming.args_cname, self.error_value())) + code.putln("if (unlikely(%s > 0)) {" % Naming.nargs_cname) + code.put('__Pyx_RaiseArgtupleInvalid(%s, 1, 0, 0, %s); return %s;' % ( + self.name.as_c_string_literal(), Naming.nargs_cname, self.error_value())) code.putln("}") if self.starstar_arg: @@ -3556,69 +3916,66 @@ class DefNodeWrapper(FuncDefNode): else: kwarg_check = "%s" % Naming.kwds_cname else: - kwarg_check = "unlikely(%s) && unlikely(PyDict_Size(%s) > 0)" % ( - Naming.kwds_cname, Naming.kwds_cname) + kwarg_check = "unlikely(%s) && __Pyx_NumKwargs_%s(%s)" % ( + Naming.kwds_cname, self.signature.fastvar, Naming.kwds_cname) code.globalstate.use_utility_code( UtilityCode.load_cached("KeywordStringCheck", "FunctionArguments.c")) code.putln( - "if (%s && unlikely(!__Pyx_CheckKeywordStrings(%s, \"%s\", %d))) return %s;" % ( - kwarg_check, Naming.kwds_cname, self.name, + "if (%s && unlikely(!__Pyx_CheckKeywordStrings(%s, %s, %d))) return %s;" % ( + kwarg_check, Naming.kwds_cname, self.name.as_c_string_literal(), bool(self.starstar_arg), self.error_value())) if self.starstar_arg and self.starstar_arg.entry.cf_used: - if all(ref.node.allow_null for ref in self.starstar_arg.entry.cf_references): - code.putln("if (%s) {" % kwarg_check) - code.putln("%s = PyDict_Copy(%s); if (unlikely(!%s)) return %s;" % ( - self.starstar_arg.entry.cname, - Naming.kwds_cname, - self.starstar_arg.entry.cname, - self.error_value())) - code.put_gotref(self.starstar_arg.entry.cname) - code.putln("} else {") - code.putln("%s = NULL;" % (self.starstar_arg.entry.cname,)) - code.putln("}") - self.starstar_arg.entry.xdecref_cleanup = 1 - else: - code.put("%s = (%s) ? PyDict_Copy(%s) : PyDict_New(); " % ( - self.starstar_arg.entry.cname, - Naming.kwds_cname, - Naming.kwds_cname)) - code.putln("if (unlikely(!%s)) return %s;" % ( - self.starstar_arg.entry.cname, self.error_value())) - self.starstar_arg.entry.xdecref_cleanup = 0 - code.put_gotref(self.starstar_arg.entry.cname) + code.putln("if (%s) {" % kwarg_check) + code.putln("%s = __Pyx_KwargsAsDict_%s(%s, %s);" % ( + self.starstar_arg.entry.cname, + self.signature.fastvar, + Naming.kwds_cname, + Naming.kwvalues_cname)) + code.putln("if (unlikely(!%s)) return %s;" % ( + self.starstar_arg.entry.cname, self.error_value())) + code.put_gotref(self.starstar_arg.entry.cname, py_object_type) + code.putln("} else {") + code.putln("%s = PyDict_New();" % (self.starstar_arg.entry.cname,)) + code.putln("if (unlikely(!%s)) return %s;" % ( + self.starstar_arg.entry.cname, self.error_value())) + code.put_var_gotref(self.starstar_arg.entry) + self.starstar_arg.entry.xdecref_cleanup = False + code.putln("}") if self.self_in_stararg and not self.target.is_staticmethod: + assert not self.signature.use_fastcall # need to create a new tuple with 'self' inserted as first item - code.put("%s = PyTuple_New(PyTuple_GET_SIZE(%s)+1); if (unlikely(!%s)) " % ( + code.put("%s = PyTuple_New(%s + 1); if (unlikely(!%s)) " % ( self.star_arg.entry.cname, - Naming.args_cname, + Naming.nargs_cname, self.star_arg.entry.cname)) if self.starstar_arg and self.starstar_arg.entry.cf_used: code.putln("{") - code.put_xdecref_clear(self.starstar_arg.entry.cname, py_object_type) + code.put_var_xdecref_clear(self.starstar_arg.entry) code.putln("return %s;" % self.error_value()) code.putln("}") else: code.putln("return %s;" % self.error_value()) - code.put_gotref(self.star_arg.entry.cname) + code.put_var_gotref(self.star_arg.entry) code.put_incref(Naming.self_cname, py_object_type) - code.put_giveref(Naming.self_cname) + code.put_giveref(Naming.self_cname, py_object_type) code.putln("PyTuple_SET_ITEM(%s, 0, %s);" % ( self.star_arg.entry.cname, Naming.self_cname)) temp = code.funcstate.allocate_temp(PyrexTypes.c_py_ssize_t_type, manage_ref=False) - code.putln("for (%s=0; %s < PyTuple_GET_SIZE(%s); %s++) {" % ( - temp, temp, Naming.args_cname, temp)) + code.putln("for (%s=0; %s < %s; %s++) {" % ( + temp, temp, Naming.nargs_cname, temp)) code.putln("PyObject* item = PyTuple_GET_ITEM(%s, %s);" % ( Naming.args_cname, temp)) code.put_incref("item", py_object_type) - code.put_giveref("item") + code.put_giveref("item", py_object_type) code.putln("PyTuple_SET_ITEM(%s, %s+1, item);" % ( self.star_arg.entry.cname, temp)) code.putln("}") code.funcstate.release_temp(temp) self.star_arg.entry.xdecref_cleanup = 0 elif self.star_arg: + assert not self.signature.use_fastcall code.put_incref(Naming.args_cname, py_object_type) code.putln("%s = %s;" % ( self.star_arg.entry.cname, @@ -3626,11 +3983,17 @@ class DefNodeWrapper(FuncDefNode): self.star_arg.entry.xdecref_cleanup = 0 def generate_tuple_and_keyword_parsing_code(self, args, success_label, code): + code.globalstate.use_utility_code( + UtilityCode.load_cached("fastcall", "FunctionArguments.c")) + + self_name_csafe = self.name.as_c_string_literal() + argtuple_error_label = code.new_label("argtuple_error") positional_args = [] required_kw_only_args = [] optional_kw_only_args = [] + num_pos_only_args = 0 for arg in args: if arg.is_generic: if arg.default: @@ -3643,6 +4006,8 @@ class DefNodeWrapper(FuncDefNode): required_kw_only_args.append(arg) elif not arg.is_self_arg and not arg.is_type_arg: positional_args.append(arg) + if arg.pos_only: + num_pos_only_args += 1 # sort required kw-only args before optional ones to avoid special # cases in the unpacking code @@ -3661,10 +4026,12 @@ class DefNodeWrapper(FuncDefNode): code.putln('{') all_args = tuple(positional_args) + tuple(kw_only_args) - code.putln("static PyObject **%s[] = {%s,0};" % ( + non_posonly_args = [arg for arg in all_args if not arg.pos_only] + non_pos_args_id = ','.join( + ['&%s' % code.intern_identifier(arg.entry.name) for arg in non_posonly_args] + ['0']) + code.putln("PyObject **%s[] = {%s};" % ( Naming.pykwdlist_cname, - ','.join(['&%s' % code.intern_identifier(arg.name) - for arg in all_args]))) + non_pos_args_id)) # Before being converted and assigned to the target variables, # borrowed references to all unpacked argument values are @@ -3676,14 +4043,43 @@ class DefNodeWrapper(FuncDefNode): # was passed for them. self.generate_argument_values_setup_code(all_args, code) + # If all args are positional-only, we can raise an error + # straight away if we receive a non-empty kw-dict. + # This requires a PyDict_Size call. This call is wasteful + # for functions which do accept kw-args, so we do not generate + # the PyDict_Size call unless all args are positional-only. + accept_kwd_args = non_posonly_args or self.starstar_arg + if accept_kwd_args: + kw_unpacking_condition = Naming.kwds_cname + else: + kw_unpacking_condition = "%s && __Pyx_NumKwargs_%s(%s) > 0" % ( + Naming.kwds_cname, self.signature.fastvar, Naming.kwds_cname) + + if self.num_required_kw_args > 0: + kw_unpacking_condition = "likely(%s)" % kw_unpacking_condition + # --- optimised code when we receive keyword arguments - code.putln("if (%s(%s)) {" % ( - (self.num_required_kw_args > 0) and "likely" or "unlikely", - Naming.kwds_cname)) - self.generate_keyword_unpacking_code( - min_positional_args, max_positional_args, - has_fixed_positional_count, has_kw_only_args, - all_args, argtuple_error_label, code) + code.putln("if (%s) {" % kw_unpacking_condition) + + if accept_kwd_args: + self.generate_keyword_unpacking_code( + min_positional_args, max_positional_args, + has_fixed_positional_count, has_kw_only_args, all_args, argtuple_error_label, code) + else: + # Here we do not accept kw-args but we are passed a non-empty kw-dict. + # We call ParseOptionalKeywords which will raise an appropriate error if + # the kw-args dict passed is non-empty (which it will be, since kw_unpacking_condition is true) + code.globalstate.use_utility_code( + UtilityCode.load_cached("ParseKeywords", "FunctionArguments.c")) + code.putln('if (likely(__Pyx_ParseOptionalKeywords(%s, %s, %s, %s, %s, %s, %s) < 0)) %s' % ( + Naming.kwds_cname, + Naming.kwvalues_cname, + Naming.pykwdlist_cname, + self.starstar_arg.entry.cname if self.starstar_arg else 0, + 'values', + 0, + self_name_csafe, + code.error_goto(self.pos))) # --- optimised code when we do not receive any keyword arguments if (self.num_required_kw_args and min_positional_args > 0) or min_positional_args == max_positional_args: @@ -3693,20 +4089,20 @@ class DefNodeWrapper(FuncDefNode): compare = '!=' else: compare = '<' - code.putln('} else if (PyTuple_GET_SIZE(%s) %s %d) {' % ( - Naming.args_cname, compare, min_positional_args)) + code.putln('} else if (unlikely(%s %s %d)) {' % ( + Naming.nargs_cname, compare, min_positional_args)) code.put_goto(argtuple_error_label) if self.num_required_kw_args: # pure error case: keywords required but not passed if max_positional_args > min_positional_args and not self.star_arg: - code.putln('} else if (PyTuple_GET_SIZE(%s) > %d) {' % ( - Naming.args_cname, max_positional_args)) + code.putln('} else if (unlikely(%s > %d)) {' % ( + Naming.nargs_cname, max_positional_args)) code.put_goto(argtuple_error_label) code.putln('} else {') for i, arg in enumerate(kw_only_args): if not arg.default: - pystring_cname = code.intern_identifier(arg.name) + pystring_cname = code.intern_identifier(arg.entry.name) # required keyword-only argument missing code.globalstate.use_utility_code( UtilityCode.load_cached("RaiseKeywordRequired", "FunctionArguments.c")) @@ -3723,11 +4119,12 @@ class DefNodeWrapper(FuncDefNode): # parse the exact number of positional arguments from # the args tuple for i, arg in enumerate(positional_args): - code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (i, Naming.args_cname, i)) + code.putln("values[%d] = __Pyx_Arg_%s(%s, %d);" % ( + i, self.signature.fastvar, Naming.args_cname, i)) else: # parse the positional arguments from the variable length # args tuple and reject illegal argument tuple sizes - code.putln('switch (PyTuple_GET_SIZE(%s)) {' % Naming.args_cname) + code.putln('switch (%s) {' % Naming.nargs_cname) if self.star_arg: code.putln('default:') reversed_args = list(enumerate(positional_args))[::-1] @@ -3736,7 +4133,8 @@ class DefNodeWrapper(FuncDefNode): if i != reversed_args[0][0]: code.putln('CYTHON_FALLTHROUGH;') code.put('case %2d: ' % (i+1)) - code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (i, Naming.args_cname, i)) + code.putln("values[%d] = __Pyx_Arg_%s(%s, %d);" % ( + i, self.signature.fastvar, Naming.args_cname, i)) if min_positional_args == 0: code.putln('CYTHON_FALLTHROUGH;') code.put('case 0: ') @@ -3751,7 +4149,7 @@ class DefNodeWrapper(FuncDefNode): code.put_goto(argtuple_error_label) code.putln('}') - code.putln('}') # end of the conditional unpacking blocks + code.putln('}') # end of the conditional unpacking blocks # Convert arg values to their final type and assign them. # Also inject non-Python default arguments, which do cannot @@ -3759,17 +4157,17 @@ class DefNodeWrapper(FuncDefNode): for i, arg in enumerate(all_args): self.generate_arg_assignment(arg, "values[%d]" % i, code) - code.putln('}') # end of the whole argument unpacking block + code.putln('}') # end of the whole argument unpacking block if code.label_used(argtuple_error_label): code.put_goto(success_label) code.put_label(argtuple_error_label) code.globalstate.use_utility_code( UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c")) - code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, PyTuple_GET_SIZE(%s)); ' % ( - self.name, has_fixed_positional_count, + code.put('__Pyx_RaiseArgtupleInvalid(%s, %d, %d, %d, %s); ' % ( + self_name_csafe, has_fixed_positional_count, min_positional_args, max_positional_args, - Naming.args_cname)) + Naming.nargs_cname)) code.putln(code.error_goto(self.pos)) def generate_arg_assignment(self, arg, item, code): @@ -3792,8 +4190,7 @@ class DefNodeWrapper(FuncDefNode): arg.entry.cname, arg.calculate_default_value_code(code))) if arg.type.is_memoryviewslice: - code.put_incref_memoryviewslice(arg.entry.cname, - have_gil=True) + code.put_var_incref_memoryviewslice(arg.entry, have_gil=True) code.putln('}') else: error(arg.pos, "Cannot convert Python object argument to type '%s'" % arg.type) @@ -3805,26 +4202,30 @@ class DefNodeWrapper(FuncDefNode): self.starstar_arg.entry.cname, self.starstar_arg.entry.cname, self.error_value())) - code.put_gotref(self.starstar_arg.entry.cname) + code.put_var_gotref(self.starstar_arg.entry) if self.star_arg: self.star_arg.entry.xdecref_cleanup = 0 - code.putln('if (PyTuple_GET_SIZE(%s) > %d) {' % ( - Naming.args_cname, - max_positional_args)) - code.putln('%s = PyTuple_GetSlice(%s, %d, PyTuple_GET_SIZE(%s));' % ( - self.star_arg.entry.cname, Naming.args_cname, - max_positional_args, Naming.args_cname)) - code.putln("if (unlikely(!%s)) {" % self.star_arg.entry.cname) - if self.starstar_arg: - code.put_decref_clear(self.starstar_arg.entry.cname, py_object_type) - code.put_finish_refcount_context() - code.putln('return %s;' % self.error_value()) - code.putln('}') - code.put_gotref(self.star_arg.entry.cname) - code.putln('} else {') - code.put("%s = %s; " % (self.star_arg.entry.cname, Naming.empty_tuple)) - code.put_incref(Naming.empty_tuple, py_object_type) - code.putln('}') + if max_positional_args == 0: + # If there are no positional arguments, use the args tuple + # directly + assert not self.signature.use_fastcall + code.put_incref(Naming.args_cname, py_object_type) + code.putln("%s = %s;" % (self.star_arg.entry.cname, Naming.args_cname)) + else: + # It is possible that this is a slice of "negative" length, + # as in args[5:3]. That's not a problem, the function below + # handles that efficiently and returns the empty tuple. + code.putln('%s = __Pyx_ArgsSlice_%s(%s, %d, %s);' % ( + self.star_arg.entry.cname, self.signature.fastvar, + Naming.args_cname, max_positional_args, Naming.nargs_cname)) + code.putln("if (unlikely(!%s)) {" % + self.star_arg.entry.type.nullcheck_string(self.star_arg.entry.cname)) + if self.starstar_arg: + code.put_var_decref_clear(self.starstar_arg.entry) + code.put_finish_refcount_context() + code.putln('return %s;' % self.error_value()) + code.putln('}') + code.put_var_gotref(self.star_arg.entry) def generate_argument_values_setup_code(self, args, code): max_args = len(args) @@ -3846,22 +4247,45 @@ class DefNodeWrapper(FuncDefNode): code.putln('values[%d] = %s;' % (i, arg.type.as_pyobject(default_value))) def generate_keyword_unpacking_code(self, min_positional_args, max_positional_args, - has_fixed_positional_count, has_kw_only_args, - all_args, argtuple_error_label, code): + has_fixed_positional_count, + has_kw_only_args, all_args, argtuple_error_label, code): + # First we count how many arguments must be passed as positional + num_required_posonly_args = num_pos_only_args = 0 + for i, arg in enumerate(all_args): + if arg.pos_only: + num_pos_only_args += 1 + if not arg.default: + num_required_posonly_args += 1 + code.putln('Py_ssize_t kw_args;') - code.putln('const Py_ssize_t pos_args = PyTuple_GET_SIZE(%s);' % Naming.args_cname) # copy the values from the args tuple and check that it's not too long - code.putln('switch (pos_args) {') + code.putln('switch (%s) {' % Naming.nargs_cname) if self.star_arg: code.putln('default:') - for i in range(max_positional_args-1, -1, -1): + + for i in range(max_positional_args-1, num_required_posonly_args-1, -1): code.put('case %2d: ' % (i+1)) - code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % ( - i, Naming.args_cname, i)) + code.putln("values[%d] = __Pyx_Arg_%s(%s, %d);" % ( + i, self.signature.fastvar, Naming.args_cname, i)) code.putln('CYTHON_FALLTHROUGH;') - code.putln('case 0: break;') + if num_required_posonly_args > 0: + code.put('case %2d: ' % num_required_posonly_args) + for i in range(num_required_posonly_args-1, -1, -1): + code.putln("values[%d] = __Pyx_Arg_%s(%s, %d);" % ( + i, self.signature.fastvar, Naming.args_cname, i)) + code.putln('break;') + for i in range(num_required_posonly_args-2, -1, -1): + code.put('case %2d: ' % (i+1)) + code.putln('CYTHON_FALLTHROUGH;') + + code.put('case 0: ') + if num_required_posonly_args == 0: + code.putln('break;') + else: + # catch-all for not enough pos-only args passed + code.put_goto(argtuple_error_label) if not self.star_arg: - code.put('default: ') # more arguments than allowed + code.put('default: ') # more arguments than allowed code.put_goto(argtuple_error_label) code.putln('}') @@ -3874,7 +4298,10 @@ class DefNodeWrapper(FuncDefNode): # If we received kwargs, fill up the positional/required # arguments with values from the kw dict - code.putln('kw_args = PyDict_Size(%s);' % Naming.kwds_cname) + self_name_csafe = self.name.as_c_string_literal() + + code.putln('kw_args = __Pyx_NumKwargs_%s(%s);' % ( + self.signature.fastvar, Naming.kwds_cname)) if self.num_required_args or max_positional_args > 0: last_required_arg = -1 for i, arg in enumerate(all_args): @@ -3882,30 +4309,32 @@ class DefNodeWrapper(FuncDefNode): last_required_arg = i if last_required_arg < max_positional_args: last_required_arg = max_positional_args-1 - if max_positional_args > 0: - code.putln('switch (pos_args) {') - for i, arg in enumerate(all_args[:last_required_arg+1]): - if max_positional_args > 0 and i <= max_positional_args: - if i != 0: + if max_positional_args > num_pos_only_args: + code.putln('switch (%s) {' % Naming.nargs_cname) + for i, arg in enumerate(all_args[num_pos_only_args:last_required_arg+1], num_pos_only_args): + if max_positional_args > num_pos_only_args and i <= max_positional_args: + if i != num_pos_only_args: code.putln('CYTHON_FALLTHROUGH;') if self.star_arg and i == max_positional_args: code.putln('default:') else: code.putln('case %2d:' % i) - pystring_cname = code.intern_identifier(arg.name) + pystring_cname = code.intern_identifier(arg.entry.name) if arg.default: if arg.kw_only: # optional kw-only args are handled separately below continue code.putln('if (kw_args > 0) {') # don't overwrite default argument - code.putln('PyObject* value = __Pyx_PyDict_GetItemStr(%s, %s);' % ( - Naming.kwds_cname, pystring_cname)) + code.putln('PyObject* value = __Pyx_GetKwValue_%s(%s, %s, %s);' % ( + self.signature.fastvar, Naming.kwds_cname, Naming.kwvalues_cname, pystring_cname)) code.putln('if (value) { values[%d] = value; kw_args--; }' % i) + code.putln('else if (unlikely(PyErr_Occurred())) %s' % code.error_goto(self.pos)) code.putln('}') else: - code.putln('if (likely((values[%d] = __Pyx_PyDict_GetItemStr(%s, %s)) != 0)) kw_args--;' % ( - i, Naming.kwds_cname, pystring_cname)) + code.putln('if (likely((values[%d] = __Pyx_GetKwValue_%s(%s, %s, %s)) != 0)) kw_args--;' % ( + i, self.signature.fastvar, Naming.kwds_cname, Naming.kwvalues_cname, pystring_cname)) + code.putln('else if (unlikely(PyErr_Occurred())) %s' % code.error_goto(self.pos)) if i < min_positional_args: if i == 0: # special case: we know arg 0 is missing @@ -3918,8 +4347,8 @@ class DefNodeWrapper(FuncDefNode): code.putln('else {') code.globalstate.use_utility_code( UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c")) - code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, %d); ' % ( - self.name, has_fixed_positional_count, + code.put('__Pyx_RaiseArgtupleInvalid(%s, %d, %d, %d, %d); ' % ( + self_name_csafe, has_fixed_positional_count, min_positional_args, max_positional_args, i)) code.putln(code.error_goto(self.pos)) code.putln('}') @@ -3927,11 +4356,11 @@ class DefNodeWrapper(FuncDefNode): code.putln('else {') code.globalstate.use_utility_code( UtilityCode.load_cached("RaiseKeywordRequired", "FunctionArguments.c")) - code.put('__Pyx_RaiseKeywordRequired("%s", %s); ' % ( - self.name, pystring_cname)) + code.put('__Pyx_RaiseKeywordRequired(%s, %s); ' % ( + self_name_csafe, pystring_cname)) code.putln(code.error_goto(self.pos)) code.putln('}') - if max_positional_args > 0: + if max_positional_args > num_pos_only_args: code.putln('}') if has_kw_only_args: @@ -3947,34 +4376,69 @@ class DefNodeWrapper(FuncDefNode): # arguments, this will always do the right thing for unpacking # keyword arguments, so that we can concentrate on optimising # common cases above. + # + # ParseOptionalKeywords() needs to know how many of the arguments + # that could be passed as keywords have in fact been passed as + # positional args. + if num_pos_only_args > 0: + # There are positional-only arguments which we don't want to count, + # since they cannot be keyword arguments. Subtract the number of + # pos-only arguments from the number of positional arguments we got. + # If we get a negative number then none of the keyword arguments were + # passed as positional args. + code.putln('const Py_ssize_t kwd_pos_args = (unlikely(%s < %d)) ? 0 : %s - %d;' % ( + Naming.nargs_cname, num_pos_only_args, + Naming.nargs_cname, num_pos_only_args, + )) + elif max_positional_args > 0: + code.putln('const Py_ssize_t kwd_pos_args = %s;' % Naming.nargs_cname) + if max_positional_args == 0: pos_arg_count = "0" elif self.star_arg: - code.putln("const Py_ssize_t used_pos_args = (pos_args < %d) ? pos_args : %d;" % ( - max_positional_args, max_positional_args)) + # If there is a *arg, the number of used positional args could be larger than + # the number of possible keyword arguments. But ParseOptionalKeywords() uses the + # number of positional args as an index into the keyword argument name array, + # if this is larger than the number of kwd args we get a segfault. So round + # this down to max_positional_args - num_pos_only_args (= num possible kwd args). + code.putln("const Py_ssize_t used_pos_args = (kwd_pos_args < %d) ? kwd_pos_args : %d;" % ( + max_positional_args - num_pos_only_args, max_positional_args - num_pos_only_args)) pos_arg_count = "used_pos_args" else: - pos_arg_count = "pos_args" + pos_arg_count = "kwd_pos_args" + if num_pos_only_args < len(all_args): + values_array = 'values + %d' % num_pos_only_args + else: + values_array = 'values' code.globalstate.use_utility_code( UtilityCode.load_cached("ParseKeywords", "FunctionArguments.c")) - code.putln('if (unlikely(__Pyx_ParseOptionalKeywords(%s, %s, %s, values, %s, "%s") < 0)) %s' % ( + code.putln('if (unlikely(__Pyx_ParseOptionalKeywords(%s, %s, %s, %s, %s, %s, %s) < 0)) %s' % ( Naming.kwds_cname, + Naming.kwvalues_cname, Naming.pykwdlist_cname, self.starstar_arg and self.starstar_arg.entry.cname or '0', + values_array, pos_arg_count, - self.name, + self_name_csafe, code.error_goto(self.pos))) code.putln('}') def generate_optional_kwonly_args_unpacking_code(self, all_args, code): optional_args = [] first_optional_arg = -1 + num_posonly_args = 0 for i, arg in enumerate(all_args): + if arg.pos_only: + num_posonly_args += 1 if not arg.kw_only or not arg.default: continue if not optional_args: first_optional_arg = i optional_args.append(arg.name) + if num_posonly_args > 0: + posonly_correction = '-%d' % num_posonly_args + else: + posonly_correction = '' if optional_args: if len(optional_args) > 1: # if we receive more than the named kwargs, we either have **kwargs @@ -3990,9 +4454,14 @@ class DefNodeWrapper(FuncDefNode): else: code.putln('if (kw_args == 1) {') code.putln('const Py_ssize_t index = %d;' % first_optional_arg) - code.putln('PyObject* value = __Pyx_PyDict_GetItemStr(%s, *%s[index]);' % ( - Naming.kwds_cname, Naming.pykwdlist_cname)) + code.putln('PyObject* value = __Pyx_GetKwValue_%s(%s, %s, *%s[index%s]);' % ( + self.signature.fastvar, + Naming.kwds_cname, + Naming.kwvalues_cname, + Naming.pykwdlist_cname, + posonly_correction)) code.putln('if (value) { values[index] = value; kw_args--; }') + code.putln('else if (unlikely(PyErr_Occurred())) %s' % code.error_goto(self.pos)) if len(optional_args) > 1: code.putln('}') code.putln('}') @@ -4061,6 +4530,36 @@ class DefNodeWrapper(FuncDefNode): arg.type.is_buffer or arg.type.is_memoryviewslice): self.generate_arg_none_check(arg, code) + if self.target.entry.is_special: + for n in reversed(range(len(self.args), self.signature.max_num_fixed_args())): + # for special functions with optional args (e.g. power which can + # take 2 or 3 args), unused args are None since this is what the + # compilers sets + if self.target.entry.name == "__ipow__": + # Bug in Python < 3.8 - __ipow__ is used as a binary function + # and attempts to access the third argument will always fail + code.putln("#if PY_VERSION_HEX >= 0x03080000") + code.putln("if (unlikely(unused_arg_%s != Py_None)) {" % n) + code.putln( + 'PyErr_SetString(PyExc_TypeError, ' + '"%s() takes %s arguments but %s were given");' % ( + self.target.entry.qualified_name, self.signature.max_num_fixed_args(), n)) + code.putln("%s;" % code.error_goto(self.pos)) + code.putln("}") + if self.target.entry.name == "__ipow__": + code.putln("#endif /*PY_VERSION_HEX >= 0x03080000*/") + if self.target.entry.name == "__ipow__" and len(self.args) != 2: + # It's basically impossible to safely support it: + # Class().__ipow__(1) is guaranteed to crash. + # Therefore, raise an error. + # Use "if" instead of "#if" to avoid warnings about unused variables + code.putln("if ((PY_VERSION_HEX < 0x03080000)) {") + code.putln( + 'PyErr_SetString(PyExc_NotImplementedError, ' + '"3-argument %s cannot be used in Python<3.8");' % ( + self.target.entry.qualified_name)) + code.putln("%s;" % code.error_goto(self.pos)) + code.putln('}') def error_value(self): return self.signature.error_value @@ -4073,9 +4572,7 @@ class GeneratorDefNode(DefNode): # is_generator = True - is_coroutine = False is_iterable_coroutine = False - is_asyncgen = False gen_type_name = 'Generator' needs_closure = True @@ -4110,7 +4607,7 @@ class GeneratorDefNode(DefNode): code.putln('%s = __Pyx_CyFunction_GetClassObj(%s);' % ( classobj_cname, Naming.self_cname)) code.put_incref(classobj_cname, py_object_type) - code.put_giveref(classobj_cname) + code.put_giveref(classobj_cname, py_object_type) code.put_finish_refcount_context() code.putln('return (PyObject *) gen;') code.putln('}') @@ -4234,7 +4731,7 @@ class GeneratorBodyDefNode(DefNode): code.putln("%s = %s; %s" % ( Naming.retval_cname, comp_init, code.error_goto_if_null(Naming.retval_cname, self.pos))) - code.put_gotref(Naming.retval_cname) + code.put_gotref(Naming.retval_cname, py_object_type) # ----- Function body self.generate_function_body(env, code) @@ -4280,7 +4777,7 @@ class GeneratorBodyDefNode(DefNode): # ----- Non-error return cleanup code.put_label(code.return_label) if self.is_inlined: - code.put_xgiveref(Naming.retval_cname) + code.put_xgiveref(Naming.retval_cname, py_object_type) else: code.put_xdecref_clear(Naming.retval_cname, py_object_type) # For Py3.7, clearing is already done below. @@ -4357,7 +4854,10 @@ class OverrideCheckNode(StatNode): return self def generate_execution_code(self, code): - interned_attr_cname = code.intern_identifier(self.py_func.entry.name) + # For fused functions, look up the dispatch function, not the specialisation. + method_entry = self.py_func.fused_py_func.entry if self.py_func.fused_py_func else self.py_func.entry + interned_attr_cname = code.intern_identifier(method_entry.name) + # Check to see if we are an extension type if self.py_func.is_module_scope: self_arg = "((PyObject *)%s)" % Naming.module_cname @@ -4369,8 +4869,8 @@ class OverrideCheckNode(StatNode): if self.py_func.is_module_scope: code.putln("else {") else: - code.putln("else if (unlikely((Py_TYPE(%s)->tp_dictoffset != 0)" - " || (Py_TYPE(%s)->tp_flags & (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))) {" % ( + code.putln("else if (unlikely((Py_TYPE(%s)->tp_dictoffset != 0) || " + "__Pyx_PyType_HasFeature(Py_TYPE(%s), (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))) {" % ( self_arg, self_arg)) code.putln("#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS") @@ -4396,12 +4896,16 @@ class OverrideCheckNode(StatNode): err = code.error_goto_if_null(func_node_temp, self.pos) code.putln("%s = __Pyx_PyObject_GetAttrStr(%s, %s); %s" % ( func_node_temp, self_arg, interned_attr_cname, err)) - code.put_gotref(func_node_temp) + code.put_gotref(func_node_temp, py_object_type) - is_builtin_function_or_method = "PyCFunction_Check(%s)" % func_node_temp is_overridden = "(PyCFunction_GET_FUNCTION(%s) != (PyCFunction)(void*)%s)" % ( - func_node_temp, self.py_func.entry.func_cname) - code.putln("if (!%s || %s) {" % (is_builtin_function_or_method, is_overridden)) + func_node_temp, method_entry.func_cname) + code.putln("#ifdef __Pyx_CyFunction_USED") + code.putln("if (!__Pyx_IsCyOrPyCFunction(%s)" % func_node_temp) + code.putln("#else") + code.putln("if (!PyCFunction_Check(%s)" % func_node_temp) + code.putln("#endif") + code.putln(" || %s) {" % is_overridden) self.body.generate_execution_code(code) code.putln("}") @@ -4443,25 +4947,31 @@ class PyClassDefNode(ClassDefNode): # A Python class definition. # # name EncodedString Name of the class - # doc string or None + # doc string or None The class docstring # body StatNode Attribute definition code # entry Symtab.Entry # scope PyClassScope # decorators [DecoratorNode] list of decorators or None + # bases ExprNode Expression that evaluates to a tuple of base classes # # The following subnodes are constructed internally: # + # doc_node NameNode '__doc__' name that is made available to the class body # dict DictNode Class dictionary or Py3 namespace # classobj ClassNode Class object # target NameNode Variable to assign class object to + # orig_bases None or ExprNode "bases" before transformation by PEP560 __mro_entries__, + # used to create the __orig_bases__ attribute - child_attrs = ["body", "dict", "metaclass", "mkw", "bases", "class_result", - "target", "class_cell", "decorators"] + child_attrs = ["doc_node", "body", "dict", "metaclass", "mkw", "bases", "class_result", + "target", "class_cell", "decorators", "orig_bases"] decorators = None class_result = None is_py3_style_class = False # Python3 style class (kwargs) metaclass = None mkw = None + doc_node = None + orig_bases = None def __init__(self, pos, name, bases, doc, body, decorators=None, keyword_args=None, force_py3_semantics=False): @@ -4475,6 +4985,7 @@ class PyClassDefNode(ClassDefNode): if self.doc and Options.docstrings: doc = embed_position(self.pos, self.doc) doc_node = ExprNodes.StringNode(pos, value=doc) + self.doc_node = ExprNodes.NameNode(name=EncodedString('__doc__'), type=py_object_type, pos=pos) else: doc_node = None @@ -4523,7 +5034,9 @@ class PyClassDefNode(ClassDefNode): self.classobj = ExprNodes.Py3ClassNode( pos, name=name, class_def_node=self, doc=doc_node, calculate_metaclass=needs_metaclass_calculation, - allow_py2_metaclass=allow_py2_metaclass) + allow_py2_metaclass=allow_py2_metaclass, + force_type=force_py3_semantics, + ) else: # no bases, no metaclass => old style class creation self.dict = ExprNodes.DictNode(pos, key_value_pairs=[]) @@ -4560,7 +5073,7 @@ class PyClassDefNode(ClassDefNode): return cenv def analyse_declarations(self, env): - class_result = self.classobj + unwrapped_class_result = class_result = self.classobj if self.decorators: from .ExprNodes import SimpleCallNode for decorator in self.decorators[::-1]: @@ -4579,9 +5092,27 @@ class PyClassDefNode(ClassDefNode): cenv = self.create_scope(env) cenv.directives = env.directives cenv.class_obj_cname = self.target.entry.cname + if self.doc_node: + self.doc_node.analyse_target_declaration(cenv) self.body.analyse_declarations(cenv) + unwrapped_class_result.analyse_annotations(cenv) + + update_bases_functype = PyrexTypes.CFuncType( + PyrexTypes.py_object_type, [ + PyrexTypes.CFuncTypeArg("bases", PyrexTypes.py_object_type, None) + ]) def analyse_expressions(self, env): + if self.bases and not (self.bases.is_sequence_constructor and len(self.bases.args) == 0): + from .ExprNodes import PythonCapiCallNode, CloneNode + # handle the Python 3.7 __mro_entries__ transformation + orig_bases = self.bases.analyse_expressions(env) + self.bases = PythonCapiCallNode(orig_bases.pos, + function_name="__Pyx_PEP560_update_bases", + func_type=self.update_bases_functype, + utility_code=UtilityCode.load_cached('Py3UpdateBases', 'ObjectHandling.c'), + args=[CloneNode(orig_bases)]) + self.orig_bases = orig_bases if self.bases: self.bases = self.bases.analyse_expressions(env) if self.mkw: @@ -4592,7 +5123,7 @@ class PyClassDefNode(ClassDefNode): self.class_result = self.class_result.analyse_expressions(env) cenv = self.scope self.body = self.body.analyse_expressions(cenv) - self.target.analyse_target_expression(env, self.classobj) + self.target = self.target.analyse_target_expression(env, self.classobj) self.class_cell = self.class_cell.analyse_expressions(cenv) return self @@ -4604,6 +5135,8 @@ class PyClassDefNode(ClassDefNode): code.mark_pos(self.pos) code.pyclass_stack.append(self) cenv = self.scope + if self.orig_bases: + self.orig_bases.generate_evaluation_code(code) if self.bases: self.bases.generate_evaluation_code(code) if self.mkw: @@ -4611,6 +5144,17 @@ class PyClassDefNode(ClassDefNode): if self.metaclass: self.metaclass.generate_evaluation_code(code) self.dict.generate_evaluation_code(code) + if self.orig_bases: + # update __orig_bases__ if needed + code.putln("if (%s != %s) {" % (self.bases.result(), self.orig_bases.result())) + code.putln( + code.error_goto_if_neg('PyDict_SetItemString(%s, "__orig_bases__", %s)' % ( + self.dict.result(), self.orig_bases.result()), + self.pos + )) + code.putln("}") + self.orig_bases.generate_disposal_code(code) + self.orig_bases.free_temps(code) cenv.namespace_cname = cenv.class_obj_cname = self.dict.result() class_cell = self.class_cell @@ -4677,6 +5221,10 @@ class CClassDefNode(ClassDefNode): decorators = None shadow = False + @property + def punycode_class_name(self): + return punycodify_name(self.class_name) + def buffer_defaults(self, env): if not hasattr(self, '_buffer_defaults'): from . import Buffer @@ -4713,6 +5261,8 @@ class CClassDefNode(ClassDefNode): api=self.api, buffer_defaults=self.buffer_defaults(env), shadow=self.shadow) + if self.bases and len(self.bases.args) > 1: + self.entry.type.multiple_bases = True def analyse_declarations(self, env): #print "CClassDefNode.analyse_declarations:", self.class_name @@ -4757,7 +5307,8 @@ class CClassDefNode(ClassDefNode): error(base.pos, "Base class '%s' of type '%s' is final" % ( base_type, self.class_name)) elif base_type.is_builtin_type and \ - base_type.name in ('tuple', 'str', 'bytes'): + base_type.name in ('tuple', 'bytes'): + # str in Py2 is also included in this, but now checked at run-time error(base.pos, "inheritance from PyVarObject types like '%s' is not currently supported" % base_type.name) else: @@ -4783,7 +5334,7 @@ class CClassDefNode(ClassDefNode): if self.visibility == 'extern': if (self.module_name == '__builtin__' and self.class_name in Builtin.builtin_types and - env.qualified_name[:8] != 'cpython.'): # allow overloaded names for cimporting from cpython + env.qualified_name[:8] != 'cpython.'): # allow overloaded names for cimporting from cpython warning(self.pos, "%s already a builtin Cython type" % self.class_name, 1) self.entry = home_scope.declare_c_class( @@ -4801,6 +5352,8 @@ class CClassDefNode(ClassDefNode): api=self.api, buffer_defaults=self.buffer_defaults(env), shadow=self.shadow) + if self.bases and len(self.bases.args) > 1: + self.entry.type.multiple_bases = True if self.shadow: home_scope.lookup(self.class_name).as_variable = self.entry @@ -4809,6 +5362,15 @@ class CClassDefNode(ClassDefNode): self.scope = scope = self.entry.type.scope if scope is not None: scope.directives = env.directives + if "dataclasses.dataclass" in env.directives: + is_frozen = False + # Retrieve the @dataclass config (args, kwargs), as passed into the decorator. + dataclass_config = env.directives["dataclasses.dataclass"] + if dataclass_config: + decorator_kwargs = dataclass_config[1] + frozen_flag = decorator_kwargs.get('frozen') + is_frozen = frozen_flag and frozen_flag.is_literal and frozen_flag.value + scope.is_c_dataclass_scope = "frozen" if is_frozen else True if self.doc and Options.docstrings: scope.doc = embed_position(self.pos, self.doc) @@ -4868,71 +5430,206 @@ class CClassDefNode(ClassDefNode): # This is needed to generate evaluation code for # default values of method arguments. code.mark_pos(self.pos) - if self.body: - self.body.generate_execution_code(code) if not self.entry.type.early_init: + bases = None if self.type_init_args: + # Extract bases tuple and validate 'best base' by actually calling 'type()'. + bases = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True) + self.type_init_args.generate_evaluation_code(code) - bases = "PyTuple_GET_ITEM(%s, 1)" % self.type_init_args.result() + code.putln("%s = PyTuple_GET_ITEM(%s, 1);" % (bases, self.type_init_args.result())) + code.put_incref(bases, PyrexTypes.py_object_type) + first_base = "((PyTypeObject*)PyTuple_GET_ITEM(%s, 0))" % bases # Let Python do the base types compatibility checking. - trial_type = code.funcstate.allocate_temp(PyrexTypes.py_object_type, True) - code.putln("%s = PyType_Type.tp_new(&PyType_Type, %s, NULL);" % ( + trial_type = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True) + code.putln("%s = __Pyx_PyType_GetSlot(&PyType_Type, tp_new, newfunc)(&PyType_Type, %s, NULL);" % ( trial_type, self.type_init_args.result())) code.putln(code.error_goto_if_null(trial_type, self.pos)) - code.put_gotref(trial_type) - code.putln("if (((PyTypeObject*) %s)->tp_base != %s) {" % ( + code.put_gotref(trial_type, py_object_type) + code.putln("if (__Pyx_PyType_GetSlot((PyTypeObject*) %s, tp_base, PyTypeObject*) != %s) {" % ( trial_type, first_base)) - code.putln("PyErr_Format(PyExc_TypeError, \"best base '%s' must be equal to first base '%s'\",") - code.putln(" ((PyTypeObject*) %s)->tp_base->tp_name, %s->tp_name);" % ( - trial_type, first_base)) + trial_type_base = "__Pyx_PyType_GetSlot((PyTypeObject*) %s, tp_base, PyTypeObject*)" % trial_type + code.putln("__Pyx_TypeName base_name = __Pyx_PyType_GetName(%s);" % trial_type_base) + code.putln("__Pyx_TypeName type_name = __Pyx_PyType_GetName(%s);" % first_base) + code.putln("PyErr_Format(PyExc_TypeError, " + "\"best base '\" __Pyx_FMT_TYPENAME \"' must be equal to first base '\" __Pyx_FMT_TYPENAME \"'\",") + code.putln(" base_name, type_name);") + code.putln("__Pyx_DECREF_TypeName(base_name);") + code.putln("__Pyx_DECREF_TypeName(type_name);") code.putln(code.error_goto(self.pos)) code.putln("}") - code.funcstate.release_temp(trial_type) - code.put_incref(bases, PyrexTypes.py_object_type) - code.put_giveref(bases) - code.putln("%s.tp_bases = %s;" % (self.entry.type.typeobj_cname, bases)) + code.put_decref_clear(trial_type, PyrexTypes.py_object_type) + code.funcstate.release_temp(trial_type) + self.type_init_args.generate_disposal_code(code) self.type_init_args.free_temps(code) - self.generate_type_ready_code(self.entry, code, True) + self.generate_type_ready_code(self.entry, code, bases_tuple_cname=bases, check_heap_type_bases=True) + if bases is not None: + code.put_decref_clear(bases, PyrexTypes.py_object_type) + code.funcstate.release_temp(bases) + + if self.body: + self.body.generate_execution_code(code) # Also called from ModuleNode for early init types. @staticmethod - def generate_type_ready_code(entry, code, heap_type_bases=False): + def generate_type_ready_code(entry, code, bases_tuple_cname=None, check_heap_type_bases=False): # Generate a call to PyType_Ready for an extension # type defined in this module. type = entry.type - typeobj_cname = type.typeobj_cname + typeptr_cname = type.typeptr_cname scope = type.scope if not scope: # could be None if there was an error return - if entry.visibility != 'extern': - for slot in TypeSlots.slot_table: - slot.generate_dynamic_init_code(scope, code) - if heap_type_bases: - code.globalstate.use_utility_code( - UtilityCode.load_cached('PyType_Ready', 'ExtensionTypes.c')) - readyfunc = "__Pyx_PyType_Ready" + if entry.visibility == 'extern': + # Generate code to initialise the typeptr of an external extension + # type defined in this module to point to its type object. + if type.typeobj_cname: + # FIXME: this should not normally be set :-? + assert not type.typeobj_cname + code.putln("%s = &%s;" % ( + type.typeptr_cname, + type.typeobj_cname, + )) + return + # TODO: remove 'else:' and dedent + else: + assert typeptr_cname + assert type.typeobj_cname + typespec_cname = "%s_spec" % type.typeobj_cname + code.putln("#if CYTHON_USE_TYPE_SPECS") + tuple_temp = None + if not bases_tuple_cname and scope.parent_type.base_type: + tuple_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True) + code.putln("%s = PyTuple_Pack(1, (PyObject *)%s); %s" % ( + tuple_temp, + scope.parent_type.base_type.typeptr_cname, + code.error_goto_if_null(tuple_temp, entry.pos), + )) + code.put_gotref(tuple_temp, py_object_type) + + if bases_tuple_cname or tuple_temp: + if check_heap_type_bases: + code.globalstate.use_utility_code( + UtilityCode.load_cached('ValidateBasesTuple', 'ExtensionTypes.c')) + code.put_error_if_neg(entry.pos, "__Pyx_validate_bases_tuple(%s.name, %s, %s)" % ( + typespec_cname, + TypeSlots.get_slot_by_name("tp_dictoffset", scope.directives).slot_code(scope), + bases_tuple_cname or tuple_temp, + )) + + code.putln("%s = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(%s, &%s, %s);" % ( + typeptr_cname, + Naming.module_cname, + typespec_cname, + bases_tuple_cname or tuple_temp, + )) + if tuple_temp: + code.put_xdecref_clear(tuple_temp, type=py_object_type) + code.funcstate.release_temp(tuple_temp) + code.putln(code.error_goto_if_null(typeptr_cname, entry.pos)) else: - readyfunc = "PyType_Ready" - code.putln( - "if (%s(&%s) < 0) %s" % ( - readyfunc, - typeobj_cname, - code.error_goto(entry.pos))) - # Don't inherit tp_print from builtin types, restoring the + code.putln( + "%s = (PyTypeObject *) __Pyx_PyType_FromModuleAndSpec(%s, &%s, NULL); %s" % ( + typeptr_cname, + Naming.module_cname, + typespec_cname, + code.error_goto_if_null(typeptr_cname, entry.pos), + )) + + # The buffer interface is not currently supported by PyType_FromSpec(). + buffer_slot = TypeSlots.get_slot_by_name("tp_as_buffer", code.globalstate.directives) + if not buffer_slot.is_empty(scope): + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") + code.putln("%s->%s = %s;" % ( + typeptr_cname, + buffer_slot.slot_name, + buffer_slot.slot_code(scope), + )) + # Still need to inherit buffer methods since PyType_Ready() didn't do it for us. + for buffer_method_name in ("__getbuffer__", "__releasebuffer__"): + buffer_slot = TypeSlots.get_slot_table( + code.globalstate.directives).get_slot_by_method_name(buffer_method_name) + if buffer_slot.slot_code(scope) == "0" and not TypeSlots.get_base_slot_function(scope, buffer_slot): + code.putln("if (!%s->tp_as_buffer->%s &&" + " %s->tp_base->tp_as_buffer &&" + " %s->tp_base->tp_as_buffer->%s) {" % ( + typeptr_cname, buffer_slot.slot_name, + typeptr_cname, + typeptr_cname, buffer_slot.slot_name, + )) + code.putln("%s->tp_as_buffer->%s = %s->tp_base->tp_as_buffer->%s;" % ( + typeptr_cname, buffer_slot.slot_name, + typeptr_cname, buffer_slot.slot_name, + )) + code.putln("}") + code.putln("#elif defined(Py_bf_getbuffer) && defined(Py_bf_releasebuffer)") + code.putln("/* PY_VERSION_HEX >= 0x03090000 || Py_LIMITED_API >= 0x030B0000 */") + code.putln("#elif defined(_MSC_VER)") + code.putln("#pragma message (\"The buffer protocol is not supported in the Limited C-API < 3.11.\")") + code.putln("#else") + code.putln("#warning \"The buffer protocol is not supported in the Limited C-API < 3.11.\"") + code.putln("#endif") + + code.globalstate.use_utility_code( + UtilityCode.load_cached("FixUpExtensionType", "ExtensionTypes.c")) + code.put_error_if_neg(entry.pos, "__Pyx_fix_up_extension_type_from_spec(&%s, %s)" % ( + typespec_cname, typeptr_cname)) + + code.putln("#else") + if bases_tuple_cname: + code.put_incref(bases_tuple_cname, py_object_type) + code.put_giveref(bases_tuple_cname, py_object_type) + code.putln("%s.tp_bases = %s;" % (type.typeobj_cname, bases_tuple_cname)) + code.putln("%s = &%s;" % ( + typeptr_cname, + type.typeobj_cname, + )) + code.putln("#endif") # if CYTHON_USE_TYPE_SPECS + + base_type = type.base_type + while base_type: + if base_type.is_external and not base_type.objstruct_cname == "PyTypeObject": + # 'type' is special-cased because it is actually based on PyHeapTypeObject + # Variable length bases are allowed if the current class doesn't grow + code.putln("if (sizeof(%s%s) != sizeof(%s%s)) {" % ( + "" if type.typedef_flag else "struct ", type.objstruct_cname, + "" if base_type.typedef_flag else "struct ", base_type.objstruct_cname)) + code.globalstate.use_utility_code( + UtilityCode.load_cached("ValidateExternBase", "ExtensionTypes.c")) + code.put_error_if_neg(entry.pos, "__Pyx_validate_extern_base(%s)" % ( + type.base_type.typeptr_cname)) + code.putln("}") + break + base_type = base_type.base_type + + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") + # FIXME: these still need to get initialised even with the limited-API + for slot in TypeSlots.get_slot_table(code.globalstate.directives): + slot.generate_dynamic_init_code(scope, code) + code.putln("#endif") + + code.putln("#if !CYTHON_USE_TYPE_SPECS") + code.globalstate.use_utility_code( + UtilityCode.load_cached('PyType_Ready', 'ExtensionTypes.c')) + code.put_error_if_neg(entry.pos, "__Pyx_PyType_Ready(%s)" % typeptr_cname) + code.putln("#endif") + + # Don't inherit tp_print from builtin types in Python 2, restoring the # behavior of using tp_repr or tp_str instead. # ("tp_print" was renamed to "tp_vectorcall_offset" in Py3.8b1) - code.putln("#if PY_VERSION_HEX < 0x030800B1") - code.putln("%s.tp_print = 0;" % typeobj_cname) + code.putln("#if PY_MAJOR_VERSION < 3") + code.putln("%s->tp_print = 0;" % typeptr_cname) code.putln("#endif") # Use specialised attribute lookup for types with generic lookup but no instance dict. getattr_slot_func = TypeSlots.get_slot_code_by_name(scope, 'tp_getattro') dictoffset_slot_func = TypeSlots.get_slot_code_by_name(scope, 'tp_dictoffset') if getattr_slot_func == '0' and dictoffset_slot_func == '0': + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") # FIXME if type.is_final_type: py_cfunc = "__Pyx_PyObject_GenericGetAttrNoDict" # grepable utility_func = "PyObject_GenericGetAttrNoDict" @@ -4942,11 +5639,12 @@ class CClassDefNode(ClassDefNode): code.globalstate.use_utility_code(UtilityCode.load_cached(utility_func, "ObjectHandling.c")) code.putln("if ((CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP) &&" - " likely(!%s.tp_dictoffset && %s.tp_getattro == PyObject_GenericGetAttr)) {" % ( - typeobj_cname, typeobj_cname)) - code.putln("%s.tp_getattro = %s;" % ( - typeobj_cname, py_cfunc)) + " likely(!%s->tp_dictoffset && %s->tp_getattro == PyObject_GenericGetAttr)) {" % ( + typeptr_cname, typeptr_cname)) + code.putln("%s->tp_getattro = %s;" % ( + typeptr_cname, py_cfunc)) code.putln("}") + code.putln("#endif") # if !CYTHON_COMPILING_IN_LIMITED_API # Fix special method docstrings. This is a bit of a hack, but # unless we let PyType_Ready create the slot wrappers we have @@ -4955,19 +5653,20 @@ class CClassDefNode(ClassDefNode): is_buffer = func.name in ('__getbuffer__', '__releasebuffer__') if (func.is_special and Options.docstrings and func.wrapperbase_cname and not is_buffer): - slot = TypeSlots.method_name_to_slot.get(func.name) + slot = TypeSlots.get_slot_table( + entry.type.scope.directives).get_slot_by_method_name(func.name) preprocessor_guard = slot.preprocessor_guard_code() if slot else None if preprocessor_guard: code.putln(preprocessor_guard) code.putln('#if CYTHON_UPDATE_DESCRIPTOR_DOC') code.putln("{") code.putln( - 'PyObject *wrapper = PyObject_GetAttrString((PyObject *)&%s, "%s"); %s' % ( - typeobj_cname, + 'PyObject *wrapper = PyObject_GetAttrString((PyObject *)%s, "%s"); %s' % ( + typeptr_cname, func.name, code.error_goto_if_null('wrapper', entry.pos))) code.putln( - "if (Py_TYPE(wrapper) == &PyWrapperDescr_Type) {") + "if (__Pyx_IS_TYPE(wrapper, &PyWrapperDescr_Type)) {") code.putln( "%s = *((PyWrapperDescrObject *)wrapper)->d_base;" % ( func.wrapperbase_cname)) @@ -4981,34 +5680,34 @@ class CClassDefNode(ClassDefNode): code.putln('#endif') if preprocessor_guard: code.putln('#endif') + if type.vtable_cname: code.globalstate.use_utility_code( UtilityCode.load_cached('SetVTable', 'ImportExport.c')) - code.putln( - "if (__Pyx_SetVtable(%s.tp_dict, %s) < 0) %s" % ( - typeobj_cname, - type.vtabptr_cname, - code.error_goto(entry.pos))) - if heap_type_bases: - code.globalstate.use_utility_code( - UtilityCode.load_cached('MergeVTables', 'ImportExport.c')) - code.putln("if (__Pyx_MergeVtables(&%s) < 0) %s" % ( - typeobj_cname, - code.error_goto(entry.pos))) + code.put_error_if_neg(entry.pos, "__Pyx_SetVtable(%s, %s)" % ( + typeptr_cname, + type.vtabptr_cname, + )) + # TODO: find a way to make this work with the Limited API! + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") + code.globalstate.use_utility_code( + UtilityCode.load_cached('MergeVTables', 'ImportExport.c')) + code.put_error_if_neg(entry.pos, "__Pyx_MergeVtables(%s)" % typeptr_cname) + code.putln("#endif") if not type.scope.is_internal and not type.scope.directives.get('internal'): # scope.is_internal is set for types defined by # Cython (such as closures), the 'internal' # directive is set by users - code.putln( - 'if (PyObject_SetAttr(%s, %s, (PyObject *)&%s) < 0) %s' % ( - Naming.module_cname, - code.intern_identifier(scope.class_name), - typeobj_cname, - code.error_goto(entry.pos))) + code.put_error_if_neg(entry.pos, "PyObject_SetAttr(%s, %s, (PyObject *) %s)" % ( + Naming.module_cname, + code.intern_identifier(scope.class_name), + typeptr_cname, + )) + weakref_entry = scope.lookup_here("__weakref__") if not scope.is_closure_class_scope else None if weakref_entry: if weakref_entry.type is py_object_type: - tp_weaklistoffset = "%s.tp_weaklistoffset" % typeobj_cname + tp_weaklistoffset = "%s->tp_weaklistoffset" % typeptr_cname if type.typedef_flag: objstruct = type.objstruct_cname else: @@ -5020,21 +5719,16 @@ class CClassDefNode(ClassDefNode): weakref_entry.cname)) else: error(weakref_entry.pos, "__weakref__ slot must be of type 'object'") + if scope.lookup_here("__reduce_cython__") if not scope.is_closure_class_scope else None: # Unfortunately, we cannot reliably detect whether a # superclass defined __reduce__ at compile time, so we must # do so at runtime. code.globalstate.use_utility_code( UtilityCode.load_cached('SetupReduce', 'ExtensionTypes.c')) - code.putln('if (__Pyx_setup_reduce((PyObject*)&%s) < 0) %s' % ( - typeobj_cname, - code.error_goto(entry.pos))) - # Generate code to initialise the typeptr of an extension - # type defined in this module to point to its type object. - if type.typeobj_cname: - code.putln( - "%s = &%s;" % ( - type.typeptr_cname, type.typeobj_cname)) + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") # FIXME + code.put_error_if_neg(entry.pos, "__Pyx_setup_reduce((PyObject *) %s)" % typeptr_cname) + code.putln("#endif") def annotate(self, code): if self.type_init_args: @@ -5048,14 +5742,13 @@ class PropertyNode(StatNode): # # name string # doc EncodedString or None Doc string - # entry Symtab.Entry + # entry Symtab.Entry The Entry of the property attribute # body StatListNode child_attrs = ["body"] def analyse_declarations(self, env): self.entry = env.declare_property(self.name, self.doc, self.pos) - self.entry.scope.directives = env.directives self.body.analyse_declarations(self.entry.scope) def analyse_expressions(self, env): @@ -5072,6 +5765,44 @@ class PropertyNode(StatNode): self.body.annotate(code) +class CPropertyNode(StatNode): + """Definition of a C property, backed by a CFuncDefNode getter. + """ + # name string + # doc EncodedString or None Doc string of the property + # entry Symtab.Entry The Entry of the property attribute + # body StatListNode[CFuncDefNode] (for compatibility with PropertyNode) + + child_attrs = ["body"] + is_cproperty = True + + @property + def cfunc(self): + stats = self.body.stats + assert stats and isinstance(stats[0], CFuncDefNode), stats + return stats[0] + + def analyse_declarations(self, env): + scope = PropertyScope(self.name, class_scope=env) + self.body.analyse_declarations(scope) + entry = self.entry = env.declare_property( + self.name, self.doc, self.pos, ctype=self.cfunc.return_type, property_scope=scope) + entry.getter_cname = self.cfunc.entry.cname + + def analyse_expressions(self, env): + self.body = self.body.analyse_expressions(env) + return self + + def generate_function_definitions(self, env, code): + self.body.generate_function_definitions(env, code) + + def generate_execution_code(self, code): + pass + + def annotate(self, code): + self.body.annotate(code) + + class GlobalNode(StatNode): # Global variable declaration. # @@ -5207,12 +5938,14 @@ class SingleAssignmentNode(AssignmentNode): # rhs ExprNode Right hand side # first bool Is this guaranteed the first assignment to lhs? # is_overloaded_assignment bool Is this assignment done via an overloaded operator= + # is_assignment_expression bool Internally SingleAssignmentNode is used to implement assignment expressions # exception_check # exception_value child_attrs = ["lhs", "rhs"] first = False is_overloaded_assignment = False + is_assignment_expression = False declaration_only = False def analyse_declarations(self, env): @@ -5297,7 +6030,17 @@ class SingleAssignmentNode(AssignmentNode): if self.declaration_only: return else: - self.lhs.analyse_target_declaration(env) + if self.is_assignment_expression: + self.lhs.analyse_assignment_expression_target_declaration(env) + else: + self.lhs.analyse_target_declaration(env) + # if an entry doesn't exist that just implies that lhs isn't made up purely + # of AttributeNodes and NameNodes - it isn't useful as a known path to + # a standard library module + if (self.lhs.is_attribute or self.lhs.is_name) and self.lhs.entry and not self.lhs.entry.known_standard_library_import: + stdlib_import_name = self.rhs.get_known_standard_library_import() + if stdlib_import_name: + self.lhs.entry.known_standard_library_import = stdlib_import_name def analyse_types(self, env, use_temp=0): from . import ExprNodes @@ -5320,8 +6063,8 @@ class SingleAssignmentNode(AssignmentNode): elif self.lhs.type.is_array: if not isinstance(self.lhs, ExprNodes.SliceIndexNode): # cannot assign to C array, only to its full slice - self.lhs = ExprNodes.SliceIndexNode(self.lhs.pos, base=self.lhs, start=None, stop=None) - self.lhs = self.lhs.analyse_target_types(env) + lhs = ExprNodes.SliceIndexNode(self.lhs.pos, base=self.lhs, start=None, stop=None) + self.lhs = lhs.analyse_target_types(env) if self.lhs.type.is_cpp_class: op = env.lookup_operator_for_types(self.pos, '=', [self.lhs.type, self.rhs.type]) @@ -5821,7 +6564,7 @@ class ExecStatNode(StatNode): arg.free_temps(code) code.putln( code.error_goto_if_null(temp_result, self.pos)) - code.put_gotref(temp_result) + code.put_gotref(temp_result, py_object_type) code.put_decref_clear(temp_result, py_object_type) code.funcstate.release_temp(temp_result) @@ -6062,9 +6805,15 @@ class RaiseStatNode(StatNode): # exc_value ExprNode or None # exc_tb ExprNode or None # cause ExprNode or None + # + # set in FlowControl + # in_try_block bool child_attrs = ["exc_type", "exc_value", "exc_tb", "cause"] is_terminator = True + builtin_exc_name = None + wrap_tuple_value = False + in_try_block = False def analyse_expressions(self, env): if self.exc_type: @@ -6072,6 +6821,12 @@ class RaiseStatNode(StatNode): self.exc_type = exc_type.coerce_to_pyobject(env) if self.exc_value: exc_value = self.exc_value.analyse_types(env) + if self.wrap_tuple_value: + if exc_value.type is Builtin.tuple_type or not exc_value.type.is_builtin_type: + # prevent tuple values from being interpreted as argument value tuples + from .ExprNodes import TupleNode + exc_value = TupleNode(exc_value.pos, args=[exc_value.coerce_to_pyobject(env)], slow=True) + exc_value = exc_value.analyse_types(env, skip_children=True) self.exc_value = exc_value.coerce_to_pyobject(env) if self.exc_tb: exc_tb = self.exc_tb.analyse_types(env) @@ -6080,7 +6835,6 @@ class RaiseStatNode(StatNode): cause = self.cause.analyse_types(env) self.cause = cause.coerce_to_pyobject(env) # special cases for builtin exceptions - self.builtin_exc_name = None if self.exc_type and not self.exc_value and not self.exc_tb: exc = self.exc_type from . import ExprNodes @@ -6088,9 +6842,19 @@ class RaiseStatNode(StatNode): not (exc.args or (exc.arg_tuple is not None and exc.arg_tuple.args))): exc = exc.function # extract the exception type if exc.is_name and exc.entry.is_builtin: + from . import Symtab self.builtin_exc_name = exc.name if self.builtin_exc_name == 'MemoryError': - self.exc_type = None # has a separate implementation + self.exc_type = None # has a separate implementation + elif (self.builtin_exc_name == 'StopIteration' and + env.is_local_scope and env.name == "__next__" and + env.parent_scope and env.parent_scope.is_c_class_scope and + not self.in_try_block): + # tp_iternext is allowed to return NULL without raising StopIteration. + # For the sake of simplicity, only allow this to happen when not in + # a try block + self.exc_type = None + return self nogil_check = Node.gil_error @@ -6101,6 +6865,11 @@ class RaiseStatNode(StatNode): if self.builtin_exc_name == 'MemoryError': code.putln('PyErr_NoMemory(); %s' % code.error_goto(self.pos)) return + elif self.builtin_exc_name == 'StopIteration' and not self.exc_type: + code.putln('%s = 1;' % Naming.error_without_exception_cname) + code.putln('%s;' % code.error_goto(None)) + code.funcstate.error_without_exception = True + return if self.exc_type: self.exc_type.generate_evaluation_code(code) @@ -6175,10 +6944,10 @@ class ReraiseStatNode(StatNode): vars = code.funcstate.exc_vars if vars: code.globalstate.use_utility_code(restore_exception_utility_code) - code.put_giveref(vars[0]) - code.put_giveref(vars[1]) + code.put_giveref(vars[0], py_object_type) + code.put_giveref(vars[1], py_object_type) # fresh exceptions may not have a traceback yet (-> finally!) - code.put_xgiveref(vars[2]) + code.put_xgiveref(vars[2], py_object_type) code.putln("__Pyx_ErrRestoreWithState(%s, %s, %s);" % tuple(vars)) for varname in vars: code.put("%s = 0; " % varname) @@ -6189,65 +6958,55 @@ class ReraiseStatNode(StatNode): UtilityCode.load_cached("ReRaiseException", "Exceptions.c")) code.putln("__Pyx_ReraiseException(); %s" % code.error_goto(self.pos)) + class AssertStatNode(StatNode): # assert statement # - # cond ExprNode - # value ExprNode or None + # condition ExprNode + # value ExprNode or None + # exception (Raise/GIL)StatNode created from 'value' in PostParse transform - child_attrs = ["cond", "value"] + child_attrs = ["condition", "value", "exception"] + exception = None + + def analyse_declarations(self, env): + assert self.value is None, "Message should have been replaced in PostParse()" + assert self.exception is not None, "Message should have been replaced in PostParse()" + self.exception.analyse_declarations(env) def analyse_expressions(self, env): - self.cond = self.cond.analyse_boolean_expression(env) - if self.value: - value = self.value.analyse_types(env) - if value.type is Builtin.tuple_type or not value.type.is_builtin_type: - # prevent tuple values from being interpreted as argument value tuples - from .ExprNodes import TupleNode - value = TupleNode(value.pos, args=[value], slow=True) - self.value = value.analyse_types(env, skip_children=True).coerce_to_pyobject(env) - else: - self.value = value.coerce_to_pyobject(env) + self.condition = self.condition.analyse_temp_boolean_expression(env) + self.exception = self.exception.analyse_expressions(env) return self - nogil_check = Node.gil_error - gil_message = "Raising exception" - def generate_execution_code(self, code): + code.globalstate.use_utility_code( + UtilityCode.load_cached("AssertionsEnabled", "Exceptions.c")) code.putln("#ifndef CYTHON_WITHOUT_ASSERTIONS") - code.putln("if (unlikely(!Py_OptimizeFlag)) {") + code.putln("if (unlikely(__pyx_assertions_enabled())) {") code.mark_pos(self.pos) - self.cond.generate_evaluation_code(code) - code.putln( - "if (unlikely(!%s)) {" % self.cond.result()) - if self.value: - self.value.generate_evaluation_code(code) - code.putln( - "PyErr_SetObject(PyExc_AssertionError, %s);" % self.value.py_result()) - self.value.generate_disposal_code(code) - self.value.free_temps(code) - else: - code.putln( - "PyErr_SetNone(PyExc_AssertionError);") + self.condition.generate_evaluation_code(code) code.putln( - code.error_goto(self.pos)) + "if (unlikely(!%s)) {" % self.condition.result()) + self.exception.generate_execution_code(code) code.putln( "}") - self.cond.generate_disposal_code(code) - self.cond.free_temps(code) + self.condition.generate_disposal_code(code) + self.condition.free_temps(code) code.putln( "}") + code.putln("#else") + # avoid unused labels etc. + code.putln("if ((1)); else %s" % code.error_goto(self.pos, used=False)) code.putln("#endif") def generate_function_definitions(self, env, code): - self.cond.generate_function_definitions(env, code) - if self.value is not None: - self.value.generate_function_definitions(env, code) + self.condition.generate_function_definitions(env, code) + self.exception.generate_function_definitions(env, code) def annotate(self, code): - self.cond.annotate(code) - if self.value: - self.value.annotate(code) + self.condition.annotate(code) + self.exception.annotate(code) class IfStatNode(StatNode): @@ -6274,13 +7033,9 @@ class IfStatNode(StatNode): code.mark_pos(self.pos) end_label = code.new_label() last = len(self.if_clauses) - if self.else_clause: - # If the 'else' clause is 'unlikely', then set the preceding 'if' clause to 'likely' to reflect that. - self._set_branch_hint(self.if_clauses[-1], self.else_clause, inverse=True) - else: + if not self.else_clause: last -= 1 # avoid redundant goto at end of last if-clause for i, if_clause in enumerate(self.if_clauses): - self._set_branch_hint(if_clause, if_clause.body) if_clause.generate_execution_code(code, end_label, is_last=i == last) if self.else_clause: code.mark_pos(self.else_clause.pos) @@ -6289,21 +7044,6 @@ class IfStatNode(StatNode): code.putln("}") code.put_label(end_label) - def _set_branch_hint(self, clause, statements_node, inverse=False): - if not statements_node.is_terminator: - return - if not isinstance(statements_node, StatListNode) or not statements_node.stats: - return - # Anything that unconditionally raises exceptions should be considered unlikely. - if isinstance(statements_node.stats[-1], (RaiseStatNode, ReraiseStatNode)): - if len(statements_node.stats) > 1: - # Allow simple statements before the 'raise', but no conditions, loops, etc. - non_branch_nodes = (ExprStatNode, AssignmentNode, DelStatNode, GlobalNode, NonlocalNode) - for node in statements_node.stats[:-1]: - if not isinstance(node, non_branch_nodes): - return - clause.branch_hint = 'likely' if inverse else 'unlikely' - def generate_function_definitions(self, env, code): for clause in self.if_clauses: clause.generate_function_definitions(env, code) @@ -6589,7 +7329,7 @@ class DictIterationNextNode(Node): # evaluate all coercions before the assignments for var, result, target in assignments: - code.put_gotref(var.result()) + var.generate_gotref(code) for var, result, target in assignments: result.generate_evaluation_code(code) for var, result, target in assignments: @@ -6651,7 +7391,7 @@ class SetIterationNextNode(Node): code.funcstate.release_temp(result_temp) # evaluate all coercions before the assignments - code.put_gotref(value_ref.result()) + value_ref.generate_gotref(code) self.coerced_value_var.generate_evaluation_code(code) self.value_target.generate_assignment_code(self.coerced_value_var, code) value_ref.release(code) @@ -6719,39 +7459,33 @@ class _ForInStatNode(LoopNode, StatNode): code.mark_pos(self.pos) code.put_label(code.continue_label) code.putln("}") - break_label = code.break_label + + # clean up before we enter the 'else:' branch + self.iterator.generate_disposal_code(code) + + else_label = code.new_label("for_else") if self.else_clause else None + end_label = code.new_label("for_end") + label_intercepts = code.label_interceptor( + [code.break_label], + [end_label], + skip_to_label=else_label or end_label, + pos=self.pos, + ) + + code.mark_pos(self.pos) + for _ in label_intercepts: + self.iterator.generate_disposal_code(code) + code.set_loop_labels(old_loop_labels) + self.iterator.free_temps(code) if self.else_clause: - # In nested loops, the 'else' block can contain 'continue' or 'break' - # statements for the outer loop, but we may need to generate cleanup code - # before taking those paths, so we intercept them here. - orig_exit_labels = (code.continue_label, code.break_label) - code.continue_label = code.new_label('outer_continue') - code.break_label = code.new_label('outer_break') - code.putln("/*else*/ {") + code.put_label(else_label) self.else_clause.generate_execution_code(code) code.putln("}") - needs_goto_end = not self.else_clause.is_terminator - for exit_label, orig_exit_label in zip([code.continue_label, code.break_label], orig_exit_labels): - if not code.label_used(exit_label): - continue - if needs_goto_end: - code.put_goto(break_label) - needs_goto_end = False - code.mark_pos(self.pos) - code.put_label(exit_label) - self.iterator.generate_disposal_code(code) - code.put_goto(orig_exit_label) - code.set_loop_labels(old_loop_labels) - - code.mark_pos(self.pos) - if code.label_used(break_label): - code.put_label(break_label) - self.iterator.generate_disposal_code(code) - self.iterator.free_temps(code) + code.put_label(end_label) def generate_function_definitions(self, env, code): self.target.generate_function_definitions(env, code) @@ -6969,7 +7703,7 @@ class ForFromStatNode(LoopNode, StatNode): target_node.result(), interned_cname, code.error_goto_if_null(target_node.result(), self.target.pos))) - code.put_gotref(target_node.result()) + target_node.generate_gotref(code) else: target_node = self.target from_py_node = ExprNodes.CoerceFromPyTypeNode( @@ -7105,7 +7839,7 @@ class WithStatNode(StatNode): code.intern_identifier(EncodedString('__aexit__' if self.is_async else '__exit__')), code.error_goto_if_null(self.exit_var, self.pos), )) - code.put_gotref(self.exit_var) + code.put_gotref(self.exit_var, py_object_type) # need to free exit_var in the face of exceptions during setup old_error_label = code.new_error_label() @@ -7249,17 +7983,17 @@ class TryExceptStatNode(StatNode): save_exc.putln("__Pyx_ExceptionSave(%s);" % ( ', '.join(['&%s' % var for var in exc_save_vars]))) for var in exc_save_vars: - save_exc.put_xgotref(var) + save_exc.put_xgotref(var, py_object_type) def restore_saved_exception(): for name in exc_save_vars: - code.put_xgiveref(name) + code.put_xgiveref(name, py_object_type) code.putln("__Pyx_ExceptionReset(%s);" % ', '.join(exc_save_vars)) else: # try block cannot raise exceptions, but we had to allocate the temps above, # so just keep the C compiler from complaining about them being unused - mark_vars_used = ["(void)%s;" % var for var in exc_save_vars] + mark_vars_used = ["(void)%s;" % var for var in exc_save_vars] save_exc.putln("%s /* mark used */" % ' '.join(mark_vars_used)) def restore_saved_exception(): @@ -7297,19 +8031,17 @@ class TryExceptStatNode(StatNode): if not self.has_default_clause: code.put_goto(except_error_label) - for exit_label, old_label in [(except_error_label, old_error_label), - (try_break_label, old_break_label), - (try_continue_label, old_continue_label), - (try_return_label, old_return_label), - (except_return_label, old_return_label)]: - if code.label_used(exit_label): - if not normal_case_terminates and not code.label_used(try_end_label): - code.put_goto(try_end_label) - code.put_label(exit_label) - code.mark_pos(self.pos, trace=False) - if can_raise: - restore_saved_exception() - code.put_goto(old_label) + label_intercepts = code.label_interceptor( + [except_error_label, try_break_label, try_continue_label, try_return_label, except_return_label], + [old_error_label, old_break_label, old_continue_label, old_return_label, old_return_label], + skip_to_label=try_end_label if not normal_case_terminates and not code.label_used(try_end_label) else None, + pos=self.pos, + trace=False, + ) + + for _ in label_intercepts: + if can_raise: + restore_saved_exception() if code.label_used(except_end_label): if not normal_case_terminates and not code.label_used(try_end_label): @@ -7402,17 +8134,40 @@ class ExceptClauseNode(Node): for _ in range(3)] code.globalstate.use_utility_code(UtilityCode.load_cached("PyErrFetchRestore", "Exceptions.c")) code.putln("__Pyx_ErrFetch(&%s, &%s, &%s);" % tuple(exc_vars)) - code.globalstate.use_utility_code(UtilityCode.load_cached("FastTypeChecks", "ModuleSetupCode.c")) - exc_test_func = "__Pyx_PyErr_GivenExceptionMatches(%s, %%s)" % exc_vars[0] + exc_type = exc_vars[0] else: - exc_vars = () - code.globalstate.use_utility_code(UtilityCode.load_cached("PyErrExceptionMatches", "Exceptions.c")) - exc_test_func = "__Pyx_PyErr_ExceptionMatches(%s)" + exc_vars = exc_type = None - exc_tests = [] for pattern in self.pattern: pattern.generate_evaluation_code(code) - exc_tests.append(exc_test_func % pattern.py_result()) + patterns = [pattern.py_result() for pattern in self.pattern] + + exc_tests = [] + if exc_type: + code.globalstate.use_utility_code( + UtilityCode.load_cached("FastTypeChecks", "ModuleSetupCode.c")) + if len(patterns) == 2: + exc_tests.append("__Pyx_PyErr_GivenExceptionMatches2(%s, %s, %s)" % ( + exc_type, patterns[0], patterns[1], + )) + else: + exc_tests.extend( + "__Pyx_PyErr_GivenExceptionMatches(%s, %s)" % (exc_type, pattern) + for pattern in patterns + ) + elif len(patterns) == 2: + code.globalstate.use_utility_code( + UtilityCode.load_cached("FastTypeChecks", "ModuleSetupCode.c")) + exc_tests.append("__Pyx_PyErr_ExceptionMatches2(%s, %s)" % ( + patterns[0], patterns[1], + )) + else: + code.globalstate.use_utility_code( + UtilityCode.load_cached("PyErrExceptionMatches", "Exceptions.c")) + exc_tests.extend( + "__Pyx_PyErr_ExceptionMatches(%s)" % pattern + for pattern in patterns + ) match_flag = code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False) code.putln("%s = %s;" % (match_flag, ' || '.join(exc_tests))) @@ -7420,7 +8175,7 @@ class ExceptClauseNode(Node): pattern.generate_disposal_code(code) pattern.free_temps(code) - if has_non_literals: + if exc_vars: code.putln("__Pyx_ErrRestore(%s, %s, %s);" % tuple(exc_vars)) code.putln(' '.join(["%s = 0;" % var for var in exc_vars])) for temp in exc_vars: @@ -7455,7 +8210,7 @@ class ExceptClauseNode(Node): code.putln("if (__Pyx_GetException(%s) < 0) %s" % ( exc_args, code.error_goto(self.pos))) for var in exc_vars: - code.put_gotref(var) + code.put_gotref(var, py_object_type) if self.target: self.exc_value.set_var(exc_vars[1]) self.exc_value.generate_evaluation_code(code) @@ -7464,9 +8219,7 @@ class ExceptClauseNode(Node): for tempvar, node in zip(exc_vars, self.excinfo_target.args): node.set_var(tempvar) - old_break_label, old_continue_label = code.break_label, code.continue_label - code.break_label = code.new_label('except_break') - code.continue_label = code.new_label('except_continue') + old_loop_labels = code.new_loop_labels("except_") old_exc_vars = code.funcstate.exc_vars code.funcstate.exc_vars = exc_vars @@ -7480,15 +8233,11 @@ class ExceptClauseNode(Node): code.put_xdecref_clear(var, py_object_type) code.put_goto(end_label) - for new_label, old_label in [(code.break_label, old_break_label), - (code.continue_label, old_continue_label)]: - if code.label_used(new_label): - code.put_label(new_label) - for var in exc_vars: - code.put_decref_clear(var, py_object_type) - code.put_goto(old_label) - code.break_label = old_break_label - code.continue_label = old_continue_label + for _ in code.label_interceptor(code.get_loop_labels(), old_loop_labels): + for var in exc_vars: + code.put_decref_clear(var, py_object_type) + + code.set_loop_labels(old_loop_labels) for temp in exc_vars: code.funcstate.release_temp(temp) @@ -7644,12 +8393,8 @@ class TryFinallyStatNode(StatNode): code.funcstate.release_temp(exc_filename_cname) code.put_goto(old_error_label) - for new_label, old_label in zip(code.get_all_labels(), finally_old_labels): - if not code.label_used(new_label): - continue - code.put_label(new_label) + for _ in code.label_interceptor(code.get_all_labels(), finally_old_labels): self.put_error_cleaner(code, exc_vars) - code.put_goto(old_label) for cname in exc_vars: code.funcstate.release_temp(cname) @@ -7659,6 +8404,7 @@ class TryFinallyStatNode(StatNode): return_label = code.return_label exc_vars = () + # TODO: use code.label_interceptor()? for i, (new_label, old_label) in enumerate(zip(new_labels, old_labels)): if not code.label_used(new_label): continue @@ -7737,7 +8483,7 @@ class TryFinallyStatNode(StatNode): " unlikely(__Pyx_GetException(&%s, &%s, &%s) < 0)) " "__Pyx_ErrFetch(&%s, &%s, &%s);" % (exc_vars[:3] * 2)) for var in exc_vars: - code.put_xgotref(var) + code.put_xgotref(var, py_object_type) if exc_lineno_cnames: code.putln("%s = %s; %s = %s; %s = %s;" % ( exc_lineno_cnames[0], Naming.lineno_cname, @@ -7758,11 +8504,11 @@ class TryFinallyStatNode(StatNode): # unused utility functions and/or temps code.putln("if (PY_MAJOR_VERSION >= 3) {") for var in exc_vars[3:]: - code.put_xgiveref(var) + code.put_xgiveref(var, py_object_type) code.putln("__Pyx_ExceptionReset(%s, %s, %s);" % exc_vars[3:]) code.putln("}") for var in exc_vars[:3]: - code.put_xgiveref(var) + code.put_xgiveref(var, py_object_type) code.putln("__Pyx_ErrRestore(%s, %s, %s);" % exc_vars[:3]) if self.is_try_finally_in_nogil: @@ -7784,7 +8530,7 @@ class TryFinallyStatNode(StatNode): # unused utility functions and/or temps code.putln("if (PY_MAJOR_VERSION >= 3) {") for var in exc_vars[3:]: - code.put_xgiveref(var) + code.put_xgiveref(var, py_object_type) code.putln("__Pyx_ExceptionReset(%s, %s, %s);" % exc_vars[3:]) code.putln("}") for var in exc_vars[:3]: @@ -7811,11 +8557,16 @@ class GILStatNode(NogilTryFinallyStatNode): # 'with gil' or 'with nogil' statement # # state string 'gil' or 'nogil' + # scope_gil_state_known bool For nogil functions this can be False, since they can also be run with gil + # set to False by GilCheck transform + child_attrs = ["condition"] + NogilTryFinallyStatNode.child_attrs state_temp = None + scope_gil_state_known = True - def __init__(self, pos, state, body): + def __init__(self, pos, state, body, condition=None): self.state = state + self.condition = condition self.create_state_temp_if_needed(pos, state, body) TryFinallyStatNode.__init__( self, pos, @@ -7842,11 +8593,18 @@ class GILStatNode(NogilTryFinallyStatNode): if self.state == 'gil': env.has_with_gil_block = True + if self.condition is not None: + self.condition.analyse_declarations(env) + return super(GILStatNode, self).analyse_declarations(env) def analyse_expressions(self, env): env.use_utility_code( UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c")) + + if self.condition is not None: + self.condition = self.condition.analyse_expressions(env) + was_nogil = env.nogil env.nogil = self.state == 'nogil' node = TryFinallyStatNode.analyse_expressions(self, env) @@ -7867,7 +8625,7 @@ class GILStatNode(NogilTryFinallyStatNode): code.put_ensure_gil(variable=variable) code.funcstate.gil_owned = True else: - code.put_release_gil(variable=variable) + code.put_release_gil(variable=variable, unknown_gil_state=not self.scope_gil_state_known) code.funcstate.gil_owned = False TryFinallyStatNode.generate_execution_code(self, code) @@ -7884,10 +8642,13 @@ class GILExitNode(StatNode): Used as the 'finally' block in a GILStatNode state string 'gil' or 'nogil' + # scope_gil_state_known bool For nogil functions this can be False, since they can also be run with gil + # set to False by GilCheck transform """ child_attrs = [] state_temp = None + scope_gil_state_known = True def analyse_expressions(self, env): return self @@ -7901,7 +8662,7 @@ class GILExitNode(StatNode): if self.state == 'gil': code.put_release_ensured_gil(variable) else: - code.put_acquire_gil(variable) + code.put_acquire_gil(variable, unknown_gil_state=not self.scope_gil_state_known) class EnsureGILNode(GILExitNode): @@ -7933,6 +8694,31 @@ utility_code_for_imports = { 'inspect': ("__Pyx_patch_inspect", "PatchInspect", "Coroutine.c"), } +def cimport_numpy_check(node, code): + # shared code between CImportStatNode and FromCImportStatNode + # check to ensure that import_array is called + for mod in code.globalstate.module_node.scope.cimported_modules: + if mod.name != node.module_name: + continue + # there are sometimes several cimported modules with the same name + # so complete the loop if necessary + import_array = mod.lookup_here("import_array") + _import_array = mod.lookup_here("_import_array") + # at least one entry used + used = (import_array and import_array.used) or (_import_array and _import_array.used) + if ((import_array or _import_array) # at least one entry found + and not used): + # sanity check that this is actually numpy and not a user pxd called "numpy" + if _import_array and _import_array.type.is_cfunction: + # warning is mainly for the sake of testing + warning(node.pos, "'numpy.import_array()' has been added automatically " + "since 'numpy' was cimported but 'numpy.import_array' was not called.", 0) + code.globalstate.use_utility_code( + UtilityCode.load_cached("NumpyImportArray", "NumpyImportArray.c") + ) + return # no need to continue once the utility code is added + + class CImportStatNode(StatNode): # cimport statement @@ -7966,7 +8752,8 @@ class CImportStatNode(StatNode): env.declare_module(top_name, top_module_scope, self.pos) else: name = self.as_name or self.module_name - env.declare_module(name, module_scope, self.pos) + entry = env.declare_module(name, module_scope, self.pos) + entry.known_standard_library_import = self.module_name if self.module_name in utility_code_for_cimports: env.use_utility_code(utility_code_for_cimports[self.module_name]()) @@ -7974,7 +8761,8 @@ class CImportStatNode(StatNode): return self def generate_execution_code(self, code): - pass + if self.module_name == "numpy": + cimport_numpy_check(self, code) class FromCImportStatNode(StatNode): @@ -7982,7 +8770,7 @@ class FromCImportStatNode(StatNode): # # module_name string Qualified name of module # relative_level int or None Relative import: number of dots before module_name - # imported_names [(pos, name, as_name, kind)] Names to be imported + # imported_names [(pos, name, as_name)] Names to be imported child_attrs = [] module_name = None @@ -7993,44 +8781,43 @@ class FromCImportStatNode(StatNode): if not env.is_module_scope: error(self.pos, "cimport only allowed at module level") return - if self.relative_level and self.relative_level > env.qualified_name.count('.'): - error(self.pos, "relative cimport beyond main package is not allowed") - return + qualified_name_components = env.qualified_name.count('.') + 1 + if self.relative_level: + if self.relative_level > qualified_name_components: + # 1. case: importing beyond package: from .. import pkg + error(self.pos, "relative cimport beyond main package is not allowed") + return + elif self.relative_level == qualified_name_components and not env.is_package: + # 2. case: importing from same level but current dir is not package: from . import module + error(self.pos, "relative cimport from non-package directory is not allowed") + return module_scope = env.find_module(self.module_name, self.pos, relative_level=self.relative_level) module_name = module_scope.qualified_name env.add_imported_module(module_scope) - for pos, name, as_name, kind in self.imported_names: + for pos, name, as_name in self.imported_names: if name == "*": for local_name, entry in list(module_scope.entries.items()): env.add_imported_entry(local_name, entry, pos) else: entry = module_scope.lookup(name) if entry: - if kind and not self.declaration_matches(entry, kind): - entry.redeclared(pos) entry.used = 1 else: - if kind == 'struct' or kind == 'union': - entry = module_scope.declare_struct_or_union( - name, kind=kind, scope=None, typedef_flag=0, pos=pos) - elif kind == 'class': - entry = module_scope.declare_c_class(name, pos=pos, module_name=module_name) + submodule_scope = env.context.find_module( + name, relative_to=module_scope, pos=self.pos, absolute_fallback=False) + if submodule_scope.parent_module is module_scope: + env.declare_module(as_name or name, submodule_scope, self.pos) else: - submodule_scope = env.context.find_module( - name, relative_to=module_scope, pos=self.pos, absolute_fallback=False) - if submodule_scope.parent_module is module_scope: - env.declare_module(as_name or name, submodule_scope, self.pos) - else: - error(pos, "Name '%s' not declared in module '%s'" % (name, module_name)) + error(pos, "Name '%s' not declared in module '%s'" % (name, module_name)) if entry: local_name = as_name or name env.add_imported_entry(local_name, entry, pos) - if module_name.startswith('cpython') or module_name.startswith('cython'): # enough for now + if module_name.startswith('cpython') or module_name.startswith('cython'): # enough for now if module_name in utility_code_for_cimports: env.use_utility_code(utility_code_for_cimports[module_name]()) - for _, name, _, _ in self.imported_names: + for _, name, _ in self.imported_names: fqname = '%s.%s' % (module_name, name) if fqname in utility_code_for_cimports: env.use_utility_code(utility_code_for_cimports[fqname]()) @@ -8053,7 +8840,8 @@ class FromCImportStatNode(StatNode): return self def generate_execution_code(self, code): - pass + if self.module_name == "numpy": + cimport_numpy_check(self, code) class FromImportStatNode(StatNode): @@ -8078,6 +8866,14 @@ class FromImportStatNode(StatNode): self.import_star = 1 else: target.analyse_target_declaration(env) + if target.entry: + if target.get_known_standard_library_import() is None: + target.entry.known_standard_library_import = EncodedString( + "%s.%s" % (self.module.module_name.value, name)) + else: + # it isn't unambiguous + target.entry.known_standard_library_import = "" + def analyse_expressions(self, env): from . import ExprNodes @@ -8135,7 +8931,7 @@ class FromImportStatNode(StatNode): self.module.py_result(), code.intern_identifier(name), code.error_goto_if_null(item_temp, self.pos))) - code.put_gotref(item_temp) + code.put_gotref(item_temp, py_object_type) if coerced_item is None: target.generate_assignment_code(self.item, code) else: @@ -8251,7 +9047,7 @@ class ParallelStatNode(StatNode, ParallelNode): seen.add(dictitem.key.value) if dictitem.key.value == 'num_threads': if not dictitem.value.is_none: - self.num_threads = dictitem.value + self.num_threads = dictitem.value elif self.is_prange and dictitem.key.value == 'chunksize': if not dictitem.value.is_none: self.chunksize = dictitem.value @@ -8512,11 +9308,7 @@ class ParallelStatNode(StatNode, ParallelNode): if self.is_parallel and not self.is_nested_prange: code.putln("/* Clean up any temporaries */") for temp, type in sorted(self.temps): - if type.is_memoryviewslice: - code.put_xdecref_memoryviewslice(temp, have_gil=False) - elif type.is_pyobject: - code.put_xdecref(temp, type) - code.putln("%s = NULL;" % temp) + code.put_xdecref_clear(temp, type, have_gil=False) def setup_parallel_control_flow_block(self, code): """ @@ -8549,7 +9341,7 @@ class ParallelStatNode(StatNode, ParallelNode): self.old_return_label = code.return_label code.return_label = code.new_label(name="return") - code.begin_block() # parallel control flow block + code.begin_block() # parallel control flow block self.begin_of_parallel_control_block_point = code.insertion_point() self.begin_of_parallel_control_block_point_after_decls = code.insertion_point() @@ -8679,7 +9471,7 @@ class ParallelStatNode(StatNode, ParallelNode): code.putln_openmp("#pragma omp critical(%s)" % section_name) ParallelStatNode.critical_section_counter += 1 - code.begin_block() # begin critical section + code.begin_block() # begin critical section c = self.begin_of_parallel_control_block_point @@ -8688,7 +9480,10 @@ class ParallelStatNode(StatNode, ParallelNode): if not lastprivate or entry.type.is_pyobject: continue - type_decl = entry.type.empty_declaration_code() + if entry.type.is_cpp_class and not entry.type.is_fake_reference and code.globalstate.directives['cpp_locals']: + type_decl = entry.type.cpp_optional_declaration_code("") + else: + type_decl = entry.type.empty_declaration_code() temp_cname = "__pyx_parallel_temp%d" % temp_count private_cname = entry.cname @@ -8702,12 +9497,19 @@ class ParallelStatNode(StatNode, ParallelNode): # Declare the parallel private in the outer block c.putln("%s %s%s;" % (type_decl, temp_cname, init)) + self.parallel_private_temps.append((temp_cname, private_cname, entry.type)) + + if entry.type.is_cpp_class: + # moving is fine because we're quitting the loop and so won't be directly accessing the variable again + code.globalstate.use_utility_code( + UtilityCode.load_cached("MoveIfSupported", "CppSupport.cpp")) + private_cname = "__PYX_STD_MOVE_IF_SUPPORTED(%s)" % private_cname # Initialize before escaping code.putln("%s = %s;" % (temp_cname, private_cname)) - self.parallel_private_temps.append((temp_cname, private_cname)) - code.end_block() # end critical section + + code.end_block() # end critical section def fetch_parallel_exception(self, code): """ @@ -8747,7 +9549,7 @@ class ParallelStatNode(StatNode, ParallelNode): pos_info = chain(*zip(self.parallel_pos_info, self.pos_info)) code.funcstate.uses_error_indicator = True code.putln("%s = %s; %s = %s; %s = %s;" % tuple(pos_info)) - code.put_gotref(Naming.parallel_exc_type) + code.put_gotref(Naming.parallel_exc_type, py_object_type) code.putln( "}") @@ -8760,7 +9562,7 @@ class ParallelStatNode(StatNode, ParallelNode): code.begin_block() code.put_ensure_gil(declare_gilstate=True) - code.put_giveref(Naming.parallel_exc_type) + code.put_giveref(Naming.parallel_exc_type, py_object_type) code.putln("__Pyx_ErrRestoreWithState(%s, %s, %s);" % self.parallel_exc) pos_info = chain(*zip(self.pos_info, self.parallel_pos_info)) code.putln("%s = %s; %s = %s; %s = %s;" % tuple(pos_info)) @@ -8826,7 +9628,10 @@ class ParallelStatNode(StatNode, ParallelNode): code.putln( "if (%s) {" % Naming.parallel_why) - for temp_cname, private_cname in self.parallel_private_temps: + for temp_cname, private_cname, temp_type in self.parallel_private_temps: + if temp_type.is_cpp_class: + # utility code was loaded earlier + temp_cname = "__PYX_STD_MOVE_IF_SUPPORTED(%s)" % temp_cname code.putln("%s = %s;" % (private_cname, temp_cname)) code.putln("switch (%s) {" % Naming.parallel_why) @@ -8848,11 +9653,11 @@ class ParallelStatNode(StatNode, ParallelNode): self.restore_parallel_exception(code) code.put_goto(code.error_label) - code.putln("}") # end switch + code.putln("}") # end switch code.putln( - "}") # end if + "}") # end if - code.end_block() # end parallel control flow block + code.end_block() # end parallel control flow block self.redef_builtin_expect_apple_gcc_bug(code) # FIXME: improve with version number for OS X Lion @@ -8971,9 +9776,6 @@ class ParallelRangeNode(ParallelStatNode): else: self.start, self.stop, self.step = self.args - if hasattr(self.schedule, 'decode'): - self.schedule = self.schedule.decode('ascii') - if self.schedule not in (None, 'static', 'dynamic', 'guided', 'runtime'): error(self.pos, "Invalid schedule argument to prange: %s" % (self.schedule,)) @@ -9028,7 +9830,8 @@ class ParallelRangeNode(ParallelStatNode): # ensure lastprivate behaviour and propagation. If the target index is # not a NameNode, it won't have an entry, and an error was issued by # ParallelRangeTransform - if hasattr(self.target, 'entry'): + target_entry = getattr(self.target, 'entry', None) + if target_entry: self.assignments[self.target.entry] = self.target.pos, None node = super(ParallelRangeNode, self).analyse_expressions(env) @@ -9141,9 +9944,12 @@ class ParallelRangeNode(ParallelStatNode): # TODO: check if the step is 0 and if so, raise an exception in a # 'with gil' block. For now, just abort - code.putln("if ((%(step)s == 0)) abort();" % fmt_dict) + if self.step is not None and self.step.has_constant_result() and self.step.constant_result == 0: + error(node.pos, "Iteration with step 0 is invalid.") + elif not fmt_dict['step'].isdigit() or int(fmt_dict['step']) == 0: + code.putln("if (((%(step)s) == 0)) abort();" % fmt_dict) - self.setup_parallel_control_flow_block(code) # parallel control flow block + self.setup_parallel_control_flow_block(code) # parallel control flow block # Note: nsteps is private in an outer scope if present code.putln("%(nsteps)s = (%(stop)s - %(start)s + %(step)s - %(step)s/abs(%(step)s)) / %(step)s;" % fmt_dict) @@ -9155,9 +9961,9 @@ class ParallelRangeNode(ParallelStatNode): # erroneously believes that nsteps may be <= 0, leaving the private # target index uninitialized code.putln("if (%(nsteps)s > 0)" % fmt_dict) - code.begin_block() # if block + code.begin_block() # if block self.generate_loop(code, fmt_dict) - code.end_block() # end if block + code.end_block() # end if block self.restore_labels(code) @@ -9165,13 +9971,13 @@ class ParallelRangeNode(ParallelStatNode): if self.breaking_label_used: code.put("if (%s < 2)" % Naming.parallel_why) - code.begin_block() # else block + code.begin_block() # else block code.putln("/* else */") self.else_clause.generate_execution_code(code) - code.end_block() # end else block + code.end_block() # end else block # ------ cleanup ------ - self.end_parallel_control_flow_block(code) # end parallel control flow block + self.end_parallel_control_flow_block(code) # end parallel control flow block # And finally, release our privates and write back any closure # variables @@ -9202,7 +10008,7 @@ class ParallelRangeNode(ParallelStatNode): code.putln("") code.putln("#endif /* _OPENMP */") - code.begin_block() # pragma omp parallel begin block + code.begin_block() # pragma omp parallel begin block # Initialize the GIL if needed for this thread self.begin_parallel_block(code) diff --git a/Cython/Compiler/Optimize.py b/Cython/Compiler/Optimize.py index 7e9435ba0..fb6dc5dae 100644 --- a/Cython/Compiler/Optimize.py +++ b/Cython/Compiler/Optimize.py @@ -41,7 +41,7 @@ except ImportError: try: from __builtin__ import basestring except ImportError: - basestring = str # Python 3 + basestring = str # Python 3 def load_c_utility(name): @@ -192,19 +192,9 @@ class IterationTransform(Visitor.EnvTransform): def _optimise_for_loop(self, node, iterable, reversed=False): annotation_type = None if (iterable.is_name or iterable.is_attribute) and iterable.entry and iterable.entry.annotation: - annotation = iterable.entry.annotation + annotation = iterable.entry.annotation.expr if annotation.is_subscript: annotation = annotation.base # container base type - # FIXME: generalise annotation evaluation => maybe provide a "qualified name" also for imported names? - if annotation.is_name: - if annotation.entry and annotation.entry.qualified_name == 'typing.Dict': - annotation_type = Builtin.dict_type - elif annotation.name == 'Dict': - annotation_type = Builtin.dict_type - if annotation.entry and annotation.entry.qualified_name in ('typing.Set', 'typing.FrozenSet'): - annotation_type = Builtin.set_type - elif annotation.name in ('Set', 'FrozenSet'): - annotation_type = Builtin.set_type if Builtin.dict_type in (iterable.type, annotation_type): # like iterating over dict.keys() @@ -228,6 +218,12 @@ class IterationTransform(Visitor.EnvTransform): return self._transform_bytes_iteration(node, iterable, reversed=reversed) if iterable.type is Builtin.unicode_type: return self._transform_unicode_iteration(node, iterable, reversed=reversed) + # in principle _transform_indexable_iteration would work on most of the above, and + # also tuple and list. However, it probably isn't quite as optimized + if iterable.type is Builtin.bytearray_type: + return self._transform_indexable_iteration(node, iterable, is_mutable=True, reversed=reversed) + if isinstance(iterable, ExprNodes.CoerceToPyTypeNode) and iterable.arg.type.is_memoryviewslice: + return self._transform_indexable_iteration(node, iterable.arg, is_mutable=False, reversed=reversed) # the rest is based on function calls if not isinstance(iterable, ExprNodes.SimpleCallNode): @@ -323,6 +319,92 @@ class IterationTransform(Visitor.EnvTransform): return self._optimise_for_loop(node, arg, reversed=True) + def _transform_indexable_iteration(self, node, slice_node, is_mutable, reversed=False): + """In principle can handle any iterable that Cython has a len() for and knows how to index""" + unpack_temp_node = UtilNodes.LetRefNode( + slice_node.as_none_safe_node("'NoneType' is not iterable"), + may_hold_none=False, is_temp=True + ) + + start_node = ExprNodes.IntNode( + node.pos, value='0', constant_result=0, type=PyrexTypes.c_py_ssize_t_type) + def make_length_call(): + # helper function since we need to create this node for a couple of places + builtin_len = ExprNodes.NameNode(node.pos, name="len", + entry=Builtin.builtin_scope.lookup("len")) + return ExprNodes.SimpleCallNode(node.pos, + function=builtin_len, + args=[unpack_temp_node] + ) + length_temp = UtilNodes.LetRefNode(make_length_call(), type=PyrexTypes.c_py_ssize_t_type, is_temp=True) + end_node = length_temp + + if reversed: + relation1, relation2 = '>', '>=' + start_node, end_node = end_node, start_node + else: + relation1, relation2 = '<=', '<' + + counter_ref = UtilNodes.LetRefNode(pos=node.pos, type=PyrexTypes.c_py_ssize_t_type) + + target_value = ExprNodes.IndexNode(slice_node.pos, base=unpack_temp_node, + index=counter_ref) + + target_assign = Nodes.SingleAssignmentNode( + pos = node.target.pos, + lhs = node.target, + rhs = target_value) + + # analyse with boundscheck and wraparound + # off (because we're confident we know the size) + env = self.current_env() + new_directives = Options.copy_inherited_directives(env.directives, boundscheck=False, wraparound=False) + target_assign = Nodes.CompilerDirectivesNode( + target_assign.pos, + directives=new_directives, + body=target_assign, + ) + + body = Nodes.StatListNode( + node.pos, + stats = [target_assign]) # exclude node.body for now to not reanalyse it + if is_mutable: + # We need to be slightly careful here that we are actually modifying the loop + # bounds and not a temp copy of it. Setting is_temp=True on length_temp seems + # to ensure this. + # If this starts to fail then we could insert an "if out_of_bounds: break" instead + loop_length_reassign = Nodes.SingleAssignmentNode(node.pos, + lhs = length_temp, + rhs = make_length_call()) + body.stats.append(loop_length_reassign) + + loop_node = Nodes.ForFromStatNode( + node.pos, + bound1=start_node, relation1=relation1, + target=counter_ref, + relation2=relation2, bound2=end_node, + step=None, body=body, + else_clause=node.else_clause, + from_range=True) + + ret = UtilNodes.LetNode( + unpack_temp_node, + UtilNodes.LetNode( + length_temp, + # TempResultFromStatNode provides the framework where the "counter_ref" + # temp is set up and can be assigned to. However, we don't need the + # result it returns so wrap it in an ExprStatNode. + Nodes.ExprStatNode(node.pos, + expr=UtilNodes.TempResultFromStatNode( + counter_ref, + loop_node + ) + ) + ) + ).analyse_expressions(env) + body.stats.insert(1, node.body) + return ret + PyBytes_AS_STRING_func_type = PyrexTypes.CFuncType( PyrexTypes.c_char_ptr_type, [ PyrexTypes.CFuncTypeArg("s", Builtin.bytes_type, None) @@ -1144,7 +1226,7 @@ class SwitchTransform(Visitor.EnvTransform): # integers on iteration, whereas Py2 returns 1-char byte # strings characters = string_literal.value - characters = list(set([ characters[i:i+1] for i in range(len(characters)) ])) + characters = list({ characters[i:i+1] for i in range(len(characters)) }) characters.sort() return [ ExprNodes.CharNode(string_literal.pos, value=charval, constant_result=charval) @@ -1156,7 +1238,8 @@ class SwitchTransform(Visitor.EnvTransform): return self.NO_MATCH elif common_var is not None and not is_common_value(var, common_var): return self.NO_MATCH - elif not (var.type.is_int or var.type.is_enum) or sum([not (cond.type.is_int or cond.type.is_enum) for cond in conditions]): + elif not (var.type.is_int or var.type.is_enum) or any( + [not (cond.type.is_int or cond.type.is_enum) for cond in conditions]): return self.NO_MATCH return not_in, var, conditions @@ -1573,7 +1656,7 @@ class EarlyReplaceBuiltinCalls(Visitor.EnvTransform): utility_code = utility_code) def _error_wrong_arg_count(self, function_name, node, args, expected=None): - if not expected: # None or 0 + if not expected: # None or 0 arg_str = '' elif isinstance(expected, basestring) or expected > 1: arg_str = '...' @@ -1727,7 +1810,7 @@ class EarlyReplaceBuiltinCalls(Visitor.EnvTransform): arg = pos_args[0] if isinstance(arg, ExprNodes.ComprehensionNode) and arg.type is Builtin.list_type: - list_node = pos_args[0] + list_node = arg loop_node = list_node.loop elif isinstance(arg, ExprNodes.GeneratorExpressionNode): @@ -1757,7 +1840,11 @@ class EarlyReplaceBuiltinCalls(Visitor.EnvTransform): # Interestingly, PySequence_List works on a lot of non-sequence # things as well. list_node = loop_node = ExprNodes.PythonCapiCallNode( - node.pos, "PySequence_List", self.PySequence_List_func_type, + node.pos, + "__Pyx_PySequence_ListKeepNew" + if arg.is_temp and arg.type in (PyrexTypes.py_object_type, Builtin.list_type) + else "PySequence_List", + self.PySequence_List_func_type, args=pos_args, is_temp=True) result_node = UtilNodes.ResultRefNode( @@ -1803,7 +1890,7 @@ class EarlyReplaceBuiltinCalls(Visitor.EnvTransform): if not yield_expression.is_literal or not yield_expression.type.is_int: return node except AttributeError: - return node # in case we don't have a type yet + return node # in case we don't have a type yet # special case: old Py2 backwards compatible "sum([int_const for ...])" # can safely be unpacked into a genexpr @@ -2018,7 +2105,8 @@ class InlineDefNodeCalls(Visitor.NodeRefCleanupMixin, Visitor.EnvTransform): return node inlined = ExprNodes.InlinedDefNodeCallNode( node.pos, function_name=function_name, - function=function, args=node.args) + function=function, args=node.args, + generator_arg_tag=node.generator_arg_tag) if inlined.can_be_inlined(): return self.replace(node, inlined) return node @@ -2097,12 +2185,13 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, func_arg = arg.args[0] if func_arg.type is Builtin.float_type: return func_arg.as_none_safe_node("float() argument must be a string or a number, not 'NoneType'") - elif func_arg.type.is_pyobject: + elif func_arg.type.is_pyobject and arg.function.cname == "__Pyx_PyObject_AsDouble": return ExprNodes.PythonCapiCallNode( node.pos, '__Pyx_PyNumber_Float', self.PyNumber_Float_func_type, args=[func_arg], py_name='float', is_temp=node.is_temp, + utility_code = UtilityCode.load_cached("pynumber_float", "TypeConversion.c"), result_is_used=node.result_is_used, ).coerce_to(node.type, self.current_env()) return node @@ -2210,6 +2299,9 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, if func_arg.type.is_int or node.type.is_int: if func_arg.type == node.type: return func_arg + elif func_arg.type in (PyrexTypes.c_py_ucs4_type, PyrexTypes.c_py_unicode_type): + # need to parse (<Py_UCS4>'1') as digit 1 + return self._pyucs4_to_number(node, function.name, func_arg) elif node.type.assignable_from(func_arg.type) or func_arg.type.is_float: return ExprNodes.TypecastNode(node.pos, operand=func_arg, type=node.type) elif func_arg.type.is_float and node.type.is_numeric: @@ -2230,13 +2322,40 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, if func_arg.type.is_float or node.type.is_float: if func_arg.type == node.type: return func_arg + elif func_arg.type in (PyrexTypes.c_py_ucs4_type, PyrexTypes.c_py_unicode_type): + # need to parse (<Py_UCS4>'1') as digit 1 + return self._pyucs4_to_number(node, function.name, func_arg) elif node.type.assignable_from(func_arg.type) or func_arg.type.is_float: return ExprNodes.TypecastNode( node.pos, operand=func_arg, type=node.type) return node + pyucs4_int_func_type = PyrexTypes.CFuncType( + PyrexTypes.c_int_type, [ + PyrexTypes.CFuncTypeArg("arg", PyrexTypes.c_py_ucs4_type, None) + ], + exception_value="-1") + + pyucs4_double_func_type = PyrexTypes.CFuncType( + PyrexTypes.c_double_type, [ + PyrexTypes.CFuncTypeArg("arg", PyrexTypes.c_py_ucs4_type, None) + ], + exception_value="-1.0") + + def _pyucs4_to_number(self, node, py_type_name, func_arg): + assert py_type_name in ("int", "float") + return ExprNodes.PythonCapiCallNode( + node.pos, "__Pyx_int_from_UCS4" if py_type_name == "int" else "__Pyx_double_from_UCS4", + func_type=self.pyucs4_int_func_type if py_type_name == "int" else self.pyucs4_double_func_type, + args=[func_arg], + py_name=py_type_name, + is_temp=node.is_temp, + result_is_used=node.result_is_used, + utility_code=UtilityCode.load_cached("int_pyucs4" if py_type_name == "int" else "float_pyucs4", "Builtins.c"), + ).coerce_to(node.type, self.current_env()) + def _error_wrong_arg_count(self, function_name, node, args, expected=None): - if not expected: # None or 0 + if not expected: # None or 0 arg_str = '' elif isinstance(expected, basestring) or expected > 1: arg_str = '...' @@ -2316,6 +2435,38 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, return ExprNodes.CachedBuiltinMethodCallNode( node, function.obj, attr_name, arg_list) + PyObject_String_func_type = PyrexTypes.CFuncType( + PyrexTypes.py_object_type, [ # Change this to Builtin.str_type when removing Py2 support. + PyrexTypes.CFuncTypeArg("obj", PyrexTypes.py_object_type, None) + ]) + + def _handle_simple_function_str(self, node, function, pos_args): + """Optimize single argument calls to str(). + """ + if len(pos_args) != 1: + if len(pos_args) == 0: + return ExprNodes.StringNode(node.pos, value=EncodedString(), constant_result='') + return node + arg = pos_args[0] + + if arg.type is Builtin.str_type: + if not arg.may_be_none(): + return arg + + cname = "__Pyx_PyStr_Str" + utility_code = UtilityCode.load_cached('PyStr_Str', 'StringTools.c') + else: + cname = '__Pyx_PyObject_Str' + utility_code = UtilityCode.load_cached('PyObject_Str', 'StringTools.c') + + return ExprNodes.PythonCapiCallNode( + node.pos, cname, self.PyObject_String_func_type, + args=pos_args, + is_temp=node.is_temp, + utility_code=utility_code, + py_name="str" + ) + PyObject_Unicode_func_type = PyrexTypes.CFuncType( Builtin.unicode_type, [ PyrexTypes.CFuncTypeArg("obj", PyrexTypes.py_object_type, None) @@ -2387,8 +2538,14 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, return node arg = pos_args[0] return ExprNodes.PythonCapiCallNode( - node.pos, "PySequence_List", self.PySequence_List_func_type, - args=pos_args, is_temp=node.is_temp) + node.pos, + "__Pyx_PySequence_ListKeepNew" + if node.is_temp and arg.is_temp and arg.type in (PyrexTypes.py_object_type, Builtin.list_type) + else "PySequence_List", + self.PySequence_List_func_type, + args=pos_args, + is_temp=node.is_temp, + ) PyList_AsTuple_func_type = PyrexTypes.CFuncType( Builtin.tuple_type, [ @@ -2489,20 +2646,49 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, elif len(pos_args) != 1: self._error_wrong_arg_count('float', node, pos_args, '0 or 1') return node + func_arg = pos_args[0] if isinstance(func_arg, ExprNodes.CoerceToPyTypeNode): func_arg = func_arg.arg if func_arg.type is PyrexTypes.c_double_type: return func_arg + elif func_arg.type in (PyrexTypes.c_py_ucs4_type, PyrexTypes.c_py_unicode_type): + # need to parse (<Py_UCS4>'1') as digit 1 + return self._pyucs4_to_number(node, function.name, func_arg) elif node.type.assignable_from(func_arg.type) or func_arg.type.is_numeric: return ExprNodes.TypecastNode( node.pos, operand=func_arg, type=node.type) + + arg = None + if func_arg.type is Builtin.bytes_type: + cfunc_name = "__Pyx_PyBytes_AsDouble" + utility_code_name = 'pybytes_as_double' + elif func_arg.type is Builtin.bytearray_type: + cfunc_name = "__Pyx_PyByteArray_AsDouble" + utility_code_name = 'pybytes_as_double' + elif func_arg.type is Builtin.unicode_type: + cfunc_name = "__Pyx_PyUnicode_AsDouble" + utility_code_name = 'pyunicode_as_double' + elif func_arg.type is Builtin.str_type: + cfunc_name = "__Pyx_PyString_AsDouble" + utility_code_name = 'pystring_as_double' + elif func_arg.type is Builtin.long_type: + cfunc_name = "PyLong_AsDouble" + else: + arg = func_arg # no need for an additional None check + cfunc_name = "__Pyx_PyObject_AsDouble" + utility_code_name = 'pyobject_as_double' + + if arg is None: + arg = func_arg.as_none_safe_node( + "float() argument must be a string or a number, not 'NoneType'") + return ExprNodes.PythonCapiCallNode( - node.pos, "__Pyx_PyObject_AsDouble", + node.pos, cfunc_name, self.PyObject_AsDouble_func_type, - args = pos_args, + args = [arg], is_temp = node.is_temp, - utility_code = load_c_utility('pyobject_as_double'), + utility_code = load_c_utility(utility_code_name) if utility_code_name else None, py_name = "float") PyNumber_Int_func_type = PyrexTypes.CFuncType( @@ -2556,17 +2742,59 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, # coerce back to Python object as that's the result we are expecting return operand.coerce_to_pyobject(self.current_env()) + PyMemoryView_FromObject_func_type = PyrexTypes.CFuncType( + Builtin.memoryview_type, [ + PyrexTypes.CFuncTypeArg("value", PyrexTypes.py_object_type, None) + ]) + + PyMemoryView_FromBuffer_func_type = PyrexTypes.CFuncType( + Builtin.memoryview_type, [ + PyrexTypes.CFuncTypeArg("value", Builtin.py_buffer_type, None) + ]) + + def _handle_simple_function_memoryview(self, node, function, pos_args): + if len(pos_args) != 1: + self._error_wrong_arg_count('memoryview', node, pos_args, '1') + return node + else: + if pos_args[0].type.is_pyobject: + return ExprNodes.PythonCapiCallNode( + node.pos, "PyMemoryView_FromObject", + self.PyMemoryView_FromObject_func_type, + args = [pos_args[0]], + is_temp = node.is_temp, + py_name = "memoryview") + elif pos_args[0].type.is_ptr and pos_args[0].base_type is Builtin.py_buffer_type: + # TODO - this currently doesn't work because the buffer fails a + # "can coerce to python object" test earlier. But it'd be nice to support + return ExprNodes.PythonCapiCallNode( + node.pos, "PyMemoryView_FromBuffer", + self.PyMemoryView_FromBuffer_func_type, + args = [pos_args[0]], + is_temp = node.is_temp, + py_name = "memoryview") + return node + + ### builtin functions Pyx_strlen_func_type = PyrexTypes.CFuncType( PyrexTypes.c_size_t_type, [ PyrexTypes.CFuncTypeArg("bytes", PyrexTypes.c_const_char_ptr_type, None) - ]) + ], + nogil=True) + + Pyx_ssize_strlen_func_type = PyrexTypes.CFuncType( + PyrexTypes.c_py_ssize_t_type, [ + PyrexTypes.CFuncTypeArg("bytes", PyrexTypes.c_const_char_ptr_type, None) + ], + exception_value="-1") Pyx_Py_UNICODE_strlen_func_type = PyrexTypes.CFuncType( - PyrexTypes.c_size_t_type, [ + PyrexTypes.c_py_ssize_t_type, [ PyrexTypes.CFuncTypeArg("unicode", PyrexTypes.c_const_py_unicode_ptr_type, None) - ]) + ], + exception_value="-1") PyObject_Size_func_type = PyrexTypes.CFuncType( PyrexTypes.c_py_ssize_t_type, [ @@ -2585,7 +2813,7 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, Builtin.dict_type: "PyDict_Size", }.get - _ext_types_with_pysize = set(["cpython.array.array"]) + _ext_types_with_pysize = {"cpython.array.array"} def _handle_simple_function_len(self, node, function, pos_args): """Replace len(char*) by the equivalent call to strlen(), @@ -2600,18 +2828,19 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, arg = arg.arg if arg.type.is_string: new_node = ExprNodes.PythonCapiCallNode( - node.pos, "strlen", self.Pyx_strlen_func_type, + node.pos, "__Pyx_ssize_strlen", self.Pyx_ssize_strlen_func_type, args = [arg], is_temp = node.is_temp, - utility_code = UtilityCode.load_cached("IncludeStringH", "StringTools.c")) + utility_code = UtilityCode.load_cached("ssize_strlen", "StringTools.c")) elif arg.type.is_pyunicode_ptr: new_node = ExprNodes.PythonCapiCallNode( - node.pos, "__Pyx_Py_UNICODE_strlen", self.Pyx_Py_UNICODE_strlen_func_type, + node.pos, "__Pyx_Py_UNICODE_ssize_strlen", self.Pyx_Py_UNICODE_strlen_func_type, args = [arg], - is_temp = node.is_temp) + is_temp = node.is_temp, + utility_code = UtilityCode.load_cached("ssize_pyunicode_strlen", "StringTools.c")) elif arg.type.is_memoryviewslice: func_type = PyrexTypes.CFuncType( - PyrexTypes.c_size_t_type, [ + PyrexTypes.c_py_ssize_t_type, [ PyrexTypes.CFuncTypeArg("memoryviewslice", arg.type, None) ], nogil=True) new_node = ExprNodes.PythonCapiCallNode( @@ -2622,7 +2851,7 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, if cfunc_name is None: arg_type = arg.type if ((arg_type.is_extension_type or arg_type.is_builtin_type) - and arg_type.entry.qualified_name in self._ext_types_with_pysize): + and arg_type.entry.qualified_name in self._ext_types_with_pysize): cfunc_name = 'Py_SIZE' else: return node @@ -2698,6 +2927,9 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, builtin_type = None if builtin_type is not None: type_check_function = entry.type.type_check_function(exact=False) + if type_check_function == '__Pyx_Py3Int_Check' and builtin_type is Builtin.int_type: + # isinstance(x, int) should really test for 'int' in Py2, not 'int | long' + type_check_function = "PyInt_Check" if type_check_function in tests: continue tests.append(type_check_function) @@ -2781,11 +3013,9 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, return node type_arg = args[0] if not obj.is_name or not type_arg.is_name: - # play safe - return node + return node # not a simple case if obj.type != Builtin.type_type or type_arg.type != Builtin.type_type: - # not a known type, play safe - return node + return node # not a known type if not type_arg.type_entry or not obj.type_entry: if obj.name != type_arg.name: return node @@ -2847,6 +3077,13 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, is_temp=node.is_temp ) + def _handle_any_slot__class__(self, node, function, args, + is_unbound_method, kwargs=None): + # The purpose of this function is to handle calls to instance.__class__() so that + # it doesn't get handled by the __Pyx_CallUnboundCMethod0 mechanism. + # TODO: optimizations of the instance.__class__() call might be possible in future. + return node + ### methods of builtin types PyObject_Append_func_type = PyrexTypes.CFuncType( @@ -3182,6 +3419,9 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, def _handle_simple_method_object___sub__(self, node, function, args, is_unbound_method): return self._optimise_num_binop('Subtract', node, function, args, is_unbound_method) + def _handle_simple_method_object___mul__(self, node, function, args, is_unbound_method): + return self._optimise_num_binop('Multiply', node, function, args, is_unbound_method) + def _handle_simple_method_object___eq__(self, node, function, args, is_unbound_method): return self._optimise_num_binop('Eq', node, function, args, is_unbound_method) @@ -3261,6 +3501,9 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, """ Optimise math operators for (likely) float or small integer operations. """ + if getattr(node, "special_bool_cmp_function", None): + return node # already optimized + if len(args) != 2: return node @@ -3271,66 +3514,15 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, else: return node - # When adding IntNode/FloatNode to something else, assume other operand is also numeric. - # Prefer constants on RHS as they allows better size control for some operators. - num_nodes = (ExprNodes.IntNode, ExprNodes.FloatNode) - if isinstance(args[1], num_nodes): - if args[0].type is not PyrexTypes.py_object_type: - return node - numval = args[1] - arg_order = 'ObjC' - elif isinstance(args[0], num_nodes): - if args[1].type is not PyrexTypes.py_object_type: - return node - numval = args[0] - arg_order = 'CObj' - else: - return node - - if not numval.has_constant_result(): - return node - - is_float = isinstance(numval, ExprNodes.FloatNode) - num_type = PyrexTypes.c_double_type if is_float else PyrexTypes.c_long_type - if is_float: - if operator not in ('Add', 'Subtract', 'Remainder', 'TrueDivide', 'Divide', 'Eq', 'Ne'): - return node - elif operator == 'Divide': - # mixed old-/new-style division is not currently optimised for integers - return node - elif abs(numval.constant_result) > 2**30: - # Cut off at an integer border that is still safe for all operations. + result = optimise_numeric_binop(operator, node, ret_type, args[0], args[1]) + if not result: return node - - if operator in ('TrueDivide', 'FloorDivide', 'Divide', 'Remainder'): - if args[1].constant_result == 0: - # Don't optimise division by 0. :) - return node - - args = list(args) - args.append((ExprNodes.FloatNode if is_float else ExprNodes.IntNode)( - numval.pos, value=numval.value, constant_result=numval.constant_result, - type=num_type)) - inplace = node.inplace if isinstance(node, ExprNodes.NumBinopNode) else False - args.append(ExprNodes.BoolNode(node.pos, value=inplace, constant_result=inplace)) - if is_float or operator not in ('Eq', 'Ne'): - # "PyFloatBinop" and "PyIntBinop" take an additional "check for zero division" argument. - zerodivision_check = arg_order == 'CObj' and ( - not node.cdivision if isinstance(node, ExprNodes.DivNode) else False) - args.append(ExprNodes.BoolNode(node.pos, value=zerodivision_check, constant_result=zerodivision_check)) - - utility_code = TempitaUtilityCode.load_cached( - "PyFloatBinop" if is_float else "PyIntCompare" if operator in ('Eq', 'Ne') else "PyIntBinop", - "Optimize.c", - context=dict(op=operator, order=arg_order, ret_type=ret_type)) + func_cname, utility_code, extra_args, num_type = result + args = list(args)+extra_args call_node = self._substitute_method_call( node, function, - "__Pyx_Py%s_%s%s%s" % ( - 'Float' if is_float else 'Int', - '' if ret_type.is_pyobject else 'Bool', - operator, - arg_order), + func_cname, self.Pyx_BinopInt_func_types[(num_type, ret_type)], '__%s__' % operator[:3].lower(), is_unbound_method, args, may_return_none=True, @@ -3389,6 +3581,8 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, PyrexTypes.CFuncTypeArg("uchar", PyrexTypes.c_py_ucs4_type, None), ]) + # DISABLED: Return value can only be one character, which is not correct. + ''' def _inject_unicode_character_conversion(self, node, function, args, is_unbound_method): if is_unbound_method or len(args) != 1: return node @@ -3407,9 +3601,10 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, func_call = func_call.coerce_to_pyobject(self.current_env) return func_call - _handle_simple_method_unicode_lower = _inject_unicode_character_conversion - _handle_simple_method_unicode_upper = _inject_unicode_character_conversion - _handle_simple_method_unicode_title = _inject_unicode_character_conversion + #_handle_simple_method_unicode_lower = _inject_unicode_character_conversion + #_handle_simple_method_unicode_upper = _inject_unicode_character_conversion + #_handle_simple_method_unicode_title = _inject_unicode_character_conversion + ''' PyUnicode_Splitlines_func_type = PyrexTypes.CFuncType( Builtin.list_type, [ @@ -3448,6 +3643,8 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, return node if len(args) < 2: args.append(ExprNodes.NullNode(node.pos)) + else: + self._inject_null_for_none(args, 1) self._inject_int_default_argument( node, args, 2, PyrexTypes.c_py_ssize_t_type, "-1") @@ -3788,7 +3985,7 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, if not stop: # use strlen() to find the string length, just as CPython would if not string_node.is_name: - string_node = UtilNodes.LetRefNode(string_node) # used twice + string_node = UtilNodes.LetRefNode(string_node) # used twice temps.append(string_node) stop = ExprNodes.PythonCapiCallNode( string_node.pos, "strlen", self.Pyx_strlen_func_type, @@ -3963,13 +4160,35 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, format_args=[attr_name]) return self_arg + obj_to_obj_func_type = PyrexTypes.CFuncType( + PyrexTypes.py_object_type, [ + PyrexTypes.CFuncTypeArg("obj", PyrexTypes.py_object_type, None) + ]) + + def _inject_null_for_none(self, args, index): + if len(args) <= index: + return + arg = args[index] + args[index] = ExprNodes.NullNode(arg.pos) if arg.is_none else ExprNodes.PythonCapiCallNode( + arg.pos, "__Pyx_NoneAsNull", + self.obj_to_obj_func_type, + args=[arg.coerce_to_simple(self.current_env())], + is_temp=0, + ) + def _inject_int_default_argument(self, node, args, arg_index, type, default_value): + # Python usually allows passing None for range bounds, + # so we treat that as requesting the default. assert len(args) >= arg_index - if len(args) == arg_index: + if len(args) == arg_index or args[arg_index].is_none: args.append(ExprNodes.IntNode(node.pos, value=str(default_value), type=type, constant_result=default_value)) else: - args[arg_index] = args[arg_index].coerce_to(type, self.current_env()) + arg = args[arg_index].coerce_to(type, self.current_env()) + if isinstance(arg, ExprNodes.CoerceFromPyTypeNode): + # Add a runtime check for None and map it to the default value. + arg.special_none_cvalue = str(default_value) + args[arg_index] = arg def _inject_bint_default_argument(self, node, args, arg_index, default_value): assert len(args) >= arg_index @@ -3981,6 +4200,75 @@ class OptimizeBuiltinCalls(Visitor.NodeRefCleanupMixin, args[arg_index] = args[arg_index].coerce_to_boolean(self.current_env()) +def optimise_numeric_binop(operator, node, ret_type, arg0, arg1): + """ + Optimise math operators for (likely) float or small integer operations. + """ + # When adding IntNode/FloatNode to something else, assume other operand is also numeric. + # Prefer constants on RHS as they allows better size control for some operators. + num_nodes = (ExprNodes.IntNode, ExprNodes.FloatNode) + if isinstance(arg1, num_nodes): + if arg0.type is not PyrexTypes.py_object_type: + return None + numval = arg1 + arg_order = 'ObjC' + elif isinstance(arg0, num_nodes): + if arg1.type is not PyrexTypes.py_object_type: + return None + numval = arg0 + arg_order = 'CObj' + else: + return None + + if not numval.has_constant_result(): + return None + + # is_float is an instance check rather that numval.type.is_float because + # it will often be a Python float type rather than a C float type + is_float = isinstance(numval, ExprNodes.FloatNode) + num_type = PyrexTypes.c_double_type if is_float else PyrexTypes.c_long_type + if is_float: + if operator not in ('Add', 'Subtract', 'Remainder', 'TrueDivide', 'Divide', 'Eq', 'Ne'): + return None + elif operator == 'Divide': + # mixed old-/new-style division is not currently optimised for integers + return None + elif abs(numval.constant_result) > 2**30: + # Cut off at an integer border that is still safe for all operations. + return None + + if operator in ('TrueDivide', 'FloorDivide', 'Divide', 'Remainder'): + if arg1.constant_result == 0: + # Don't optimise division by 0. :) + return None + + extra_args = [] + + extra_args.append((ExprNodes.FloatNode if is_float else ExprNodes.IntNode)( + numval.pos, value=numval.value, constant_result=numval.constant_result, + type=num_type)) + inplace = node.inplace if isinstance(node, ExprNodes.NumBinopNode) else False + extra_args.append(ExprNodes.BoolNode(node.pos, value=inplace, constant_result=inplace)) + if is_float or operator not in ('Eq', 'Ne'): + # "PyFloatBinop" and "PyIntBinop" take an additional "check for zero division" argument. + zerodivision_check = arg_order == 'CObj' and ( + not node.cdivision if isinstance(node, ExprNodes.DivNode) else False) + extra_args.append(ExprNodes.BoolNode(node.pos, value=zerodivision_check, constant_result=zerodivision_check)) + + utility_code = TempitaUtilityCode.load_cached( + "PyFloatBinop" if is_float else "PyIntCompare" if operator in ('Eq', 'Ne') else "PyIntBinop", + "Optimize.c", + context=dict(op=operator, order=arg_order, ret_type=ret_type)) + + func_cname = "__Pyx_Py%s_%s%s%s" % ( + 'Float' if is_float else 'Int', + '' if ret_type.is_pyobject else 'Bool', + operator, + arg_order) + + return func_cname, utility_code, extra_args, num_type + + unicode_tailmatch_utility_code = UtilityCode.load_cached('unicode_tailmatch', 'StringTools.c') bytes_tailmatch_utility_code = UtilityCode.load_cached('bytes_tailmatch', 'StringTools.c') str_tailmatch_utility_code = UtilityCode.load_cached('str_tailmatch', 'StringTools.c') @@ -4439,25 +4727,25 @@ class ConstantFolding(Visitor.VisitorTransform, SkipDeclarations): args = [] items = [] - def add(arg): + def add(parent, arg): if arg.is_dict_literal: - if items: - items[0].key_value_pairs.extend(arg.key_value_pairs) + if items and items[-1].reject_duplicates == arg.reject_duplicates: + items[-1].key_value_pairs.extend(arg.key_value_pairs) else: items.append(arg) - elif isinstance(arg, ExprNodes.MergedDictNode): + elif isinstance(arg, ExprNodes.MergedDictNode) and parent.reject_duplicates == arg.reject_duplicates: for child_arg in arg.keyword_args: - add(child_arg) + add(arg, child_arg) else: if items: - args.append(items[0]) + args.extend(items) del items[:] args.append(arg) for arg in node.keyword_args: - add(arg) + add(node, arg) if items: - args.append(items[0]) + args.extend(items) if len(args) == 1: arg = args[0] @@ -4546,22 +4834,20 @@ class ConstantFolding(Visitor.VisitorTransform, SkipDeclarations): cascades = [[node.operand1]] final_false_result = [] - def split_cascades(cmp_node): + cmp_node = node + while cmp_node is not None: if cmp_node.has_constant_result(): if not cmp_node.constant_result: # False => short-circuit final_false_result.append(self._bool_node(cmp_node, False)) - return + break else: # True => discard and start new cascade cascades.append([cmp_node.operand2]) else: # not constant => append to current cascade cascades[-1].append(cmp_node) - if cmp_node.cascade: - split_cascades(cmp_node.cascade) - - split_cascades(node) + cmp_node = cmp_node.cascade cmp_nodes = [] for cascade in cascades: @@ -4707,6 +4993,30 @@ class ConstantFolding(Visitor.VisitorTransform, SkipDeclarations): return None return node + def visit_GILStatNode(self, node): + self.visitchildren(node) + if node.condition is None: + return node + + if node.condition.has_constant_result(): + # Condition is True - Modify node to be a normal + # GILStatNode with condition=None + if node.condition.constant_result: + node.condition = None + + # Condition is False - the body of the GILStatNode + # should run without changing the state of the gil + # return the body of the GILStatNode + else: + return node.body + + # If condition is not constant we keep the GILStatNode as it is. + # Either it will later become constant (e.g. a `numeric is int` + # expression in a fused type function) and then when ConstantFolding + # runs again it will be handled or a later transform (i.e. GilCheck) + # will raise an error + return node + # in the future, other nodes can have their own handler method here # that can replace them with a constant result node @@ -4723,6 +5033,7 @@ class FinalOptimizePhase(Visitor.EnvTransform, Visitor.NodeRefCleanupMixin): - isinstance -> typecheck for cdef types - eliminate checks for None and/or types that became redundant after tree changes - eliminate useless string formatting steps + - inject branch hints for unlikely if-cases that only raise exceptions - replace Python function calls that look like method calls by a faster PyMethodCallNode """ in_loop = False @@ -4821,6 +5132,48 @@ class FinalOptimizePhase(Visitor.EnvTransform, Visitor.NodeRefCleanupMixin): self.in_loop = old_val return node + def visit_IfStatNode(self, node): + """Assign 'unlikely' branch hints to if-clauses that only raise exceptions. + """ + self.visitchildren(node) + last_non_unlikely_clause = None + for i, if_clause in enumerate(node.if_clauses): + self._set_ifclause_branch_hint(if_clause, if_clause.body) + if not if_clause.branch_hint: + last_non_unlikely_clause = if_clause + if node.else_clause and last_non_unlikely_clause: + # If the 'else' clause is 'unlikely', then set the preceding 'if' clause to 'likely' to reflect that. + self._set_ifclause_branch_hint(last_non_unlikely_clause, node.else_clause, inverse=True) + return node + + def _set_ifclause_branch_hint(self, clause, statements_node, inverse=False): + """Inject a branch hint if the if-clause unconditionally leads to a 'raise' statement. + """ + if not statements_node.is_terminator: + return + # Allow simple statements, but no conditions, loops, etc. + non_branch_nodes = ( + Nodes.ExprStatNode, + Nodes.AssignmentNode, + Nodes.AssertStatNode, + Nodes.DelStatNode, + Nodes.GlobalNode, + Nodes.NonlocalNode, + ) + statements = [statements_node] + for next_node_pos, node in enumerate(statements, 1): + if isinstance(node, Nodes.GILStatNode): + statements.insert(next_node_pos, node.body) + continue + if isinstance(node, Nodes.StatListNode): + statements[next_node_pos:next_node_pos] = node.stats + continue + if not isinstance(node, non_branch_nodes): + if next_node_pos == len(statements) and isinstance(node, (Nodes.RaiseStatNode, Nodes.ReraiseStatNode)): + # Anything that unconditionally raises exceptions at the end should be considered unlikely. + clause.branch_hint = 'likely' if inverse else 'unlikely' + break + class ConsolidateOverflowCheck(Visitor.CythonTransform): """ diff --git a/Cython/Compiler/Options.py b/Cython/Compiler/Options.py index d03119fca..cd1bb0431 100644 --- a/Cython/Compiler/Options.py +++ b/Cython/Compiler/Options.py @@ -4,6 +4,10 @@ from __future__ import absolute_import +import os + +from Cython import Utils + class ShouldBeFromDirective(object): @@ -25,15 +29,14 @@ class ShouldBeFromDirective(object): raise RuntimeError(repr(self)) def __repr__(self): - return ( - "Illegal access of '%s' from Options module rather than directive '%s'" - % (self.options_name, self.directive_name)) + return "Illegal access of '%s' from Options module rather than directive '%s'" % ( + self.options_name, self.directive_name) """ The members of this module are documented using autodata in Cython/docs/src/reference/compilation.rst. -See http://www.sphinx-doc.org/en/master/ext/autodoc.html#directive-autoattribute +See https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#directive-autoattribute for how autodata works. Descriptions of those members should start with a #: Donc forget to keep the docs in sync by removing and adding @@ -48,11 +51,6 @@ docstrings = True #: Embed the source code position in the docstrings of functions and classes. embed_pos_in_docstring = False -#: Copy the original source code line by line into C code comments -#: in the generated code file to help with understanding the output. -#: This is also required for coverage analysis. -emit_code_comments = True - # undocumented pre_import = None @@ -168,8 +166,19 @@ def get_directive_defaults(): _directive_defaults[old_option.directive_name] = value return _directive_defaults +def copy_inherited_directives(outer_directives, **new_directives): + # A few directives are not copied downwards and this function removes them. + # For example, test_assert_path_exists and test_fail_if_path_exists should not be inherited + # otherwise they can produce very misleading test failures + new_directives_out = dict(outer_directives) + for name in ('test_assert_path_exists', 'test_fail_if_path_exists', 'test_assert_c_code_has', 'test_fail_if_c_code_has'): + new_directives_out.pop(name, None) + new_directives_out.update(new_directives) + return new_directives_out + # Declare compiler directives _directive_defaults = { + 'binding': True, # was False before 3.0 'boundscheck' : True, 'nonecheck' : False, 'initializedcheck' : True, @@ -178,11 +187,12 @@ _directive_defaults = { 'auto_pickle': None, 'cdivision': False, # was True before 0.12 'cdivision_warnings': False, - 'c_api_binop_methods': True, - 'cpow': True, + 'cpow': None, # was True before 3.0 + # None (not set by user) is treated as slightly different from False + 'c_api_binop_methods': False, # was True before 3.0 'overflowcheck': False, 'overflowcheck.fold': True, - 'always_allow_keywords': False, + 'always_allow_keywords': True, 'allow_none_for_extension_args': True, 'wraparound' : True, 'ccomplex' : False, # use C99/C++ for complex types and arith @@ -209,6 +219,8 @@ _directive_defaults = { 'old_style_globals': False, 'np_pythran': False, 'fast_gil': False, + 'cpp_locals': False, # uses std::optional for C++ locals, so that they work more like Python locals + 'legacy_implicit_noexcept': False, # set __file__ and/or __path__ to known source/target path at import time (instead of not having them available) 'set_initial_path' : None, # SOURCEFILE or "/full/path/to/module" @@ -238,10 +250,10 @@ _directive_defaults = { # test support 'test_assert_path_exists' : [], 'test_fail_if_path_exists' : [], + 'test_assert_c_code_has' : [], + 'test_fail_if_c_code_has' : [], # experimental, subject to change - 'binding': None, - 'formal_grammar': False, } @@ -295,6 +307,12 @@ def normalise_encoding_name(option_name, encoding): return name return encoding +# use as a sential value to defer analysis of the arguments +# instead of analysing them in InterpretCompilerDirectives. The dataclass directives are quite +# complicated and it's easier to deal with them at the point the dataclass is created +class DEFER_ANALYSIS_OF_ARGUMENTS: + pass +DEFER_ANALYSIS_OF_ARGUMENTS = DEFER_ANALYSIS_OF_ARGUMENTS() # Override types possibilities above, if needed directive_types = { @@ -302,12 +320,15 @@ directive_types = { 'auto_pickle': bool, 'locals': dict, 'final' : bool, # final cdef classes and methods + 'collection_type': one_of('sequence'), 'nogil' : bool, 'internal' : bool, # cdef class visibility in the module dict 'infer_types' : bool, # values can be True/None/False 'binding' : bool, 'cfunc' : None, # decorators do not take directive value 'ccall' : None, + 'ufunc': None, + 'cpow' : bool, 'inline' : None, 'staticmethod' : None, 'cclass' : None, @@ -319,7 +340,10 @@ directive_types = { 'freelist': int, 'c_string_type': one_of('bytes', 'bytearray', 'str', 'unicode'), 'c_string_encoding': normalise_encoding_name, - 'cpow': bool + 'trashcan': bool, + 'total_ordering': None, + 'dataclasses.dataclass': DEFER_ANALYSIS_OF_ARGUMENTS, + 'dataclasses.field': DEFER_ANALYSIS_OF_ARGUMENTS, } for key, val in _directive_defaults.items(): @@ -330,6 +354,7 @@ directive_scopes = { # defaults to available everywhere # 'module', 'function', 'class', 'with statement' 'auto_pickle': ('module', 'cclass'), 'final' : ('cclass', 'function'), + 'collection_type': ('cclass',), 'nogil' : ('function', 'with statement'), 'inline' : ('function',), 'cfunc' : ('function', 'with statement'), @@ -348,9 +373,10 @@ directive_scopes = { # defaults to available everywhere 'set_initial_path' : ('module',), 'test_assert_path_exists' : ('function', 'class', 'cclass'), 'test_fail_if_path_exists' : ('function', 'class', 'cclass'), + 'test_assert_c_code_has' : ('module',), + 'test_fail_if_c_code_has' : ('module',), 'freelist': ('cclass',), 'emit_code_comments': ('module',), - 'annotation_typing': ('module',), # FIXME: analysis currently lacks more specific function scope # Avoid scope-specific to/from_py_functions for c_string. 'c_string_type': ('module',), 'c_string_encoding': ('module',), @@ -362,6 +388,26 @@ directive_scopes = { # defaults to available everywhere 'np_pythran': ('module',), 'fast_gil': ('module',), 'iterable_coroutine': ('module', 'function'), + 'trashcan' : ('cclass',), + 'total_ordering': ('cclass', ), + 'dataclasses.dataclass' : ('class', 'cclass',), + 'cpp_locals': ('module', 'function', 'cclass'), # I don't think they make sense in a with_statement + 'ufunc': ('function',), + 'legacy_implicit_noexcept': ('module', ), +} + + +# a list of directives that (when used as a decorator) are only applied to +# the object they decorate and not to its children. +immediate_decorator_directives = { + 'cfunc', 'ccall', 'cclass', 'dataclasses.dataclass', 'ufunc', + # function signature directives + 'inline', 'exceptval', 'returns', + # class directives + 'freelist', 'no_gc', 'no_gc_clear', 'type_version_tag', 'final', + 'auto_pickle', 'internal', 'collection_type', 'total_ordering', + # testing directives + 'test_fail_if_path_exists', 'test_assert_path_exists', } @@ -476,6 +522,11 @@ def parse_directive_list(s, relaxed_bool=False, ignore_unknown=False, result[directive] = parsed_value if not found and not ignore_unknown: raise ValueError('Unknown option: "%s"' % name) + elif directive_types.get(name) is list: + if name in result: + result[name].append(value) + else: + result[name] = [value] else: parsed_value = parse_directive_value(name, value, relaxed_bool=relaxed_bool) result[name] = parsed_value @@ -550,3 +601,190 @@ def parse_compile_time_env(s, current_settings=None): name, value = [s.strip() for s in item.split('=', 1)] result[name] = parse_variable_value(value) return result + + +# ------------------------------------------------------------------------ +# CompilationOptions are constructed from user input and are the `option` +# object passed throughout the compilation pipeline. + +class CompilationOptions(object): + r""" + See default_options at the end of this module for a list of all possible + options and CmdLine.usage and CmdLine.parse_command_line() for their + meaning. + """ + def __init__(self, defaults=None, **kw): + self.include_path = [] + if defaults: + if isinstance(defaults, CompilationOptions): + defaults = defaults.__dict__ + else: + defaults = default_options + + options = dict(defaults) + options.update(kw) + + # let's assume 'default_options' contains a value for most known compiler options + # and validate against them + unknown_options = set(options) - set(default_options) + # ignore valid options that are not in the defaults + unknown_options.difference_update(['include_path']) + if unknown_options: + message = "got unknown compilation option%s, please remove: %s" % ( + 's' if len(unknown_options) > 1 else '', + ', '.join(unknown_options)) + raise ValueError(message) + + directive_defaults = get_directive_defaults() + directives = dict(options['compiler_directives']) # copy mutable field + # check for invalid directives + unknown_directives = set(directives) - set(directive_defaults) + if unknown_directives: + message = "got unknown compiler directive%s: %s" % ( + 's' if len(unknown_directives) > 1 else '', + ', '.join(unknown_directives)) + raise ValueError(message) + options['compiler_directives'] = directives + if directives.get('np_pythran', False) and not options['cplus']: + import warnings + warnings.warn("C++ mode forced when in Pythran mode!") + options['cplus'] = True + if 'language_level' in directives and 'language_level' not in kw: + options['language_level'] = directives['language_level'] + elif not options.get('language_level'): + options['language_level'] = directive_defaults.get('language_level') + if 'formal_grammar' in directives and 'formal_grammar' not in kw: + options['formal_grammar'] = directives['formal_grammar'] + if options['cache'] is True: + options['cache'] = os.path.join(Utils.get_cython_cache_dir(), 'compiler') + + self.__dict__.update(options) + + def configure_language_defaults(self, source_extension): + if source_extension == 'py': + if self.compiler_directives.get('binding') is None: + self.compiler_directives['binding'] = True + + def get_fingerprint(self): + r""" + Return a string that contains all the options that are relevant for cache invalidation. + """ + # Collect only the data that can affect the generated file(s). + data = {} + + for key, value in self.__dict__.items(): + if key in ['show_version', 'errors_to_stderr', 'verbose', 'quiet']: + # verbosity flags have no influence on the compilation result + continue + elif key in ['output_file', 'output_dir']: + # ignore the exact name of the output file + continue + elif key in ['depfile']: + # external build system dependency tracking file does not influence outputs + continue + elif key in ['timestamps']: + # the cache cares about the content of files, not about the timestamps of sources + continue + elif key in ['cache']: + # hopefully caching has no influence on the compilation result + continue + elif key in ['compiler_directives']: + # directives passed on to the C compiler do not influence the generated C code + continue + elif key in ['include_path']: + # this path changes which headers are tracked as dependencies, + # it has no influence on the generated C code + continue + elif key in ['working_path']: + # this path changes where modules and pxd files are found; + # their content is part of the fingerprint anyway, their + # absolute path does not matter + continue + elif key in ['create_extension']: + # create_extension() has already mangled the options, e.g., + # embedded_metadata, when the fingerprint is computed so we + # ignore it here. + continue + elif key in ['build_dir']: + # the (temporary) directory where we collect dependencies + # has no influence on the C output + continue + elif key in ['use_listing_file', 'generate_pxi', 'annotate', 'annotate_coverage_xml']: + # all output files are contained in the cache so the types of + # files generated must be part of the fingerprint + data[key] = value + elif key in ['formal_grammar', 'evaluate_tree_assertions']: + # these bits can change whether compilation to C passes/fails + data[key] = value + elif key in ['embedded_metadata', 'emit_linenums', + 'c_line_in_traceback', 'gdb_debug', + 'relative_path_in_code_position_comments']: + # the generated code contains additional bits when these are set + data[key] = value + elif key in ['cplus', 'language_level', 'compile_time_env', 'np_pythran']: + # assorted bits that, e.g., influence the parser + data[key] = value + elif key == ['capi_reexport_cincludes']: + if self.capi_reexport_cincludes: + # our caching implementation does not yet include fingerprints of all the header files + raise NotImplementedError('capi_reexport_cincludes is not compatible with Cython caching') + elif key == ['common_utility_include_dir']: + if self.common_utility_include_dir: + raise NotImplementedError('common_utility_include_dir is not compatible with Cython caching yet') + else: + # any unexpected option should go into the fingerprint; it's better + # to recompile than to return incorrect results from the cache. + data[key] = value + + def to_fingerprint(item): + r""" + Recursively turn item into a string, turning dicts into lists with + deterministic ordering. + """ + if isinstance(item, dict): + item = sorted([(repr(key), to_fingerprint(value)) for key, value in item.items()]) + return repr(item) + + return to_fingerprint(data) + + +# ------------------------------------------------------------------------ +# +# Set the default options depending on the platform +# +# ------------------------------------------------------------------------ + +default_options = dict( + show_version=0, + use_listing_file=0, + errors_to_stderr=1, + cplus=0, + output_file=None, + depfile=None, + annotate=None, + annotate_coverage_xml=None, + generate_pxi=0, + capi_reexport_cincludes=0, + working_path="", + timestamps=None, + verbose=0, + quiet=0, + compiler_directives={}, + embedded_metadata={}, + evaluate_tree_assertions=False, + emit_linenums=False, + relative_path_in_code_position_comments=True, + c_line_in_traceback=True, + language_level=None, # warn but default to 2 + formal_grammar=False, + gdb_debug=False, + compile_time_env=None, + module_name=None, + common_utility_include_dir=None, + output_dir=None, + build_dir=None, + cache=None, + create_extension=None, + np_pythran=False, + legacy_implicit_noexcept=None, +) diff --git a/Cython/Compiler/ParseTreeTransforms.pxd b/Cython/Compiler/ParseTreeTransforms.pxd index 2c17901fa..efbb14f70 100644 --- a/Cython/Compiler/ParseTreeTransforms.pxd +++ b/Cython/Compiler/ParseTreeTransforms.pxd @@ -1,5 +1,4 @@ - -from __future__ import absolute_import +# cython: language_level=3str cimport cython @@ -7,8 +6,8 @@ from .Visitor cimport ( CythonTransform, VisitorTransform, TreeVisitor, ScopeTrackingTransform, EnvTransform) -cdef class SkipDeclarations: # (object): - pass +# Don't include mixins, only the main classes. +#cdef class SkipDeclarations: cdef class NormalizeTree(CythonTransform): cdef bint is_in_statlist @@ -30,7 +29,7 @@ cdef map_starred_assignment(list lhs_targets, list starred_assignments, list lhs #class PxdPostParse(CythonTransform, SkipDeclarations): #class InterpretCompilerDirectives(CythonTransform, SkipDeclarations): -#class WithTransform(CythonTransform, SkipDeclarations): +#class WithTransform(VisitorTransform, SkipDeclarations): #class DecoratorTransform(CythonTransform, SkipDeclarations): #class AnalyseDeclarationsTransform(EnvTransform): diff --git a/Cython/Compiler/ParseTreeTransforms.py b/Cython/Compiler/ParseTreeTransforms.py index 0e86d5b0e..8008cba9a 100644 --- a/Cython/Compiler/ParseTreeTransforms.py +++ b/Cython/Compiler/ParseTreeTransforms.py @@ -1,3 +1,5 @@ +# cython: language_level=3str + from __future__ import absolute_import import cython @@ -54,6 +56,12 @@ class SkipDeclarations(object): def visit_CStructOrUnionDefNode(self, node): return node + def visit_CppClassNode(self, node): + if node.visibility != "extern": + # Need to traverse methods. + self.visitchildren(node) + return node + class NormalizeTree(CythonTransform): """ @@ -81,6 +89,13 @@ class NormalizeTree(CythonTransform): self.is_in_statlist = False self.is_in_expr = False + def visit_ModuleNode(self, node): + self.visitchildren(node) + if not isinstance(node.body, Nodes.StatListNode): + # This can happen when the body only consists of a single (unused) declaration and no statements. + node.body = Nodes.StatListNode(pos=node.pos, stats=[node.body]) + return node + def visit_ExprNode(self, node): stacktmp = self.is_in_expr self.is_in_expr = True @@ -170,8 +185,9 @@ class PostParse(ScopeTrackingTransform): Note: Currently Parsing.py does a lot of interpretation and reorganization that can be refactored into this transform if a more pure Abstract Syntax Tree is wanted. - """ + - Some invalid uses of := assignment expressions are detected + """ def __init__(self, context): super(PostParse, self).__init__(context) self.specialattribute_handlers = { @@ -203,7 +219,9 @@ class PostParse(ScopeTrackingTransform): node.def_node = Nodes.DefNode( node.pos, name=node.name, doc=None, args=[], star_arg=None, starstar_arg=None, - body=node.loop, is_async_def=collector.has_await) + body=node.loop, is_async_def=collector.has_await, + is_generator_expression=True) + _AssignmentExpressionChecker.do_checks(node.loop, scope_is_class=self.scope_type in ("pyclass", "cclass")) self.visitchildren(node) return node @@ -214,6 +232,7 @@ class PostParse(ScopeTrackingTransform): collector.visitchildren(node.loop) if collector.has_await: node.has_local_scope = True + _AssignmentExpressionChecker.do_checks(node.loop, scope_is_class=self.scope_type in ("pyclass", "cclass")) self.visitchildren(node) return node @@ -246,7 +265,7 @@ class PostParse(ScopeTrackingTransform): if decl is not declbase: raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE) handler(decl) - continue # Remove declaration + continue # Remove declaration raise PostParseError(decl.pos, ERR_CDEF_INCLASS) first_assignment = self.scope_type != 'module' stats.append(Nodes.SingleAssignmentNode(node.pos, @@ -349,6 +368,141 @@ class PostParse(ScopeTrackingTransform): self.visitchildren(node) return node + def visit_AssertStatNode(self, node): + """Extract the exception raising into a RaiseStatNode to simplify GIL handling. + """ + if node.exception is None: + node.exception = Nodes.RaiseStatNode( + node.pos, + exc_type=ExprNodes.NameNode(node.pos, name=EncodedString("AssertionError")), + exc_value=node.value, + exc_tb=None, + cause=None, + builtin_exc_name="AssertionError", + wrap_tuple_value=True, + ) + node.value = None + self.visitchildren(node) + return node + +class _AssignmentExpressionTargetNameFinder(TreeVisitor): + def __init__(self): + super(_AssignmentExpressionTargetNameFinder, self).__init__() + self.target_names = {} + + def find_target_names(self, target): + if target.is_name: + return [target.name] + elif target.is_sequence_constructor: + names = [] + for arg in target.args: + names.extend(self.find_target_names(arg)) + return names + # other targets are possible, but it isn't necessary to investigate them here + return [] + + def visit_ForInStatNode(self, node): + self.target_names[node] = tuple(self.find_target_names(node.target)) + self.visitchildren(node) + + def visit_ComprehensionNode(self, node): + pass # don't recurse into nested comprehensions + + def visit_LambdaNode(self, node): + pass # don't recurse into nested lambdas/generator expressions + + def visit_Node(self, node): + self.visitchildren(node) + + +class _AssignmentExpressionChecker(TreeVisitor): + """ + Enforces rules on AssignmentExpressions within generator expressions and comprehensions + """ + def __init__(self, loop_node, scope_is_class): + super(_AssignmentExpressionChecker, self).__init__() + + target_name_finder = _AssignmentExpressionTargetNameFinder() + target_name_finder.visit(loop_node) + self.target_names_dict = target_name_finder.target_names + self.in_iterator = False + self.in_nested_generator = False + self.scope_is_class = scope_is_class + self.current_target_names = () + self.all_target_names = set() + for names in self.target_names_dict.values(): + self.all_target_names.update(names) + + def _reset_state(self): + old_state = (self.in_iterator, self.in_nested_generator, self.scope_is_class, self.all_target_names, self.current_target_names) + # note: not resetting self.in_iterator here, see visit_LambdaNode() below + self.in_nested_generator = False + self.scope_is_class = False + self.current_target_names = () + self.all_target_names = set() + return old_state + + def _set_state(self, old_state): + self.in_iterator, self.in_nested_generator, self.scope_is_class, self.all_target_names, self.current_target_names = old_state + + @classmethod + def do_checks(cls, loop_node, scope_is_class): + checker = cls(loop_node, scope_is_class) + checker.visit(loop_node) + + def visit_ForInStatNode(self, node): + if self.in_nested_generator: + self.visitchildren(node) # once nested, don't do anything special + return + + current_target_names = self.current_target_names + target_name = self.target_names_dict.get(node, None) + if target_name: + self.current_target_names += target_name + + self.in_iterator = True + self.visit(node.iterator) + self.in_iterator = False + self.visitchildren(node, exclude=("iterator",)) + + self.current_target_names = current_target_names + + def visit_AssignmentExpressionNode(self, node): + if self.in_iterator: + error(node.pos, "assignment expression cannot be used in a comprehension iterable expression") + if self.scope_is_class: + error(node.pos, "assignment expression within a comprehension cannot be used in a class body") + if node.target_name in self.current_target_names: + error(node.pos, "assignment expression cannot rebind comprehension iteration variable '%s'" % + node.target_name) + elif node.target_name in self.all_target_names: + error(node.pos, "comprehension inner loop cannot rebind assignment expression target '%s'" % + node.target_name) + + def visit_LambdaNode(self, node): + # Don't reset "in_iterator" - an assignment expression in a lambda in an + # iterator is explicitly tested by the Python testcases and banned. + old_state = self._reset_state() + # the lambda node's "def_node" is not set up at this point, so we need to recurse into it explicitly. + self.visit(node.result_expr) + self._set_state(old_state) + + def visit_ComprehensionNode(self, node): + in_nested_generator = self.in_nested_generator + self.in_nested_generator = True + self.visitchildren(node) + self.in_nested_generator = in_nested_generator + + def visit_GeneratorExpressionNode(self, node): + in_nested_generator = self.in_nested_generator + self.in_nested_generator = True + # def_node isn't set up yet, so we need to visit the loop directly. + self.visit(node.loop) + self.in_nested_generator = in_nested_generator + + def visit_Node(self, node): + self.visitchildren(node) + def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence): """Replace rhs items by LetRefNodes if they appear more than once. @@ -419,7 +573,7 @@ def sort_common_subsequences(items): return b.is_sequence_constructor and contains(b.args, a) for pos, item in enumerate(items): - key = item[1] # the ResultRefNode which has already been injected into the sequences + key = item[1] # the ResultRefNode which has already been injected into the sequences new_pos = pos for i in range(pos-1, -1, -1): if lower_than(key, items[i][0]): @@ -449,7 +603,7 @@ def flatten_parallel_assignments(input, output): # recursively, so that nested structures get matched as well. rhs = input[-1] if (not (rhs.is_sequence_constructor or isinstance(rhs, ExprNodes.UnicodeNode)) - or not sum([lhs.is_sequence_constructor for lhs in input[:-1]])): + or not sum([lhs.is_sequence_constructor for lhs in input[:-1]])): output.append(input) return @@ -533,7 +687,7 @@ def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args) targets.append(expr) # the starred target itself, must be assigned a (potentially empty) list - target = lhs_args[starred].target # unpack starred node + target = lhs_args[starred].target # unpack starred node starred_rhs = rhs_args[starred:] if lhs_remaining: starred_rhs = starred_rhs[:-lhs_remaining] @@ -579,19 +733,19 @@ class PxdPostParse(CythonTransform, SkipDeclarations): err = self.ERR_INLINE_ONLY if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass' - and node.name in ('__getbuffer__', '__releasebuffer__')): - err = None # allow these slots + and node.name in ('__getbuffer__', '__releasebuffer__')): + err = None # allow these slots if isinstance(node, Nodes.CFuncDefNode): if (u'inline' in node.modifiers and - self.scope_type in ('pxd', 'cclass')): + self.scope_type in ('pxd', 'cclass')): node.inline_in_pxd = True if node.visibility != 'private': err = self.ERR_NOGO_WITH_INLINE % node.visibility elif node.api: err = self.ERR_NOGO_WITH_INLINE % 'api' else: - err = None # allow inline function + err = None # allow inline function else: err = self.ERR_INLINE_ONLY @@ -630,6 +784,9 @@ class InterpretCompilerDirectives(CythonTransform): - Command-line arguments overriding these - @cython.directivename decorators - with cython.directivename: statements + - replaces "cython.compiled" with BoolNode(value=True) + allowing unreachable blocks to be removed at a fairly early stage + before cython typing rules are forced on applied This transform is responsible for interpreting these various sources and store the directive in two ways: @@ -668,17 +825,27 @@ class InterpretCompilerDirectives(CythonTransform): 'operator.comma' : ExprNodes.c_binop_constructor(','), } - special_methods = set(['declare', 'union', 'struct', 'typedef', - 'sizeof', 'cast', 'pointer', 'compiled', - 'NULL', 'fused_type', 'parallel']) + special_methods = { + 'declare', 'union', 'struct', 'typedef', + 'sizeof', 'cast', 'pointer', 'compiled', + 'NULL', 'fused_type', 'parallel', + } special_methods.update(unop_method_nodes) - valid_parallel_directives = set([ + valid_cython_submodules = { + 'cimports', + 'dataclasses', + 'operator', + 'parallel', + 'view', + } + + valid_parallel_directives = { "parallel", "prange", "threadid", #"threadsavailable", - ]) + } def __init__(self, context, compilation_directive_defaults): super(InterpretCompilerDirectives, self).__init__(context) @@ -701,6 +868,44 @@ class InterpretCompilerDirectives(CythonTransform): error(pos, "Invalid directive: '%s'." % (directive,)) return True + def _check_valid_cython_module(self, pos, module_name): + if not module_name.startswith("cython."): + return + submodule = module_name.split('.', 2)[1] + if submodule in self.valid_cython_submodules: + return + + extra = "" + # This is very rarely used, so don't waste space on static tuples. + hints = [ + line.split() for line in """\ + imp cimports + cimp cimports + para parallel + parra parallel + dataclass dataclasses + """.splitlines()[:-1] + ] + for wrong, correct in hints: + if module_name.startswith("cython." + wrong): + extra = "Did you mean 'cython.%s' ?" % correct + break + if not extra: + is_simple_cython_name = submodule in Options.directive_types + if not is_simple_cython_name and not submodule.startswith("_"): + # Try to find it in the Shadow module (i.e. the pure Python namespace of cython.*). + # FIXME: use an internal reference of "cython.*" names instead of Shadow.py + from .. import Shadow + is_simple_cython_name = hasattr(Shadow, submodule) + if is_simple_cython_name: + extra = "Instead, use 'import cython' and then 'cython.%s'." % submodule + + error(pos, "'%s' is not a valid cython.* module%s%s" % ( + module_name, + ". " if extra else "", + extra, + )) + # Set up processing and handle the cython: comments. def visit_ModuleNode(self, node): for key in sorted(node.directive_comments): @@ -717,6 +922,12 @@ class InterpretCompilerDirectives(CythonTransform): node.cython_module_names = self.cython_module_names return node + def visit_CompilerDirectivesNode(self, node): + old_directives, self.directives = self.directives, node.directives + self.visitchildren(node) + self.directives = old_directives + return node + # The following four functions track imports and cimports that # begin with "cython" def is_cython_directive(self, name): @@ -749,22 +960,36 @@ class InterpretCompilerDirectives(CythonTransform): return result def visit_CImportStatNode(self, node): - if node.module_name == u"cython": + module_name = node.module_name + if module_name == u"cython.cimports": + error(node.pos, "Cannot cimport the 'cython.cimports' package directly, only submodules.") + if module_name.startswith(u"cython.cimports."): + if node.as_name and node.as_name != u'cython': + node.module_name = module_name[len(u"cython.cimports."):] + return node + error(node.pos, + "Python cimports must use 'from cython.cimports... import ...'" + " or 'import ... as ...', not just 'import ...'") + + if module_name == u"cython": self.cython_module_names.add(node.as_name or u"cython") - elif node.module_name.startswith(u"cython."): - if node.module_name.startswith(u"cython.parallel."): + elif module_name.startswith(u"cython."): + if module_name.startswith(u"cython.parallel."): error(node.pos, node.module_name + " is not a module") - if node.module_name == u"cython.parallel": + else: + self._check_valid_cython_module(node.pos, module_name) + + if module_name == u"cython.parallel": if node.as_name and node.as_name != u"cython": - self.parallel_directives[node.as_name] = node.module_name + self.parallel_directives[node.as_name] = module_name else: self.cython_module_names.add(u"cython") self.parallel_directives[ - u"cython.parallel"] = node.module_name + u"cython.parallel"] = module_name self.module_scope.use_utility_code( UtilityCode.load_cached("InitThreads", "ModuleSetupCode.c")) elif node.as_name: - self.directive_names[node.as_name] = node.module_name[7:] + self.directive_names[node.as_name] = module_name[7:] else: self.cython_module_names.add(u"cython") # if this cimport was a compiler directive, we don't @@ -773,26 +998,31 @@ class InterpretCompilerDirectives(CythonTransform): return node def visit_FromCImportStatNode(self, node): - if not node.relative_level and ( - node.module_name == u"cython" or node.module_name.startswith(u"cython.")): - submodule = (node.module_name + u".")[7:] + module_name = node.module_name + if module_name == u"cython.cimports" or module_name.startswith(u"cython.cimports."): + # only supported for convenience + return self._create_cimport_from_import( + node.pos, module_name, node.relative_level, node.imported_names) + elif not node.relative_level and ( + module_name == u"cython" or module_name.startswith(u"cython.")): + self._check_valid_cython_module(node.pos, module_name) + submodule = (module_name + u".")[7:] newimp = [] - - for pos, name, as_name, kind in node.imported_names: + for pos, name, as_name in node.imported_names: full_name = submodule + name qualified_name = u"cython." + full_name - if self.is_parallel_directive(qualified_name, node.pos): # from cython cimport parallel, or # from cython.parallel cimport parallel, prange, ... self.parallel_directives[as_name or name] = qualified_name elif self.is_cython_directive(full_name): self.directive_names[as_name or name] = full_name - if kind is not None: - self.context.nonfatal_error(PostParseError(pos, - "Compiler directive imports must be plain imports")) + elif full_name in ['dataclasses', 'typing']: + self.directive_names[as_name or name] = full_name + # unlike many directives, still treat it as a regular module + newimp.append((pos, name, as_name)) else: - newimp.append((pos, name, as_name, kind)) + newimp.append((pos, name, as_name)) if not newimp: return None @@ -801,9 +1031,18 @@ class InterpretCompilerDirectives(CythonTransform): return node def visit_FromImportStatNode(self, node): - if (node.module.module_name.value == u"cython") or \ - node.module.module_name.value.startswith(u"cython."): - submodule = (node.module.module_name.value + u".")[7:] + import_node = node.module + module_name = import_node.module_name.value + if module_name == u"cython.cimports" or module_name.startswith(u"cython.cimports."): + imported_names = [] + for name, name_node in node.items: + imported_names.append( + (name_node.pos, name, None if name == name_node.name else name_node.name)) + return self._create_cimport_from_import( + node.pos, module_name, import_node.level, imported_names) + elif module_name == u"cython" or module_name.startswith(u"cython."): + self._check_valid_cython_module(import_node.module_name.pos, module_name) + submodule = (module_name + u".")[7:] newimp = [] for name, name_node in node.items: full_name = submodule + name @@ -819,20 +1058,34 @@ class InterpretCompilerDirectives(CythonTransform): node.items = newimp return node + def _create_cimport_from_import(self, node_pos, module_name, level, imported_names): + if module_name == u"cython.cimports" or module_name.startswith(u"cython.cimports."): + module_name = EncodedString(module_name[len(u"cython.cimports."):]) # may be empty + + if module_name: + # from cython.cimports.a.b import x, y, z => from a.b cimport x, y, z + return Nodes.FromCImportStatNode( + node_pos, module_name=module_name, + relative_level=level, + imported_names=imported_names) + else: + # from cython.cimports import x, y, z => cimport x; cimport y; cimport z + return [ + Nodes.CImportStatNode( + pos, + module_name=dotted_name, + as_name=as_name, + is_absolute=level == 0) + for pos, dotted_name, as_name in imported_names + ] + def visit_SingleAssignmentNode(self, node): if isinstance(node.rhs, ExprNodes.ImportNode): module_name = node.rhs.module_name.value - is_parallel = (module_name + u".").startswith(u"cython.parallel.") - - if module_name != u"cython" and not is_parallel: + if module_name != u"cython" and not module_name.startswith("cython."): return node - module_name = node.rhs.module_name.value - as_name = node.lhs.name - - node = Nodes.CImportStatNode(node.pos, - module_name = module_name, - as_name = as_name) + node = Nodes.CImportStatNode(node.pos, module_name=module_name, as_name=node.lhs.name) node = self.visit_CImportStatNode(node) else: self.visitchildren(node) @@ -840,16 +1093,35 @@ class InterpretCompilerDirectives(CythonTransform): return node def visit_NameNode(self, node): + if node.annotation: + self.visitchild(node, 'annotation') if node.name in self.cython_module_names: node.is_cython_module = True else: directive = self.directive_names.get(node.name) if directive is not None: node.cython_attribute = directive + if node.as_cython_attribute() == "compiled": + return ExprNodes.BoolNode(node.pos, value=True) # replace early so unused branches can be dropped + # before they have a chance to cause compile-errors + return node + + def visit_AttributeNode(self, node): + self.visitchildren(node) + if node.as_cython_attribute() == "compiled": + return ExprNodes.BoolNode(node.pos, value=True) # replace early so unused branches can be dropped + # before they have a chance to cause compile-errors + return node + + def visit_AnnotationNode(self, node): + # for most transforms annotations are left unvisited (because they're unevaluated) + # however, it is important to pick up compiler directives from them + if node.expr: + self.visit(node.expr) return node def visit_NewExprNode(self, node): - self.visit(node.cppclass) + self.visitchild(node, 'cppclass') self.visitchildren(node) return node @@ -858,7 +1130,7 @@ class InterpretCompilerDirectives(CythonTransform): # decorator), returns a list of (directivename, value) pairs. # Otherwise, returns None if isinstance(node, ExprNodes.CallNode): - self.visit(node.function) + self.visitchild(node, 'function') optname = node.function.as_cython_attribute() if optname: directivetype = Options.directive_types.get(optname) @@ -890,7 +1162,7 @@ class InterpretCompilerDirectives(CythonTransform): if directivetype is bool: arg = ExprNodes.BoolNode(node.pos, value=True) return [self.try_to_parse_directive(optname, [arg], None, node.pos)] - elif directivetype is None: + elif directivetype is None or directivetype is Options.DEFER_ANALYSIS_OF_ARGUMENTS: return [(optname, None)] else: raise PostParseError( @@ -945,7 +1217,7 @@ class InterpretCompilerDirectives(CythonTransform): if len(args) != 0: raise PostParseError(pos, 'The %s directive takes no prepositional arguments' % optname) - return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs]) + return optname, kwds.as_python_dict() elif directivetype is list: if kwds and len(kwds.key_value_pairs) != 0: raise PostParseError(pos, @@ -957,21 +1229,42 @@ class InterpretCompilerDirectives(CythonTransform): raise PostParseError(pos, 'The %s directive takes one compile-time string argument' % optname) return (optname, directivetype(optname, str(args[0].value))) + elif directivetype is Options.DEFER_ANALYSIS_OF_ARGUMENTS: + # signal to pass things on without processing + return (optname, (args, kwds.as_python_dict() if kwds else {})) else: assert False - def visit_with_directives(self, node, directives): + def visit_with_directives(self, node, directives, contents_directives): + # contents_directives may be None if not directives: + assert not contents_directives return self.visit_Node(node) old_directives = self.directives - new_directives = dict(old_directives) - new_directives.update(directives) + new_directives = Options.copy_inherited_directives(old_directives, **directives) + if contents_directives is not None: + new_contents_directives = Options.copy_inherited_directives( + old_directives, **contents_directives) + else: + new_contents_directives = new_directives if new_directives == old_directives: return self.visit_Node(node) self.directives = new_directives + if (contents_directives is not None and + new_contents_directives != new_directives): + # we need to wrap the node body in a compiler directives node + node.body = Nodes.StatListNode( + node.body.pos, + stats=[ + Nodes.CompilerDirectivesNode( + node.body.pos, + directives=new_contents_directives, + body=node.body) + ] + ) retbody = self.visit_Node(node) self.directives = old_directives @@ -980,13 +1273,14 @@ class InterpretCompilerDirectives(CythonTransform): return Nodes.CompilerDirectivesNode( pos=retbody.pos, body=retbody, directives=new_directives) + # Handle decorators def visit_FuncDefNode(self, node): - directives = self._extract_directives(node, 'function') - return self.visit_with_directives(node, directives) + directives, contents_directives = self._extract_directives(node, 'function') + return self.visit_with_directives(node, directives, contents_directives) def visit_CVarDefNode(self, node): - directives = self._extract_directives(node, 'function') + directives, _ = self._extract_directives(node, 'function') for name, value in directives.items(): if name == 'locals': node.directive_locals = value @@ -995,27 +1289,34 @@ class InterpretCompilerDirectives(CythonTransform): node.pos, "Cdef functions can only take cython.locals(), " "staticmethod, or final decorators, got %s." % name)) - return self.visit_with_directives(node, directives) + return self.visit_with_directives(node, directives, contents_directives=None) def visit_CClassDefNode(self, node): - directives = self._extract_directives(node, 'cclass') - return self.visit_with_directives(node, directives) + directives, contents_directives = self._extract_directives(node, 'cclass') + return self.visit_with_directives(node, directives, contents_directives) def visit_CppClassNode(self, node): - directives = self._extract_directives(node, 'cppclass') - return self.visit_with_directives(node, directives) + directives, contents_directives = self._extract_directives(node, 'cppclass') + return self.visit_with_directives(node, directives, contents_directives) def visit_PyClassDefNode(self, node): - directives = self._extract_directives(node, 'class') - return self.visit_with_directives(node, directives) + directives, contents_directives = self._extract_directives(node, 'class') + return self.visit_with_directives(node, directives, contents_directives) def _extract_directives(self, node, scope_name): + """ + Returns two dicts - directives applied to this function/class + and directives applied to its contents. They aren't always the + same (since e.g. cfunc should not be applied to inner functions) + """ if not node.decorators: - return {} + return {}, {} # Split the decorators into two lists -- real decorators and directives directives = [] realdecs = [] both = [] + current_opt_dict = dict(self.directives) + missing = object() # Decorators coming first take precedence. for dec in node.decorators[::-1]: new_directives = self.try_to_parse_directives(dec.decorator) @@ -1023,8 +1324,14 @@ class InterpretCompilerDirectives(CythonTransform): for directive in new_directives: if self.check_directive_scope(node.pos, directive[0], scope_name): name, value = directive - if self.directives.get(name, object()) != value: + if current_opt_dict.get(name, missing) != value: + if name == 'cfunc' and 'ufunc' in current_opt_dict: + error(dec.pos, "Cannot apply @cfunc to @ufunc, please reverse the decorators.") directives.append(directive) + current_opt_dict[name] = value + else: + warning(dec.pos, "Directive does not change previous value (%s%s)" % ( + name, '=%r' % value if value is not None else '')) if directive[0] == 'staticmethod': both.append(dec) # Adapt scope type based on decorators that change it. @@ -1033,13 +1340,21 @@ class InterpretCompilerDirectives(CythonTransform): else: realdecs.append(dec) if realdecs and (scope_name == 'cclass' or - isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode, Nodes.CVarDefNode))): + isinstance(node, (Nodes.CClassDefNode, Nodes.CVarDefNode))): + for realdec in realdecs: + dec_pos = realdec.pos + realdec = realdec.decorator + if ((realdec.is_name and realdec.name == "dataclass") or + (realdec.is_attribute and realdec.attribute == "dataclass")): + error(dec_pos, + "Use '@cython.dataclasses.dataclass' on cdef classes to create a dataclass") + # Note - arbitrary C function decorators are caught later in DecoratorTransform raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.") node.decorators = realdecs[::-1] + both[::-1] # merge or override repeated directives optdict = {} - for directive in directives: - name, value = directive + contents_optdict = {} + for name, value in directives: if name in optdict: old_value = optdict[name] # keywords and arg lists can be merged, everything @@ -1052,7 +1367,9 @@ class InterpretCompilerDirectives(CythonTransform): optdict[name] = value else: optdict[name] = value - return optdict + if name not in Options.immediate_decorator_directives: + contents_optdict[name] = value + return optdict, contents_optdict # Handle with-statements def visit_WithStatNode(self, node): @@ -1071,7 +1388,7 @@ class InterpretCompilerDirectives(CythonTransform): if self.check_directive_scope(node.pos, name, 'with statement'): directive_dict[name] = value if directive_dict: - return self.visit_with_directives(node.body, directive_dict) + return self.visit_with_directives(node.body, directive_dict, contents_directives=None) return self.visit_Node(node) @@ -1161,7 +1478,7 @@ class ParallelRangeTransform(CythonTransform, SkipDeclarations): return node def visit_CallNode(self, node): - self.visit(node.function) + self.visitchild(node, 'function') if not self.parallel_directive: self.visitchildren(node, exclude=('function',)) return node @@ -1194,7 +1511,7 @@ class ParallelRangeTransform(CythonTransform, SkipDeclarations): "Nested parallel with blocks are disallowed") self.state = 'parallel with' - body = self.visit(node.body) + body = self.visitchild(node, 'body') self.state = None newnode.body = body @@ -1210,13 +1527,13 @@ class ParallelRangeTransform(CythonTransform, SkipDeclarations): error(node.pos, "The parallel directive must be called") return None - node.body = self.visit(node.body) + self.visitchild(node, 'body') return node def visit_ForInStatNode(self, node): "Rewrite 'for i in cython.parallel.prange(...):'" - self.visit(node.iterator) - self.visit(node.target) + self.visitchild(node, 'iterator') + self.visitchild(node, 'target') in_prange = isinstance(node.iterator.sequence, Nodes.ParallelRangeNode) @@ -1239,9 +1556,9 @@ class ParallelRangeTransform(CythonTransform, SkipDeclarations): self.state = 'prange' - self.visit(node.body) + self.visitchild(node, 'body') self.state = previous_state - self.visit(node.else_clause) + self.visitchild(node, 'else_clause') return node def visit(self, node): @@ -1250,12 +1567,13 @@ class ParallelRangeTransform(CythonTransform, SkipDeclarations): return super(ParallelRangeTransform, self).visit(node) -class WithTransform(CythonTransform, SkipDeclarations): +class WithTransform(VisitorTransform, SkipDeclarations): def visit_WithStatNode(self, node): self.visitchildren(node, 'body') pos = node.pos is_async = node.is_async body, target, manager = node.body, node.target, node.manager + manager = node.manager = ExprNodes.ProxyNode(manager) node.enter_call = ExprNodes.SimpleCallNode( pos, function=ExprNodes.AttributeNode( pos, obj=ExprNodes.CloneNode(manager), @@ -1316,6 +1634,130 @@ class WithTransform(CythonTransform, SkipDeclarations): # With statements are never inside expressions. return node + visit_Node = VisitorTransform.recurse_to_children + + +class _GeneratorExpressionArgumentsMarker(TreeVisitor, SkipDeclarations): + # called from "MarkClosureVisitor" + def __init__(self, gen_expr): + super(_GeneratorExpressionArgumentsMarker, self).__init__() + self.gen_expr = gen_expr + + def visit_ExprNode(self, node): + if not node.is_literal: + # Don't bother tagging literal nodes + assert (not node.generator_arg_tag) # nobody has tagged this first + node.generator_arg_tag = self.gen_expr + self.visitchildren(node) + + def visit_Node(self, node): + # We're only interested in the expressions that make up the iterator sequence, + # so don't go beyond ExprNodes (e.g. into ForFromStatNode). + return + + def visit_GeneratorExpressionNode(self, node): + node.generator_arg_tag = self.gen_expr + # don't visit children, can't handle overlapping tags + # (and assume generator expressions don't end up optimized out in a way + # that would require overlapping tags) + + +class _HandleGeneratorArguments(VisitorTransform, SkipDeclarations): + # used from within CreateClosureClasses + + def __call__(self, node): + from . import Visitor + assert isinstance(node, ExprNodes.GeneratorExpressionNode) + self.gen_node = node + + self.args = list(node.def_node.args) + self.call_parameters = list(node.call_parameters) + self.tag_count = 0 + self.substitutions = {} + + self.visitchildren(node) + + for k, v in self.substitutions.items(): + # doing another search for replacements here (at the end) allows us to sweep up + # CloneNodes too (which are often generated by the optimizer) + # (it could arguably be done more efficiently with a single traversal though) + Visitor.recursively_replace_node(node, k, v) + + node.def_node.args = self.args + node.call_parameters = self.call_parameters + return node + + def visit_GeneratorExpressionNode(self, node): + # a generator can also be substituted itself, so handle that case + new_node = self._handle_ExprNode(node, do_visit_children=False) + # However do not traverse into it. A new _HandleGeneratorArguments visitor will be used + # elsewhere to do that. + return node + + def _handle_ExprNode(self, node, do_visit_children): + if (node.generator_arg_tag is not None and self.gen_node is not None and + self.gen_node == node.generator_arg_tag): + pos = node.pos + # The reason for using ".x" as the name is that this is how CPython + # tracks internal variables in loops (e.g. + # { locals() for v in range(10) } + # will produce "v" and ".0"). We don't replicate this behaviour completely + # but use it as a starting point + name_source = self.tag_count + self.tag_count += 1 + name = EncodedString(".{0}".format(name_source)) + def_node = self.gen_node.def_node + if not def_node.local_scope.lookup_here(name): + from . import Symtab + cname = EncodedString(Naming.genexpr_arg_prefix + Symtab.punycodify_name(str(name_source))) + name_decl = Nodes.CNameDeclaratorNode(pos=pos, name=name) + type = node.type + if type.is_reference and not type.is_fake_reference: + # It isn't obvious whether the right thing to do would be to capture by reference or by + # value (C++ itself doesn't know either for lambda functions and forces a choice). + # However, capture by reference involves converting to FakeReference which would require + # re-analysing AttributeNodes. Therefore I've picked capture-by-value out of convenience + # TODO - could probably be optimized by making the arg a reference but the closure not + # (see https://github.com/cython/cython/issues/2468) + type = type.ref_base_type + + name_decl.type = type + new_arg = Nodes.CArgDeclNode(pos=pos, declarator=name_decl, + base_type=None, default=None, annotation=None) + new_arg.name = name_decl.name + new_arg.type = type + + self.args.append(new_arg) + node.generator_arg_tag = None # avoid the possibility of this being caught again + self.call_parameters.append(node) + new_arg.entry = def_node.declare_argument(def_node.local_scope, new_arg) + new_arg.entry.cname = cname + new_arg.entry.in_closure = True + + if do_visit_children: + # now visit the Nodes's children (but remove self.gen_node to not to further + # argument substitution) + gen_node, self.gen_node = self.gen_node, None + self.visitchildren(node) + self.gen_node = gen_node + + # replace the node inside the generator with a looked-up name + # (initialized_check can safely be False because the source variable will be checked + # before it is captured if the check is required) + name_node = ExprNodes.NameNode(pos, name=name, initialized_check=False) + name_node.entry = self.gen_node.def_node.gbody.local_scope.lookup(name_node.name) + name_node.type = name_node.entry.type + self.substitutions[node] = name_node + return name_node + if do_visit_children: + self.visitchildren(node) + return node + + def visit_ExprNode(self, node): + return self._handle_ExprNode(node, True) + + visit_Node = VisitorTransform.recurse_to_children + class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): """ @@ -1328,16 +1770,16 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): _properties = None _map_property_attribute = { - 'getter': '__get__', - 'setter': '__set__', - 'deleter': '__del__', + 'getter': EncodedString('__get__'), + 'setter': EncodedString('__set__'), + 'deleter': EncodedString('__del__'), }.get def visit_CClassDefNode(self, node): if self._properties is None: self._properties = [] self._properties.append({}) - super(DecoratorTransform, self).visit_CClassDefNode(node) + node = super(DecoratorTransform, self).visit_CClassDefNode(node) self._properties.pop() return node @@ -1347,6 +1789,32 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): warning(node.pos, "'property %s:' syntax is deprecated, use '@property'" % node.name, level) return node + def visit_CFuncDefNode(self, node): + node = self.visit_FuncDefNode(node) + if not node.decorators: + return node + elif self.scope_type != 'cclass' or self.scope_node.visibility != "extern": + # at the moment cdef functions are very restricted in what decorators they can take + # so it's simple to test for the small number of allowed decorators.... + if not (len(node.decorators) == 1 and node.decorators[0].decorator.is_name and + node.decorators[0].decorator.name == "staticmethod"): + error(node.decorators[0].pos, "Cdef functions cannot take arbitrary decorators.") + return node + + ret_node = node + decorator_node = self._find_property_decorator(node) + if decorator_node: + if decorator_node.decorator.is_name: + name = node.declared_name() + if name: + ret_node = self._add_property(node, name, decorator_node) + else: + error(decorator_node.pos, "C property decorator can only be @property") + + if node.decorators: + return self._reject_decorated_property(node, node.decorators[0]) + return ret_node + def visit_DefNode(self, node): scope_type = self.scope_type node = self.visit_FuncDefNode(node) @@ -1354,28 +1822,12 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): return node # transform @property decorators - properties = self._properties[-1] - for decorator_node in node.decorators[::-1]: + decorator_node = self._find_property_decorator(node) + if decorator_node is not None: decorator = decorator_node.decorator - if decorator.is_name and decorator.name == 'property': - if len(node.decorators) > 1: - return self._reject_decorated_property(node, decorator_node) - name = node.name - node.name = EncodedString('__get__') - node.decorators.remove(decorator_node) - stat_list = [node] - if name in properties: - prop = properties[name] - prop.pos = node.pos - prop.doc = node.doc - prop.body.stats = stat_list - return [] - prop = Nodes.PropertyNode(node.pos, name=name) - prop.doc = node.doc - prop.body = Nodes.StatListNode(node.pos, stats=stat_list) - properties[name] = prop - return [prop] - elif decorator.is_attribute and decorator.obj.name in properties: + if decorator.is_name: + return self._add_property(node, node.name, decorator_node) + else: handler_name = self._map_property_attribute(decorator.attribute) if handler_name: if decorator.obj.name != node.name: @@ -1386,7 +1838,7 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): elif len(node.decorators) > 1: return self._reject_decorated_property(node, decorator_node) else: - return self._add_to_property(properties, node, handler_name, decorator_node) + return self._add_to_property(node, handler_name, decorator_node) # we clear node.decorators, so we need to set the # is_staticmethod/is_classmethod attributes now @@ -1401,6 +1853,18 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): node.decorators = None return self.chain_decorators(node, decs, node.name) + def _find_property_decorator(self, node): + properties = self._properties[-1] + for decorator_node in node.decorators[::-1]: + decorator = decorator_node.decorator + if decorator.is_name and decorator.name == 'property': + # @property + return decorator_node + elif decorator.is_attribute and decorator.obj.name in properties: + # @prop.setter etc. + return decorator_node + return None + @staticmethod def _reject_decorated_property(node, decorator_node): # restrict transformation to outermost decorator as wrapped properties will probably not work @@ -1409,9 +1873,42 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): error(deco.pos, "Property methods with additional decorators are not supported") return node - @staticmethod - def _add_to_property(properties, node, name, decorator): + def _add_property(self, node, name, decorator_node): + if len(node.decorators) > 1: + return self._reject_decorated_property(node, decorator_node) + node.decorators.remove(decorator_node) + properties = self._properties[-1] + is_cproperty = isinstance(node, Nodes.CFuncDefNode) + body = Nodes.StatListNode(node.pos, stats=[node]) + if is_cproperty: + if name in properties: + error(node.pos, "C property redeclared") + if 'inline' not in node.modifiers: + error(node.pos, "C property method must be declared 'inline'") + prop = Nodes.CPropertyNode(node.pos, doc=node.doc, name=name, body=body) + elif name in properties: + prop = properties[name] + if prop.is_cproperty: + error(node.pos, "C property redeclared") + else: + node.name = EncodedString("__get__") + prop.pos = node.pos + prop.doc = node.doc + prop.body.stats = [node] + return None + else: + node.name = EncodedString("__get__") + prop = Nodes.PropertyNode( + node.pos, name=name, doc=node.doc, body=body) + properties[name] = prop + return prop + + def _add_to_property(self, node, name, decorator): + properties = self._properties[-1] prop = properties[node.name] + if prop.is_cproperty: + error(node.pos, "C property redeclared") + return None node.name = name node.decorators.remove(decorator) stats = prop.body.stats @@ -1421,7 +1918,7 @@ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations): break else: stats.append(node) - return [] + return None @staticmethod def chain_decorators(node, decorators, name): @@ -1499,6 +1996,10 @@ class CnameDirectivesTransform(CythonTransform, SkipDeclarations): class ForwardDeclareTypes(CythonTransform): + """ + Declare all global cdef names that we allow referencing in other places, + before declaring everything (else) in source code order. + """ def visit_CompilerDirectivesNode(self, node): env = self.module_scope @@ -1542,6 +2043,14 @@ class ForwardDeclareTypes(CythonTransform): entry.type.get_all_specialized_function_types() return node + def visit_FuncDefNode(self, node): + # no traversal needed + return node + + def visit_PyClassDefNode(self, node): + # no traversal needed + return node + class AnalyseDeclarationsTransform(EnvTransform): @@ -1622,6 +2131,9 @@ if VALUE is not None: def visit_CClassDefNode(self, node): node = self.visit_ClassDefNode(node) + if node.scope and 'dataclasses.dataclass' in node.scope.directives: + from .Dataclass import handle_cclass_dataclass + handle_cclass_dataclass(node, node.scope.directives['dataclasses.dataclass'], self) if node.scope and node.scope.implemented and node.body: stats = [] for entry in node.scope.var_entries: @@ -1633,8 +2145,8 @@ if VALUE is not None: if stats: node.body.stats += stats if (node.visibility != 'extern' - and not node.scope.lookup('__reduce__') - and not node.scope.lookup('__reduce_ex__')): + and not node.scope.lookup('__reduce__') + and not node.scope.lookup('__reduce_ex__')): self._inject_pickle_methods(node) return node @@ -1688,9 +2200,9 @@ if VALUE is not None: pickle_func = TreeFragment(u""" def __reduce_cython__(self): - raise TypeError("%(msg)s") + raise TypeError, "%(msg)s" def __setstate_cython__(self, __pyx_state): - raise TypeError("%(msg)s") + raise TypeError, "%(msg)s" """ % {'msg': msg}, level='c_class', pipeline=[NormalizeTree(None)]).substitute({}) pickle_func.analyse_declarations(node.scope) @@ -1702,10 +2214,11 @@ if VALUE is not None: if not e.type.is_pyobject: e.type.create_to_py_utility_code(env) e.type.create_from_py_utility_code(env) + all_members_names = [e.name for e in all_members] checksums = _calculate_pickle_checksums(all_members_names) - unpickle_func_name = '__pyx_unpickle_%s' % node.class_name + unpickle_func_name = '__pyx_unpickle_%s' % node.punycode_class_name # TODO(robertwb): Move the state into the third argument # so it can be pickled *after* self is memoized. @@ -1715,7 +2228,7 @@ if VALUE is not None: cdef object __pyx_result if __pyx_checksum not in %(checksums)s: from pickle import PickleError as __pyx_PickleError - raise __pyx_PickleError("Incompatible checksums (0x%%x vs %(checksums)s = (%(members)s))" %% __pyx_checksum) + raise __pyx_PickleError, "Incompatible checksums (0x%%x vs %(checksums)s = (%(members)s))" %% __pyx_checksum __pyx_result = %(class_name)s.__new__(__pyx_type) if __pyx_state is not None: %(unpickle_func_name)s__set_state(<%(class_name)s> __pyx_result, __pyx_state) @@ -1783,8 +2296,8 @@ if VALUE is not None: for decorator in old_decorators: func = decorator.decorator if (not func.is_name or - func.name not in ('staticmethod', 'classmethod') or - env.lookup_here(func.name)): + func.name not in ('staticmethod', 'classmethod') or + env.lookup_here(func.name)): # not a static or classmethod decorators.append(decorator) @@ -1802,8 +2315,10 @@ if VALUE is not None: "Handle def or cpdef fused functions" # Create PyCFunction nodes for each specialization node.stats.insert(0, node.py_func) - node.py_func = self.visit(node.py_func) + self.visitchild(node, 'py_func') node.update_fused_defnode_entry(env) + # For the moment, fused functions do not support METH_FASTCALL + node.py_func.entry.signature.use_fastcall = False pycfunc = ExprNodes.PyCFunctionNode.from_defnode(node.py_func, binding=True) pycfunc = ExprNodes.ProxyNode(pycfunc.coerce_to_temp(env)) node.resulting_fused_function = pycfunc @@ -1846,19 +2361,6 @@ if VALUE is not None: return node - def _handle_nogil_cleanup(self, lenv, node): - "Handle cleanup for 'with gil' blocks in nogil functions." - if lenv.nogil and lenv.has_with_gil_block: - # Acquire the GIL for cleanup in 'nogil' functions, by wrapping - # the entire function body in try/finally. - # The corresponding release will be taken care of by - # Nodes.FuncDefNode.generate_function_definitions() - node.body = Nodes.NogilTryFinallyStatNode( - node.body.pos, - body=node.body, - finally_clause=Nodes.EnsureGILNode(node.body.pos), - finally_except_clause=Nodes.EnsureGILNode(node.body.pos)) - def _handle_fused(self, node): if node.is_generator and node.has_fused_arguments: node.has_fused_arguments = False @@ -1890,6 +2392,8 @@ if VALUE is not None: for var, type_node in node.directive_locals.items(): if not lenv.lookup_here(var): # don't redeclare args type = type_node.analyse_as_type(lenv) + if type and type.is_fused and lenv.fused_to_specific: + type = type.specialize(lenv.fused_to_specific) if type: lenv.declare_var(var, type, type_node.pos) else: @@ -1899,17 +2403,18 @@ if VALUE is not None: node = self._create_fused_function(env, node) else: node.body.analyse_declarations(lenv) - self._handle_nogil_cleanup(lenv, node) self._super_visit_FuncDefNode(node) self.seen_vars_stack.pop() + + if "ufunc" in lenv.directives: + from . import UFuncs + return UFuncs.convert_to_ufunc(node) return node def visit_DefNode(self, node): node = self.visit_FuncDefNode(node) env = self.current_env() - if isinstance(node, Nodes.DefNode) and node.is_wrapper: - env = env.parent_scope if (not isinstance(node, Nodes.DefNode) or node.fused_py_func or node.is_generator_body or not node.needs_assignment_synthesis(env)): @@ -1959,11 +2464,17 @@ if VALUE is not None: assmt.analyse_declarations(env) return assmt + def visit_func_outer_attrs(self, node): + # any names in the outer attrs should not be looked up in the function "seen_vars_stack" + stack = self.seen_vars_stack.pop() + super(AnalyseDeclarationsTransform, self).visit_func_outer_attrs(node) + self.seen_vars_stack.append(stack) + def visit_ScopedExprNode(self, node): env = self.current_env() node.analyse_declarations(env) # the node may or may not have a local scope - if node.has_local_scope: + if node.expr_scope: self.seen_vars_stack.append(set(self.seen_vars_stack[-1])) self.enter_scope(node, node.expr_scope) node.analyse_scoped_declarations(node.expr_scope) @@ -1971,6 +2482,7 @@ if VALUE is not None: self.exit_scope() self.seen_vars_stack.pop() else: + node.analyse_scoped_declarations(env) self.visitchildren(node) return node @@ -1993,7 +2505,7 @@ if VALUE is not None: # (so it can't happen later). # Note that we don't return the original node, as it is # never used after this phase. - if True: # private (default) + if True: # private (default) return None self_value = ExprNodes.AttributeNode( @@ -2085,8 +2597,8 @@ if VALUE is not None: if node.name in self.seen_vars_stack[-1]: entry = self.current_env().lookup(node.name) if (entry is None or entry.visibility != 'extern' - and not entry.scope.is_c_class_scope): - warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2) + and not entry.scope.is_c_class_scope): + error(node.pos, "cdef variable '%s' declared after it is used" % node.name) self.visitchildren(node) return node @@ -2096,13 +2608,12 @@ if VALUE is not None: return None def visit_CnameDecoratorNode(self, node): - child_node = self.visit(node.node) + child_node = self.visitchild(node, 'node') if not child_node: return None - if type(child_node) is list: # Assignment synthesized - node.child_node = child_node[0] + if type(child_node) is list: # Assignment synthesized + node.node = child_node[0] return [node] + child_node[1:] - node.node = child_node return node def create_Property(self, entry): @@ -2122,6 +2633,11 @@ if VALUE is not None: property.doc = entry.doc return property + def visit_AssignmentExpressionNode(self, node): + self.visitchildren(node) + node.analyse_declarations(self.current_env()) + return node + def _calculate_pickle_checksums(member_names): # Cython 0.x used MD5 for the checksum, which a few Python installations remove for security reasons. @@ -2130,7 +2646,7 @@ def _calculate_pickle_checksums(member_names): member_names_string = ' '.join(member_names).encode('utf-8') hash_kwargs = {'usedforsecurity': False} if sys.version_info >= (3, 9) else {} checksums = [] - for algo_name in ['md5', 'sha256', 'sha1']: + for algo_name in ['sha256', 'sha1', 'md5']: try: mkchecksum = getattr(hashlib, algo_name) checksum = mkchecksum(member_names_string, **hash_kwargs).hexdigest() @@ -2219,7 +2735,7 @@ class CalculateQualifiedNamesTransform(EnvTransform): def visit_ClassDefNode(self, node): orig_qualified_name = self.qualified_name[:] entry = (getattr(node, 'entry', None) or # PyClass - self.current_env().lookup_here(node.name)) # CClass + self.current_env().lookup_here(node.target.name)) # CClass self._append_entry(entry) self._super_visit_ClassDefNode(node) self.qualified_name = orig_qualified_name @@ -2328,8 +2844,8 @@ class ExpandInplaceOperators(EnvTransform): operand2 = rhs, inplace=True) # Manually analyse types for new node. - lhs.analyse_target_types(env) - dup.analyse_types(env) + lhs = lhs.analyse_target_types(env) + dup.analyse_types(env) # FIXME: no need to reanalyse the copy, right? binop.analyse_operation(env) node = Nodes.SingleAssignmentNode( node.pos, @@ -2379,13 +2895,15 @@ class AdjustDefByDirectives(CythonTransform, SkipDeclarations): return_type_node = self.directives.get('returns') if return_type_node is None and self.directives['annotation_typing']: return_type_node = node.return_type_annotation - # for Python anntations, prefer safe exception handling by default + # for Python annotations, prefer safe exception handling by default if return_type_node is not None and except_val is None: except_val = (None, True) # except * elif except_val is None: - # backward compatible default: no exception check - except_val = (None, False) + # backward compatible default: no exception check, unless there's also a "@returns" declaration + except_val = (None, True if return_type_node else False) if 'ccall' in self.directives: + if 'cfunc' in self.directives: + error(node.pos, "cfunc and ccall directives cannot be combined") node = node.as_cfunction( overridable=True, modifiers=modifiers, nogil=nogil, returns=return_type_node, except_val=except_val) @@ -2437,8 +2955,6 @@ class AlignFunctionDefinitions(CythonTransform): def visit_ModuleNode(self, node): self.scope = node.scope - self.directives = node.directives - self.imported_names = set() # hack, see visit_FromImportStatNode() self.visitchildren(node) return node @@ -2476,15 +2992,45 @@ class AlignFunctionDefinitions(CythonTransform): error(pxd_def.pos, "previous declaration here") return None node = node.as_cfunction(pxd_def) - elif (self.scope.is_module_scope and self.directives['auto_cpdef'] - and not node.name in self.imported_names - and node.is_cdef_func_compatible()): - # FIXME: cpdef-ing should be done in analyse_declarations() - node = node.as_cfunction(scope=self.scope) # Enable this when nested cdef functions are allowed. # self.visitchildren(node) return node + def visit_ExprNode(self, node): + # ignore lambdas and everything else that appears in expressions + return node + + +class AutoCpdefFunctionDefinitions(CythonTransform): + + def visit_ModuleNode(self, node): + self.directives = node.directives + self.imported_names = set() # hack, see visit_FromImportStatNode() + self.scope = node.scope + self.visitchildren(node) + return node + + def visit_DefNode(self, node): + if (self.scope.is_module_scope and self.directives['auto_cpdef'] + and node.name not in self.imported_names + and node.is_cdef_func_compatible()): + # FIXME: cpdef-ing should be done in analyse_declarations() + node = node.as_cfunction(scope=self.scope) + return node + + def visit_CClassDefNode(self, node, pxd_def=None): + if pxd_def is None: + pxd_def = self.scope.lookup(node.class_name) + if pxd_def: + if not pxd_def.defined_in_pxd: + return node + outer_scope = self.scope + self.scope = pxd_def.type.scope + self.visitchildren(node) + if pxd_def: + self.scope = outer_scope + return node + def visit_FromImportStatNode(self, node): # hack to prevent conditional import fallback functions from # being cdpef-ed (global Python variables currently conflict @@ -2504,8 +3050,7 @@ class RemoveUnreachableCode(CythonTransform): if not self.current_directives['remove_unreachable']: return node self.visitchildren(node) - for idx, stat in enumerate(node.stats): - idx += 1 + for idx, stat in enumerate(node.stats, 1): if stat.is_terminator: if idx < len(node.stats): if self.current_directives['warn.unreachable']: @@ -2604,6 +3149,8 @@ class YieldNodeCollector(TreeVisitor): class MarkClosureVisitor(CythonTransform): + # In addition to marking closures this is also responsible to finding parts of the + # generator iterable and marking them def visit_ModuleNode(self, node): self.needs_closure = False @@ -2649,7 +3196,8 @@ class MarkClosureVisitor(CythonTransform): star_arg=node.star_arg, starstar_arg=node.starstar_arg, doc=node.doc, decorators=node.decorators, gbody=gbody, lambda_name=node.lambda_name, - return_type_annotation=node.return_type_annotation) + return_type_annotation=node.return_type_annotation, + is_generator_expression=node.is_generator_expression) return coroutine def visit_CFuncDefNode(self, node): @@ -2673,6 +3221,19 @@ class MarkClosureVisitor(CythonTransform): self.needs_closure = True return node + def visit_GeneratorExpressionNode(self, node): + node = self.visit_LambdaNode(node) + if not isinstance(node.loop, Nodes._ForInStatNode): + # Possibly should handle ForFromStatNode + # but for now do nothing + return node + itseq = node.loop.iterator.sequence + # literals do not need replacing with an argument + if itseq.is_literal: + return node + _GeneratorExpressionArgumentsMarker(node).visit(itseq) + return node + class CreateClosureClasses(CythonTransform): # Output closure classes in module scope for all functions @@ -2726,7 +3287,7 @@ class CreateClosureClasses(CythonTransform): if not node.py_cfunc_node: raise InternalError("DefNode does not have assignment node") inner_node = node.py_cfunc_node - inner_node.needs_self_code = False + inner_node.needs_closure_code = False node.needs_outer_scope = False if node.is_generator: @@ -2745,6 +3306,7 @@ class CreateClosureClasses(CythonTransform): as_name = '%s_%s' % ( target_module_scope.next_id(Naming.closure_class_prefix), node.entry.cname.replace('.','__')) + as_name = EncodedString(as_name) entry = target_module_scope.declare_c_class( name=as_name, pos=node.pos, defining=True, @@ -2816,6 +3378,10 @@ class CreateClosureClasses(CythonTransform): self.visitchildren(node) return node + def visit_GeneratorExpressionNode(self, node): + node = _HandleGeneratorArguments()(node) + return self.visit_LambdaNode(node) + class InjectGilHandling(VisitorTransform, SkipDeclarations): """ @@ -2824,20 +3390,20 @@ class InjectGilHandling(VisitorTransform, SkipDeclarations): Must run before the AnalyseDeclarationsTransform to make sure the GILStatNodes get set up, parallel sections know that the GIL is acquired inside of them, etc. """ - def __call__(self, root): - self.nogil = False - return super(InjectGilHandling, self).__call__(root) + nogil = False # special node handling - def visit_RaiseStatNode(self, node): - """Allow raising exceptions in nogil sections by wrapping them in a 'with gil' block.""" + def _inject_gil_in_nogil(self, node): + """Allow the (Python statement) node in nogil sections by wrapping it in a 'with gil' block.""" if self.nogil: node = Nodes.GILStatNode(node.pos, state='gil', body=node) return node + visit_RaiseStatNode = _inject_gil_in_nogil + visit_PrintStatNode = _inject_gil_in_nogil # sadly, not the function + # further candidates: - # def visit_AssertStatNode(self, node): # def visit_ReraiseStatNode(self, node): # nogil tracking @@ -2905,6 +3471,7 @@ class GilCheck(VisitorTransform): self.env_stack.append(node.local_scope) inner_nogil = node.local_scope.nogil + nogil_declarator_only = self.nogil_declarator_only if inner_nogil: self.nogil_declarator_only = True @@ -2913,13 +3480,20 @@ class GilCheck(VisitorTransform): self._visit_scoped_children(node, inner_nogil) - # This cannot be nested, so it doesn't need backup/restore - self.nogil_declarator_only = False + # FuncDefNodes can be nested, because a cpdef function contains a def function + # inside it. Therefore restore to previous state + self.nogil_declarator_only = nogil_declarator_only self.env_stack.pop() return node def visit_GILStatNode(self, node): + if node.condition is not None: + error(node.condition.pos, + "Non-constant condition in a " + "`with %s(<condition>)` statement" % node.state) + return node + if self.nogil and node.nogil_check: node.nogil_check() @@ -2933,6 +3507,8 @@ class GilCheck(VisitorTransform): else: error(node.pos, "Trying to release the GIL while it was " "previously released.") + if self.nogil_declarator_only: + node.scope_gil_state_known = False if isinstance(node.finally_clause, Nodes.StatListNode): # The finally clause of the GILStatNode is a GILExitNode, @@ -2983,6 +3559,12 @@ class GilCheck(VisitorTransform): self.visitchildren(node) return node + def visit_GILExitNode(self, node): + if self.nogil_declarator_only: + node.scope_gil_state_known = False + self.visitchildren(node) + return node + def visit_Node(self, node): if self.env_stack and self.nogil and node.nogil_check: node.nogil_check(self.env_stack[-1]) @@ -2995,6 +3577,32 @@ class GilCheck(VisitorTransform): return node +class CoerceCppTemps(EnvTransform, SkipDeclarations): + """ + For temporary expression that are implemented using std::optional it's necessary the temps are + assigned using `__pyx_t_x = value;` but accessed using `something = (*__pyx_t_x)`. This transform + inserts a coercion node to take care of this, and runs absolutely last (once nothing else can be + inserted into the tree) + + TODO: a possible alternative would be to split ExprNode.result() into ExprNode.rhs_rhs() and ExprNode.lhs_rhs()??? + """ + def visit_ModuleNode(self, node): + if self.current_env().cpp: + # skipping this makes it essentially free for C files + self.visitchildren(node) + return node + + def visit_ExprNode(self, node): + self.visitchildren(node) + if (self.current_env().directives['cpp_locals'] and + node.is_temp and node.type.is_cpp_class and + # Fake references are not replaced with "std::optional()". + not node.type.is_fake_reference): + node = ExprNodes.CppOptionalTempCoercion(node) + + return node + + class TransformBuiltinMethods(EnvTransform): """ Replace Cython's own cython.* builtins by the corresponding tree nodes. @@ -3017,9 +3625,7 @@ class TransformBuiltinMethods(EnvTransform): def visit_cython_attribute(self, node): attribute = node.as_cython_attribute() if attribute: - if attribute == u'compiled': - node = ExprNodes.BoolNode(node.pos, value=True) - elif attribute == u'__version__': + if attribute == u'__version__': from .. import __version__ as version node = ExprNodes.StringNode(node.pos, value=EncodedString(version)) elif attribute == u'NULL': @@ -3064,9 +3670,9 @@ class TransformBuiltinMethods(EnvTransform): error(self.pos, "Builtin 'vars()' called with wrong number of args, expected 0-1, got %d" % len(node.args)) if len(node.args) > 0: - return node # nothing to do + return node # nothing to do return ExprNodes.LocalsExprNode(pos, self.current_scope_node(), lenv) - else: # dir() + else: # dir() if len(node.args) > 1: error(self.pos, "Builtin 'dir()' called with wrong number of args, expected 0-1, got %d" % len(node.args)) @@ -3101,8 +3707,8 @@ class TransformBuiltinMethods(EnvTransform): def _inject_eval(self, node, func_name): lenv = self.current_env() - entry = lenv.lookup_here(func_name) - if entry or len(node.args) != 1: + entry = lenv.lookup(func_name) + if len(node.args) != 1 or (entry and not entry.is_builtin): return node # Inject globals and locals node.args.append(ExprNodes.GlobalsExprNode(node.pos)) @@ -3119,8 +3725,7 @@ class TransformBuiltinMethods(EnvTransform): return node # Inject no-args super def_node = self.current_scope_node() - if (not isinstance(def_node, Nodes.DefNode) or not def_node.args or - len(self.env_stack) < 2): + if not isinstance(def_node, Nodes.DefNode) or not def_node.args or len(self.env_stack) < 2: return node class_node, class_scope = self.env_stack[-2] if class_scope.is_py_class_scope: @@ -3259,10 +3864,17 @@ class ReplaceFusedTypeChecks(VisitorTransform): self.visitchildren(node) return self.transform(node) + def visit_GILStatNode(self, node): + """ + Fold constant condition of GILStatNode. + """ + self.visitchildren(node) + return self.transform(node) + def visit_PrimaryCmpNode(self, node): with Errors.local_errors(ignore=True): - type1 = node.operand1.analyse_as_type(self.local_scope) - type2 = node.operand2.analyse_as_type(self.local_scope) + type1 = node.operand1.analyse_as_type(self.local_scope) + type2 = node.operand2.analyse_as_type(self.local_scope) if type1 and type2: false_node = ExprNodes.BoolNode(node.pos, value=False) @@ -3398,9 +4010,14 @@ class DebugTransform(CythonTransform): else: pf_cname = node.py_func.entry.func_cname + # For functions defined using def, cname will be pyfunc_cname=__pyx_pf_* + # For functions defined using cpdef or cdef, cname will be func_cname=__pyx_f_* + # In all cases, cname will be the name of the function containing the actual code + cname = node.entry.pyfunc_cname or node.entry.func_cname + attrs = dict( name=node.entry.name or getattr(node, 'name', '<unknown>'), - cname=node.entry.func_cname, + cname=cname, pf_cname=pf_cname, qualified_name=node.local_scope.qualified_name, lineno=str(node.pos[1])) @@ -3428,10 +4045,10 @@ class DebugTransform(CythonTransform): def visit_NameNode(self, node): if (self.register_stepinto and - node.type is not None and - node.type.is_cfunction and - getattr(node, 'is_called', False) and - node.entry.func_cname is not None): + node.type is not None and + node.type.is_cfunction and + getattr(node, 'is_called', False) and + node.entry.func_cname is not None): # don't check node.entry.in_cinclude, as 'cdef extern: ...' # declared functions are not 'in_cinclude'. # This means we will list called 'cdef' functions as @@ -3450,26 +4067,16 @@ class DebugTransform(CythonTransform): it's a "relevant frame" and it will know where to set the breakpoint for 'break modulename'. """ - name = node.full_module_name.rpartition('.')[-1] - - cname_py2 = 'init' + name - cname_py3 = 'PyInit_' + name - - py2_attrs = dict( - name=name, - cname=cname_py2, + self._serialize_modulenode_as_function(node, dict( + name=node.full_module_name.rpartition('.')[-1], + cname=node.module_init_func_cname(), pf_cname='', # Ignore the qualified_name, breakpoints should be set using # `cy break modulename:lineno` for module-level breakpoints. qualified_name='', lineno='1', is_initmodule_function="True", - ) - - py3_attrs = dict(py2_attrs, cname=cname_py3) - - self._serialize_modulenode_as_function(node, py2_attrs) - self._serialize_modulenode_as_function(node, py3_attrs) + )) def _serialize_modulenode_as_function(self, node, attrs): self.tb.start('Function', attrs=attrs) diff --git a/Cython/Compiler/Parsing.pxd b/Cython/Compiler/Parsing.pxd index 25453b39a..72a855fd4 100644 --- a/Cython/Compiler/Parsing.pxd +++ b/Cython/Compiler/Parsing.pxd @@ -1,3 +1,5 @@ +# cython: language_level=3 + # We declare all of these here to type the first argument. from __future__ import absolute_import @@ -19,16 +21,17 @@ cdef p_ident_list(PyrexScanner s) cdef tuple p_binop_operator(PyrexScanner s) cdef p_binop_expr(PyrexScanner s, ops, p_sub_expr_func p_sub_expr) -cdef p_lambdef(PyrexScanner s, bint allow_conditional=*) -cdef p_lambdef_nocond(PyrexScanner s) +cdef p_lambdef(PyrexScanner s) cdef p_test(PyrexScanner s) -cdef p_test_nocond(PyrexScanner s) +cdef p_test_allow_walrus_after(PyrexScanner s) +cdef p_namedexpr_test(PyrexScanner s) cdef p_or_test(PyrexScanner s) -cdef p_rassoc_binop_expr(PyrexScanner s, ops, p_sub_expr_func p_subexpr) +cdef p_rassoc_binop_expr(PyrexScanner s, unicode op, p_sub_expr_func p_subexpr) cdef p_and_test(PyrexScanner s) cdef p_not_test(PyrexScanner s) cdef p_comparison(PyrexScanner s) cdef p_test_or_starred_expr(PyrexScanner s) +cdef p_namedexpr_test_or_starred_expr(PyrexScanner s) cdef p_starred_expr(PyrexScanner s) cdef p_cascaded_cmp(PyrexScanner s) cdef p_cmp_op(PyrexScanner s) @@ -82,6 +85,7 @@ cdef p_dict_or_set_maker(PyrexScanner s) cdef p_backquote_expr(PyrexScanner s) cdef p_simple_expr_list(PyrexScanner s, expr=*) cdef p_test_or_starred_expr_list(PyrexScanner s, expr=*) +cdef p_namedexpr_test_or_starred_expr_list(s, expr=*) cdef p_testlist(PyrexScanner s) cdef p_testlist_star_expr(PyrexScanner s) cdef p_testlist_comp(PyrexScanner s) @@ -106,7 +110,7 @@ cdef p_return_statement(PyrexScanner s) cdef p_raise_statement(PyrexScanner s) cdef p_import_statement(PyrexScanner s) cdef p_from_import_statement(PyrexScanner s, bint first_statement = *) -cdef p_imported_name(PyrexScanner s, bint is_cimport) +cdef p_imported_name(PyrexScanner s) cdef p_dotted_name(PyrexScanner s, bint as_allowed) cdef p_as_name(PyrexScanner s) cdef p_assert_statement(PyrexScanner s) @@ -126,6 +130,8 @@ cdef p_except_clause(PyrexScanner s) cdef p_include_statement(PyrexScanner s, ctx) cdef p_with_statement(PyrexScanner s) cdef p_with_items(PyrexScanner s, bint is_async=*) +cdef p_with_items_list(PyrexScanner s, bint is_async) +cdef tuple p_with_item(PyrexScanner s, bint is_async) cdef p_with_template(PyrexScanner s) cdef p_simple_statement(PyrexScanner s, bint first_statement = *) cdef p_simple_statement_list(PyrexScanner s, ctx, bint first_statement = *) @@ -139,10 +145,10 @@ cdef tuple p_suite_with_docstring(PyrexScanner s, ctx, bint with_doc_only=*) cdef tuple _extract_docstring(node) cdef p_positional_and_keyword_args(PyrexScanner s, end_sy_set, templates = *) -cpdef p_c_base_type(PyrexScanner s, bint self_flag = *, bint nonempty = *, templates = *) +cpdef p_c_base_type(PyrexScanner s, bint nonempty = *, templates = *) cdef p_calling_convention(PyrexScanner s) cdef p_c_complex_base_type(PyrexScanner s, templates = *) -cdef p_c_simple_base_type(PyrexScanner s, bint self_flag, bint nonempty, templates = *) +cdef p_c_simple_base_type(PyrexScanner s, bint nonempty, templates = *) cdef p_buffer_or_template(PyrexScanner s, base_type_node, templates) cdef p_bracketed_base_type(PyrexScanner s, base_type_node, nonempty, empty) cdef is_memoryviewslice_access(PyrexScanner s) @@ -151,7 +157,6 @@ cdef bint looking_at_name(PyrexScanner s) except -2 cdef object looking_at_expr(PyrexScanner s)# except -2 cdef bint looking_at_base_type(PyrexScanner s) except -2 cdef bint looking_at_dotted_name(PyrexScanner s) except -2 -cdef bint looking_at_call(PyrexScanner s) except -2 cdef p_sign_and_longness(PyrexScanner s) cdef p_opt_cname(PyrexScanner s) cpdef p_c_declarator(PyrexScanner s, ctx = *, bint empty = *, bint is_type = *, bint cmethod_flag = *, @@ -163,7 +168,7 @@ cdef p_c_simple_declarator(PyrexScanner s, ctx, bint empty, bint is_type, bint c bint assignable, bint nonempty) cdef p_nogil(PyrexScanner s) cdef p_with_gil(PyrexScanner s) -cdef p_exception_value_clause(PyrexScanner s) +cdef p_exception_value_clause(PyrexScanner s, ctx) cpdef p_c_arg_list(PyrexScanner s, ctx = *, bint in_pyfunc = *, bint cmethod_flag = *, bint nonempty_declarators = *, bint kw_only = *, bint annotated = *) cdef p_optional_ellipsis(PyrexScanner s) @@ -197,3 +202,4 @@ cdef dict p_compiler_directive_comments(PyrexScanner s) cdef p_template_definition(PyrexScanner s) cdef p_cpp_class_definition(PyrexScanner s, pos, ctx) cdef p_cpp_class_attribute(PyrexScanner s, ctx) +cdef p_annotation(PyrexScanner s) diff --git a/Cython/Compiler/Parsing.py b/Cython/Compiler/Parsing.py index 1f20b4c95..d7394ca6f 100644 --- a/Cython/Compiler/Parsing.py +++ b/Cython/Compiler/Parsing.py @@ -14,7 +14,7 @@ cython.declare(Nodes=object, ExprNodes=object, EncodedString=object, Builtin=object, ModuleNode=object, Utils=object, _unicode=object, _bytes=object, re=object, sys=object, _parse_escape_sequences=object, _parse_escape_sequences_raw=object, partial=object, reduce=object, _IS_PY3=cython.bint, _IS_2BYTE_UNICODE=cython.bint, - _CDEF_MODIFIERS=tuple) + _CDEF_MODIFIERS=tuple, COMMON_BINOP_MISTAKES=dict) from io import StringIO import re @@ -22,7 +22,7 @@ import sys from unicodedata import lookup as lookup_unicodechar, category as unicode_category from functools import partial, reduce -from .Scanning import PyrexScanner, FileSourceDescriptor, StringSourceDescriptor +from .Scanning import PyrexScanner, FileSourceDescriptor, tentatively_scan from . import Nodes from . import ExprNodes from . import Builtin @@ -65,7 +65,7 @@ class Ctx(object): def p_ident(s, message="Expected an identifier"): if s.sy == 'IDENT': - name = s.systring + name = s.context.intern_ustring(s.systring) s.next() return name else: @@ -74,7 +74,7 @@ def p_ident(s, message="Expected an identifier"): def p_ident_list(s): names = [] while s.sy == 'IDENT': - names.append(s.systring) + names.append(s.context.intern_ustring(s.systring)) s.next() if s.sy != ',': break @@ -103,12 +103,12 @@ def p_binop_expr(s, ops, p_sub_expr): if Future.division in s.context.future_directives: n1.truedivision = True else: - n1.truedivision = None # unknown + n1.truedivision = None # unknown return n1 #lambdef: 'lambda' [varargslist] ':' test -def p_lambdef(s, allow_conditional=True): +def p_lambdef(s): # s.sy == 'lambda' pos = s.position() s.next() @@ -119,23 +119,27 @@ def p_lambdef(s, allow_conditional=True): args, star_arg, starstar_arg = p_varargslist( s, terminator=':', annotated=False) s.expect(':') - if allow_conditional: - expr = p_test(s) - else: - expr = p_test_nocond(s) + expr = p_test(s) return ExprNodes.LambdaNode( pos, args = args, star_arg = star_arg, starstar_arg = starstar_arg, result_expr = expr) -#lambdef_nocond: 'lambda' [varargslist] ':' test_nocond - -def p_lambdef_nocond(s): - return p_lambdef(s, allow_conditional=False) - #test: or_test ['if' or_test 'else' test] | lambdef def p_test(s): + # The check for a following ':=' is only for error reporting purposes. + # It simply changes a + # expected ')', found ':=' + # message into something a bit more descriptive. + # It is close to what the PEG parser does in CPython, where an expression has + # a lookahead assertion that it isn't followed by ':=' + expr = p_test_allow_walrus_after(s) + if s.sy == ':=': + s.error("invalid syntax: assignment expression not allowed in this context") + return expr + +def p_test_allow_walrus_after(s): if s.sy == 'lambda': return p_lambdef(s) pos = s.position() @@ -149,34 +153,52 @@ def p_test(s): else: return expr -#test_nocond: or_test | lambdef_nocond +def p_namedexpr_test(s): + # defined in the LL parser as + # namedexpr_test: test [':=' test] + # The requirement that the LHS is a name is not enforced in the grammar. + # For comparison the PEG parser does: + # 1. look for "name :=", if found it's definitely a named expression + # so look for expression + # 2. Otherwise, look for expression + lhs = p_test_allow_walrus_after(s) + if s.sy == ':=': + position = s.position() + if not lhs.is_name: + s.error("Left-hand side of assignment expression must be an identifier", fatal=False) + s.next() + rhs = p_test(s) + return ExprNodes.AssignmentExpressionNode(position, lhs=lhs, rhs=rhs) + return lhs -def p_test_nocond(s): - if s.sy == 'lambda': - return p_lambdef_nocond(s) - else: - return p_or_test(s) #or_test: and_test ('or' and_test)* +COMMON_BINOP_MISTAKES = {'||': 'or', '&&': 'and'} + def p_or_test(s): - return p_rassoc_binop_expr(s, ('or',), p_and_test) + return p_rassoc_binop_expr(s, u'or', p_and_test) -def p_rassoc_binop_expr(s, ops, p_subexpr): +def p_rassoc_binop_expr(s, op, p_subexpr): n1 = p_subexpr(s) - if s.sy in ops: + if s.sy == op: pos = s.position() op = s.sy s.next() - n2 = p_rassoc_binop_expr(s, ops, p_subexpr) + n2 = p_rassoc_binop_expr(s, op, p_subexpr) n1 = ExprNodes.binop_node(pos, op, n1, n2) + elif s.sy in COMMON_BINOP_MISTAKES and COMMON_BINOP_MISTAKES[s.sy] == op: + # Only report this for the current operator since we pass through here twice for 'and' and 'or'. + warning(s.position(), + "Found the C operator '%s', did you mean the Python operator '%s'?" % (s.sy, op), + level=1) return n1 #and_test: not_test ('and' not_test)* def p_and_test(s): #return p_binop_expr(s, ('and',), p_not_test) - return p_rassoc_binop_expr(s, ('and',), p_not_test) + return p_rassoc_binop_expr(s, u'and', p_not_test) #not_test: 'not' not_test | comparison @@ -209,6 +231,12 @@ def p_test_or_starred_expr(s): else: return p_test(s) +def p_namedexpr_test_or_starred_expr(s): + if s.sy == '*': + return p_starred_expr(s) + else: + return p_namedexpr_test(s) + def p_starred_expr(s): pos = s.position() if s.sy == '*': @@ -250,10 +278,10 @@ def p_cmp_op(s): op = '!=' return op -comparison_ops = cython.declare(set, set([ +comparison_ops = cython.declare(frozenset, frozenset(( '<', '>', '==', '>=', '<=', '<>', '!=', 'in', 'is', 'not' -])) +))) #expr: xor_expr ('|' xor_expr)* @@ -316,10 +344,12 @@ def p_typecast(s): s.next() base_type = p_c_base_type(s) is_memslice = isinstance(base_type, Nodes.MemoryViewSliceTypeNode) - is_template = isinstance(base_type, Nodes.TemplatedTypeNode) - is_const = isinstance(base_type, Nodes.CConstTypeNode) - if (not is_memslice and not is_template and not is_const - and base_type.name is None): + is_other_unnamed_type = isinstance(base_type, ( + Nodes.TemplatedTypeNode, + Nodes.CConstOrVolatileTypeNode, + Nodes.CTupleBaseTypeNode, + )) + if not (is_memslice or is_other_unnamed_type) and base_type.name is None: s.error("Unknown type") declarator = p_c_declarator(s, empty = 1) if s.sy == '?': @@ -330,8 +360,7 @@ def p_typecast(s): s.expect(">") operand = p_factor(s) if is_memslice: - return ExprNodes.CythonArrayNode(pos, base_type_node=base_type, - operand=operand) + return ExprNodes.CythonArrayNode(pos, base_type_node=base_type, operand=operand) return ExprNodes.TypecastNode(pos, base_type = base_type, @@ -444,7 +473,7 @@ def p_trailer(s, node1): return p_call(s, node1) elif s.sy == '[': return p_index(s, node1) - else: # s.sy == '.' + else: # s.sy == '.' s.next() name = p_ident(s) return ExprNodes.AttributeNode(pos, @@ -480,7 +509,7 @@ def p_call_parse_args(s, allow_genexp=True): keyword_args.append(p_test(s)) starstar_seen = True else: - arg = p_test(s) + arg = p_namedexpr_test(s) if s.sy == '=': s.next() if not arg.is_name: @@ -628,9 +657,7 @@ def p_slice_element(s, follow_set): return None def expect_ellipsis(s): - s.expect('.') - s.expect('.') - s.expect('.') + s.expect('...') def make_slice_nodes(pos, subscripts): # Convert a list of subscripts as returned @@ -676,7 +703,7 @@ def p_atom(s): return p_dict_or_set_maker(s) elif sy == '`': return p_backquote_expr(s) - elif sy == '.': + elif sy == '...': expect_ellipsis(s) return ExprNodes.EllipsisNode(pos) elif sy == 'INT': @@ -821,7 +848,7 @@ def p_cat_string_literal(s): continue elif next_kind != kind: # concatenating f strings and normal strings is allowed and leads to an f string - if set([kind, next_kind]) in (set(['f', 'u']), set(['f', ''])): + if {kind, next_kind} in ({'f', 'u'}, {'f', ''}): kind = 'f' else: error(pos, "Cannot mix string literals of different types, expected %s'', got %s''" % ( @@ -1073,8 +1100,8 @@ def p_f_string(s, unicode_value, pos, is_raw): if builder.chars: values.append(ExprNodes.UnicodeNode(pos, value=builder.getstring())) builder = StringEncoding.UnicodeLiteralBuilder() - next_start, expr_node = p_f_string_expr(s, unicode_value, pos, next_start, is_raw) - values.append(expr_node) + next_start, expr_nodes = p_f_string_expr(s, unicode_value, pos, next_start, is_raw) + values.extend(expr_nodes) elif c == '}': if part == '}}': builder.append('}') @@ -1090,12 +1117,16 @@ def p_f_string(s, unicode_value, pos, is_raw): def p_f_string_expr(s, unicode_value, pos, starting_index, is_raw): - # Parses a {}-delimited expression inside an f-string. Returns a FormattedValueNode - # and the index in the string that follows the expression. + # Parses a {}-delimited expression inside an f-string. Returns a list of nodes + # [UnicodeNode?, FormattedValueNode] and the index in the string that follows + # the expression. + # + # ? = Optional i = starting_index size = len(unicode_value) conversion_char = terminal_char = format_spec = None format_spec_str = None + expr_text = None NO_CHAR = 2**30 nested_depth = 0 @@ -1135,12 +1166,15 @@ def p_f_string_expr(s, unicode_value, pos, starting_index, is_raw): elif c == '#': error(_f_string_error_pos(pos, unicode_value, i), "format string cannot include #") - elif nested_depth == 0 and c in '!:}': - # allow != as a special case - if c == '!' and i + 1 < size and unicode_value[i + 1] == '=': - i += 1 - continue - + elif nested_depth == 0 and c in '><=!:}': + # allow special cases with '!' and '=' + if i + 1 < size and c in '!=><': + if unicode_value[i + 1] == '=': + i += 2 # we checked 2, so we can skip 2: '!=', '==', '>=', '<=' + continue + elif c in '><': # allow single '<' and '>' + i += 1 + continue terminal_char = c break i += 1 @@ -1153,6 +1187,16 @@ def p_f_string_expr(s, unicode_value, pos, starting_index, is_raw): error(_f_string_error_pos(pos, unicode_value, starting_index), "empty expression not allowed in f-string") + if terminal_char == '=': + i += 1 + while i < size and unicode_value[i].isspace(): + i += 1 + + if i < size: + terminal_char = unicode_value[i] + expr_text = unicode_value[starting_index:i] + # otherwise: error will be reported below + if terminal_char == '!': i += 1 if i + 2 > size: @@ -1190,6 +1234,9 @@ def p_f_string_expr(s, unicode_value, pos, starting_index, is_raw): format_spec_str = unicode_value[start_format_spec:i] + if expr_text and conversion_char is None and format_spec_str is None: + conversion_char = 'r' + if terminal_char != '}': error(_f_string_error_pos(pos, unicode_value, i), "missing '}' in format string expression" + ( @@ -1208,13 +1255,17 @@ def p_f_string_expr(s, unicode_value, pos, starting_index, is_raw): if format_spec_str: format_spec = ExprNodes.JoinedStrNode(pos, values=p_f_string(s, format_spec_str, pos, is_raw)) - return i + 1, ExprNodes.FormattedValueNode( - pos, value=expr, conversion_char=conversion_char, format_spec=format_spec) + nodes = [] + if expr_text: + nodes.append(ExprNodes.UnicodeNode(pos, value=StringEncoding.EncodedString(expr_text))) + nodes.append(ExprNodes.FormattedValueNode(pos, value=expr, conversion_char=conversion_char, format_spec=format_spec)) + + return i + 1, nodes # since PEP 448: # list_display ::= "[" [listmaker] "]" -# listmaker ::= (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) +# listmaker ::= (named_test|star_expr) ( comp_for | (',' (named_test|star_expr))* [','] ) # comp_iter ::= comp_for | comp_if # comp_for ::= ["async"] "for" expression_list "in" testlist [comp_iter] # comp_if ::= "if" test [comp_iter] @@ -1227,7 +1278,7 @@ def p_list_maker(s): s.expect(']') return ExprNodes.ListNode(pos, args=[]) - expr = p_test_or_starred_expr(s) + expr = p_namedexpr_test_or_starred_expr(s) if s.sy in ('for', 'async'): if expr.is_starred: s.error("iterable unpacking cannot be used in comprehension") @@ -1242,7 +1293,7 @@ def p_list_maker(s): # (merged) list literal if s.sy == ',': s.next() - exprs = p_test_or_starred_expr_list(s, expr) + exprs = p_namedexpr_test_or_starred_expr_list(s, expr) else: exprs = [expr] s.expect(']') @@ -1276,7 +1327,12 @@ def p_comp_if(s, body): # s.sy == 'if' pos = s.position() s.next() - test = p_test_nocond(s) + # Note that Python 3.9+ is actually more restrictive here and Cython now follows + # the Python 3.9+ behaviour: https://github.com/python/cpython/issues/86014 + # On Python <3.9 `[i for i in range(10) if lambda: i if True else 1]` was disallowed + # but `[i for i in range(10) if lambda: i]` was allowed. + # On Python >=3.9 they're both disallowed. + test = p_or_test(s) return Nodes.IfStatNode(pos, if_clauses = [Nodes.IfClauseNode(pos, condition = test, body = p_comp_iter(s, body))], @@ -1433,6 +1489,15 @@ def p_test_or_starred_expr_list(s, expr=None): s.next() return exprs +def p_namedexpr_test_or_starred_expr_list(s, expr=None): + exprs = expr is not None and [expr] or [] + while s.sy not in expr_terminators: + exprs.append(p_namedexpr_test_or_starred_expr(s)) + if s.sy != ',': + break + s.next() + return exprs + #testlist: test (',' test)* [','] @@ -1462,10 +1527,10 @@ def p_testlist_star_expr(s): def p_testlist_comp(s): pos = s.position() - expr = p_test_or_starred_expr(s) + expr = p_namedexpr_test_or_starred_expr(s) if s.sy == ',': s.next() - exprs = p_test_or_starred_expr_list(s, expr) + exprs = p_namedexpr_test_or_starred_expr_list(s, expr) return ExprNodes.TupleNode(pos, args = exprs) elif s.sy in ('for', 'async'): return p_genexp(s, expr) @@ -1478,8 +1543,8 @@ def p_genexp(s, expr): expr.pos, expr = ExprNodes.YieldExprNode(expr.pos, arg=expr))) return ExprNodes.GeneratorExpressionNode(expr.pos, loop=loop) -expr_terminators = cython.declare(set, set([ - ')', ']', '}', ':', '=', 'NEWLINE'])) +expr_terminators = cython.declare(frozenset, frozenset(( + ')', ']', '}', ':', '=', 'NEWLINE'))) #------------------------------------------------------- @@ -1505,15 +1570,19 @@ def p_nonlocal_statement(s): def p_expression_or_assignment(s): expr = p_testlist_star_expr(s) + has_annotation = False if s.sy == ':' and (expr.is_name or expr.is_subscript or expr.is_attribute): + has_annotation = True s.next() - expr.annotation = p_test(s) + expr.annotation = p_annotation(s) + if s.sy == '=' and expr.is_starred: # This is a common enough error to make when learning Cython to let # it fail as early as possible and give a very clear error message. s.error("a starred assignment target must be in a list or tuple" " - maybe you meant to use an index assignment: var[0] = ...", pos=expr.pos) + expr_list = [expr] while s.sy == '=': s.next() @@ -1545,7 +1614,7 @@ def p_expression_or_assignment(s): rhs = expr_list[-1] if len(expr_list) == 2: - return Nodes.SingleAssignmentNode(rhs.pos, lhs=expr_list[0], rhs=rhs) + return Nodes.SingleAssignmentNode(rhs.pos, lhs=expr_list[0], rhs=rhs, first=has_annotation) else: return Nodes.CascadedAssignmentNode(rhs.pos, lhs_list=expr_list[:-1], rhs=rhs) @@ -1690,11 +1759,6 @@ def p_import_statement(s): as_name=as_name, is_absolute=is_absolute) else: - if as_name and "." in dotted_name: - name_list = ExprNodes.ListNode(pos, args=[ - ExprNodes.IdentifierStringNode(pos, value=s.context.intern_ustring("*"))]) - else: - name_list = None stat = Nodes.SingleAssignmentNode( pos, lhs=ExprNodes.NameNode(pos, name=as_name or target_name), @@ -1702,7 +1766,8 @@ def p_import_statement(s): pos, module_name=ExprNodes.IdentifierStringNode(pos, value=dotted_name), level=0 if is_absolute else None, - name_list=name_list)) + get_top_level_module='.' in dotted_name and as_name is None, + name_list=None)) stats.append(stat) return Nodes.StatListNode(pos, stats=stats) @@ -1711,11 +1776,11 @@ def p_from_import_statement(s, first_statement = 0): # s.sy == 'from' pos = s.position() s.next() - if s.sy == '.': + if s.sy in ('.', '...'): # count relative import level level = 0 - while s.sy == '.': - level += 1 + while s.sy in ('.', '...'): + level += len(s.sy) s.next() else: level = None @@ -1734,18 +1799,18 @@ def p_from_import_statement(s, first_statement = 0): is_cimport = kind == 'cimport' is_parenthesized = False if s.sy == '*': - imported_names = [(s.position(), s.context.intern_ustring("*"), None, None)] + imported_names = [(s.position(), s.context.intern_ustring("*"), None)] s.next() else: if s.sy == '(': is_parenthesized = True s.next() - imported_names = [p_imported_name(s, is_cimport)] + imported_names = [p_imported_name(s)] while s.sy == ',': s.next() if is_parenthesized and s.sy == ')': break - imported_names.append(p_imported_name(s, is_cimport)) + imported_names.append(p_imported_name(s)) if is_parenthesized: s.expect(')') if dotted_name == '__future__': @@ -1754,7 +1819,7 @@ def p_from_import_statement(s, first_statement = 0): elif level: s.error("invalid syntax") else: - for (name_pos, name, as_name, kind) in imported_names: + for (name_pos, name, as_name) in imported_names: if name == "braces": s.error("not a chance", name_pos) break @@ -1765,7 +1830,7 @@ def p_from_import_statement(s, first_statement = 0): break s.context.future_directives.add(directive) return Nodes.PassStatNode(pos) - elif kind == 'cimport': + elif is_cimport: return Nodes.FromCImportStatNode( pos, module_name=dotted_name, relative_level=level, @@ -1773,7 +1838,7 @@ def p_from_import_statement(s, first_statement = 0): else: imported_name_strings = [] items = [] - for (name_pos, name, as_name, kind) in imported_names: + for (name_pos, name, as_name) in imported_names: imported_name_strings.append( ExprNodes.IdentifierStringNode(name_pos, value=name)) items.append( @@ -1788,19 +1853,11 @@ def p_from_import_statement(s, first_statement = 0): items = items) -imported_name_kinds = cython.declare(set, set(['class', 'struct', 'union'])) - -def p_imported_name(s, is_cimport): +def p_imported_name(s): pos = s.position() - kind = None - if is_cimport and s.systring in imported_name_kinds: - kind = s.systring - warning(pos, 'the "from module cimport %s name" syntax is deprecated and ' - 'will be removed in Cython 3.0' % kind, 2) - s.next() name = p_ident(s) as_name = p_as_name(s) - return (pos, name, as_name, kind) + return (pos, name, as_name) def p_dotted_name(s, as_allowed): @@ -1834,10 +1891,11 @@ def p_assert_statement(s): value = p_test(s) else: value = None - return Nodes.AssertStatNode(pos, cond = cond, value = value) + return Nodes.AssertStatNode(pos, condition=cond, value=value) -statement_terminators = cython.declare(set, set([';', 'NEWLINE', 'EOF'])) +statement_terminators = cython.declare(frozenset, frozenset(( + ';', 'NEWLINE', 'EOF'))) def p_if_statement(s): # s.sy == 'if' @@ -1853,7 +1911,7 @@ def p_if_statement(s): def p_if_clause(s): pos = s.position() - test = p_test(s) + test = p_namedexpr_test(s) body = p_suite(s) return Nodes.IfClauseNode(pos, condition = test, body = body) @@ -1869,7 +1927,7 @@ def p_while_statement(s): # s.sy == 'while' pos = s.position() s.next() - test = p_test(s) + test = p_namedexpr_test(s) body = p_suite(s) else_clause = p_else_clause(s) return Nodes.WhileStatNode(pos, @@ -1947,7 +2005,8 @@ def p_for_from_step(s): else: return None -inequality_relations = cython.declare(set, set(['<', '<=', '>', '>='])) +inequality_relations = cython.declare(frozenset, frozenset(( + '<', '<=', '>', '>='))) def p_target(s, terminator): pos = s.position() @@ -2037,7 +2096,7 @@ def p_except_clause(s): def p_include_statement(s, ctx): pos = s.position() - s.next() # 'include' + s.next() # 'include' unicode_include_file_name = p_string_literal(s, 'u')[2] s.expect_newline("Syntax error in include statement") if s.compile_time_eval: @@ -2066,30 +2125,77 @@ def p_with_statement(s): def p_with_items(s, is_async=False): + """ + Copied from CPython: + | 'with' '(' a[asdl_withitem_seq*]=','.with_item+ ','? ')' ':' b=block { + _PyAST_With(a, b, NULL, EXTRA) } + | 'with' a[asdl_withitem_seq*]=','.with_item+ ':' tc=[TYPE_COMMENT] b=block { + _PyAST_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } + Therefore the first thing to try is the bracket-enclosed + version and if that fails try the regular version + """ + brackets_succeeded = False + items = () # unused, but static analysis fails to track that below + if s.sy == '(': + with tentatively_scan(s) as errors: + s.next() + items = p_with_items_list(s, is_async) + s.expect(")") + if s.sy != ":": + # Fail - the message doesn't matter because we'll try the + # non-bracket version so it'll never be shown + s.error("") + brackets_succeeded = not errors + if not brackets_succeeded: + # try the non-bracket version + items = p_with_items_list(s, is_async) + body = p_suite(s) + for cls, pos, kwds in reversed(items): + # construct the actual nodes now that we know what the body is + body = cls(pos, body=body, **kwds) + return body + + +def p_with_items_list(s, is_async): + items = [] + while True: + items.append(p_with_item(s, is_async)) + if s.sy != ",": + break + s.next() + if s.sy == ")": + # trailing commas allowed + break + return items + + +def p_with_item(s, is_async): + # In contrast to most parsing functions, this returns a tuple of + # class, pos, kwd_dict + # This is because GILStatNode does a reasonable amount of initialization in its + # constructor, and requires "body" to be set, which we don't currently have pos = s.position() if not s.in_python_file and s.sy == 'IDENT' and s.systring in ('nogil', 'gil'): if is_async: s.error("with gil/nogil cannot be async") state = s.systring s.next() - if s.sy == ',': + + # support conditional gil/nogil + condition = None + if s.sy == '(': s.next() - body = p_with_items(s) - else: - body = p_suite(s) - return Nodes.GILStatNode(pos, state=state, body=body) + condition = p_test(s) + s.expect(')') + + return Nodes.GILStatNode, pos, {"state": state, "condition": condition} else: manager = p_test(s) target = None if s.sy == 'IDENT' and s.systring == 'as': s.next() target = p_starred_expr(s) - if s.sy == ',': - s.next() - body = p_with_items(s, is_async=is_async) - else: - body = p_suite(s) - return Nodes.WithStatNode(pos, manager=manager, target=target, body=body, is_async=is_async) + return Nodes.WithStatNode, pos, {"manager": manager, "target": target, "is_async": is_async} def p_with_template(s): @@ -2195,7 +2301,7 @@ def p_compile_time_expr(s): def p_DEF_statement(s): pos = s.position() denv = s.compile_time_env - s.next() # 'DEF' + s.next() # 'DEF' name = p_ident(s) s.expect('=') expr = p_compile_time_expr(s) @@ -2213,7 +2319,7 @@ def p_IF_statement(s, ctx): denv = s.compile_time_env result = None while 1: - s.next() # 'IF' or 'ELIF' + s.next() # 'IF' or 'ELIF' expr = p_compile_time_expr(s) s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv)) body = p_suite(s, ctx) @@ -2243,8 +2349,16 @@ def p_statement(s, ctx, first_statement = 0): # error(s.position(), "'api' not allowed with 'ctypedef'") return p_ctypedef_statement(s, ctx) elif s.sy == 'DEF': + warning(s.position(), + "The 'DEF' statement is deprecated and will be removed in a future Cython version. " + "Consider using global variables, constants, and in-place literals instead. " + "See https://github.com/cython/cython/issues/4310", level=1) return p_DEF_statement(s) elif s.sy == 'IF': + warning(s.position(), + "The 'IF' statement is deprecated and will be removed in a future Cython version. " + "Consider using runtime conditions or C macros instead. " + "See https://github.com/cython/cython/issues/4310", level=1) return p_IF_statement(s, ctx) elif s.sy == '@': if ctx.level not in ('module', 'class', 'c_class', 'function', 'property', 'module_pxd', 'c_class_pxd', 'other'): @@ -2325,13 +2439,14 @@ def p_statement(s, ctx, first_statement = 0): else: if s.sy == 'IDENT' and s.systring == 'async': ident_name = s.systring + ident_pos = s.position() # PEP 492 enables the async/await keywords when it spots "async def ..." s.next() if s.sy == 'def': return p_async_statement(s, ctx, decorators) elif decorators: s.error("Decorators can only be followed by functions or classes") - s.put_back('IDENT', ident_name) # re-insert original token + s.put_back(u'IDENT', ident_name, ident_pos) # re-insert original token return p_simple_statement_list(s, ctx, first_statement=first_statement) @@ -2399,7 +2514,7 @@ def p_positional_and_keyword_args(s, end_sy_set, templates = None): parsed_type = False if s.sy == 'IDENT' and s.peek()[0] == '=': ident = s.systring - s.next() # s.sy is '=' + s.next() # s.sy is '=' s.next() if looking_at_expr(s): arg = p_test(s) @@ -2436,13 +2551,11 @@ def p_positional_and_keyword_args(s, end_sy_set, templates = None): s.next() return positional_args, keyword_args -def p_c_base_type(s, self_flag = 0, nonempty = 0, templates = None): - # If self_flag is true, this is the base type for the - # self argument of a C method of an extension type. +def p_c_base_type(s, nonempty=False, templates=None): if s.sy == '(': return p_c_complex_base_type(s, templates = templates) else: - return p_c_simple_base_type(s, self_flag, nonempty = nonempty, templates = templates) + return p_c_simple_base_type(s, nonempty=nonempty, templates=templates) def p_calling_convention(s): if s.sy == 'IDENT' and s.systring in calling_convention_words: @@ -2453,8 +2566,8 @@ def p_calling_convention(s): return "" -calling_convention_words = cython.declare( - set, set(["__stdcall", "__cdecl", "__fastcall"])) +calling_convention_words = cython.declare(frozenset, frozenset(( + "__stdcall", "__cdecl", "__fastcall"))) def p_c_complex_base_type(s, templates = None): @@ -2486,24 +2599,38 @@ def p_c_complex_base_type(s, templates = None): return type_node -def p_c_simple_base_type(s, self_flag, nonempty, templates = None): - #print "p_c_simple_base_type: self_flag =", self_flag, nonempty +def p_c_simple_base_type(s, nonempty, templates=None): is_basic = 0 signed = 1 longness = 0 complex = 0 module_path = [] pos = s.position() - if not s.sy == 'IDENT': - error(pos, "Expected an identifier, found '%s'" % s.sy) - if s.systring == 'const': + + # Handle const/volatile + is_const = is_volatile = 0 + while s.sy == 'IDENT': + if s.systring == 'const': + if is_const: error(pos, "Duplicate 'const'") + is_const = 1 + elif s.systring == 'volatile': + if is_volatile: error(pos, "Duplicate 'volatile'") + is_volatile = 1 + else: + break s.next() - base_type = p_c_base_type(s, self_flag=self_flag, nonempty=nonempty, templates=templates) + if is_const or is_volatile: + base_type = p_c_base_type(s, nonempty=nonempty, templates=templates) if isinstance(base_type, Nodes.MemoryViewSliceTypeNode): # reverse order to avoid having to write "(const int)[:]" - base_type.base_type_node = Nodes.CConstTypeNode(pos, base_type=base_type.base_type_node) + base_type.base_type_node = Nodes.CConstOrVolatileTypeNode(pos, + base_type=base_type.base_type_node, is_const=is_const, is_volatile=is_volatile) return base_type - return Nodes.CConstTypeNode(pos, base_type=base_type) + return Nodes.CConstOrVolatileTypeNode(pos, + base_type=base_type, is_const=is_const, is_volatile=is_volatile) + + if s.sy != 'IDENT': + error(pos, "Expected an identifier, found '%s'" % s.sy) if looking_at_base_type(s): #print "p_c_simple_base_type: looking_at_base_type at", s.position() is_basic = 1 @@ -2531,27 +2658,29 @@ def p_c_simple_base_type(s, self_flag, nonempty, templates = None): name = p_ident(s) else: name = s.systring + name_pos = s.position() s.next() if nonempty and s.sy != 'IDENT': # Make sure this is not a declaration of a variable or function. if s.sy == '(': + old_pos = s.position() s.next() if (s.sy == '*' or s.sy == '**' or s.sy == '&' or (s.sy == 'IDENT' and s.systring in calling_convention_words)): - s.put_back('(', '(') + s.put_back(u'(', u'(', old_pos) else: - s.put_back('(', '(') - s.put_back('IDENT', name) + s.put_back(u'(', u'(', old_pos) + s.put_back(u'IDENT', name, name_pos) name = None elif s.sy not in ('*', '**', '[', '&'): - s.put_back('IDENT', name) + s.put_back(u'IDENT', name, name_pos) name = None type_node = Nodes.CSimpleBaseTypeNode(pos, name = name, module_path = module_path, is_basic_c_type = is_basic, signed = signed, complex = complex, longness = longness, - is_self_arg = self_flag, templates = templates) + templates = templates) # declarations here. if s.sy == '[': @@ -2618,13 +2747,13 @@ def is_memoryviewslice_access(s): # a memoryview slice declaration is distinguishable from a buffer access # declaration by the first entry in the bracketed list. The buffer will # not have an unnested colon in the first entry; the memoryview slice will. - saved = [(s.sy, s.systring)] + saved = [(s.sy, s.systring, s.position())] s.next() retval = False if s.systring == ':': retval = True elif s.sy == 'INT': - saved.append((s.sy, s.systring)) + saved.append((s.sy, s.systring, s.position())) s.next() if s.sy == ':': retval = True @@ -2651,7 +2780,7 @@ def p_memoryviewslice_access(s, base_type_node): return result def looking_at_name(s): - return s.sy == 'IDENT' and not s.systring in calling_convention_words + return s.sy == 'IDENT' and s.systring not in calling_convention_words def looking_at_expr(s): if s.systring in base_type_start_words: @@ -2659,15 +2788,16 @@ def looking_at_expr(s): elif s.sy == 'IDENT': is_type = False name = s.systring + name_pos = s.position() dotted_path = [] s.next() while s.sy == '.': s.next() - dotted_path.append(s.systring) + dotted_path.append((s.systring, s.position())) s.expect('IDENT') - saved = s.sy, s.systring + saved = s.sy, s.systring, s.position() if s.sy == 'IDENT': is_type = True elif s.sy == '*' or s.sy == '**': @@ -2685,10 +2815,10 @@ def looking_at_expr(s): dotted_path.reverse() for p in dotted_path: - s.put_back('IDENT', p) - s.put_back('.', '.') + s.put_back(u'IDENT', *p) + s.put_back(u'.', u'.', p[1]) # gets the position slightly wrong - s.put_back('IDENT', name) + s.put_back(u'IDENT', name, name_pos) return not is_type and saved[0] else: return True @@ -2700,26 +2830,17 @@ def looking_at_base_type(s): def looking_at_dotted_name(s): if s.sy == 'IDENT': name = s.systring + name_pos = s.position() s.next() result = s.sy == '.' - s.put_back('IDENT', name) + s.put_back(u'IDENT', name, name_pos) return result else: return 0 -def looking_at_call(s): - "See if we're looking at a.b.c(" - # Don't mess up the original position, so save and restore it. - # Unfortunately there's no good way to handle this, as a subsequent call - # to next() will not advance the position until it reads a new token. - position = s.start_line, s.start_col - result = looking_at_expr(s) == u'(' - if not result: - s.start_line, s.start_col = position - return result -basic_c_type_names = cython.declare( - set, set(["void", "char", "int", "float", "double", "bint"])) +basic_c_type_names = cython.declare(frozenset, frozenset(( + "void", "char", "int", "float", "double", "bint"))) special_basic_c_types = cython.declare(dict, { # name : (signed, longness) @@ -2733,17 +2854,17 @@ special_basic_c_types = cython.declare(dict, { "Py_tss_t" : (1, 0), }) -sign_and_longness_words = cython.declare( - set, set(["short", "long", "signed", "unsigned"])) +sign_and_longness_words = cython.declare(frozenset, frozenset(( + "short", "long", "signed", "unsigned"))) base_type_start_words = cython.declare( - set, + frozenset, basic_c_type_names | sign_and_longness_words - | set(special_basic_c_types)) + | frozenset(special_basic_c_types)) -struct_enum_union = cython.declare( - set, set(["struct", "union", "enum", "packed"])) +struct_enum_union = cython.declare(frozenset, frozenset(( + "struct", "union", "enum", "packed"))) def p_sign_and_longness(s): signed = 1 @@ -2798,7 +2919,7 @@ def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0, pos = s.position() if s.sy == '[': result = p_c_array_declarator(s, result) - else: # sy == '(' + else: # sy == '(' s.next() result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag) cmethod_flag = 0 @@ -2806,7 +2927,7 @@ def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0, def p_c_array_declarator(s, base): pos = s.position() - s.next() # '[' + s.next() # '[' if s.sy != ']': dim = p_testlist(s) else: @@ -2815,14 +2936,22 @@ def p_c_array_declarator(s, base): return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim) def p_c_func_declarator(s, pos, ctx, base, cmethod_flag): - # Opening paren has already been skipped + # Opening paren has already been skipped args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag, nonempty_declarators = 0) ellipsis = p_optional_ellipsis(s) s.expect(')') nogil = p_nogil(s) - exc_val, exc_check = p_exception_value_clause(s) - # TODO - warning to enforce preferred exception specification order + exc_val, exc_check, exc_clause = p_exception_value_clause(s, ctx) + if nogil and exc_clause: + warning( + s.position(), + "The keyword 'nogil' should appear at the end of the " + "function signature line. Placing it before 'except' " + "or 'noexcept' will be disallowed in a future version " + "of Cython.", + level=2 + ) nogil = nogil or p_nogil(s) with_gil = p_with_gil(s) return Nodes.CFuncDeclaratorNode(pos, @@ -2830,49 +2959,43 @@ def p_c_func_declarator(s, pos, ctx, base, cmethod_flag): exception_value = exc_val, exception_check = exc_check, nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil) -supported_overloaded_operators = cython.declare(set, set([ +supported_overloaded_operators = cython.declare(frozenset, frozenset(( '+', '-', '*', '/', '%', '++', '--', '~', '|', '&', '^', '<<', '>>', ',', '==', '!=', '>=', '>', '<=', '<', '[]', '()', '!', '=', 'bool', -])) +))) def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag, assignable, nonempty): pos = s.position() calling_convention = p_calling_convention(s) - if s.sy == '*': + if s.sy in ('*', '**'): + # scanner returns '**' as a single token + is_ptrptr = s.sy == '**' s.next() - if s.systring == 'const': - const_pos = s.position() + + const_pos = s.position() + is_const = s.systring == 'const' and s.sy == 'IDENT' + if is_const: s.next() - const_base = p_c_declarator(s, ctx, empty = empty, - is_type = is_type, - cmethod_flag = cmethod_flag, - assignable = assignable, - nonempty = nonempty) - base = Nodes.CConstDeclaratorNode(const_pos, base = const_base) - else: - base = p_c_declarator(s, ctx, empty = empty, is_type = is_type, - cmethod_flag = cmethod_flag, - assignable = assignable, nonempty = nonempty) - result = Nodes.CPtrDeclaratorNode(pos, - base = base) - elif s.sy == '**': # scanner returns this as a single token - s.next() - base = p_c_declarator(s, ctx, empty = empty, is_type = is_type, - cmethod_flag = cmethod_flag, - assignable = assignable, nonempty = nonempty) - result = Nodes.CPtrDeclaratorNode(pos, - base = Nodes.CPtrDeclaratorNode(pos, - base = base)) - elif s.sy == '&': - s.next() - base = p_c_declarator(s, ctx, empty = empty, is_type = is_type, - cmethod_flag = cmethod_flag, - assignable = assignable, nonempty = nonempty) - result = Nodes.CReferenceDeclaratorNode(pos, base = base) + + base = p_c_declarator(s, ctx, empty=empty, is_type=is_type, + cmethod_flag=cmethod_flag, + assignable=assignable, nonempty=nonempty) + if is_const: + base = Nodes.CConstDeclaratorNode(const_pos, base=base) + if is_ptrptr: + base = Nodes.CPtrDeclaratorNode(pos, base=base) + result = Nodes.CPtrDeclaratorNode(pos, base=base) + elif s.sy == '&' or (s.sy == '&&' and s.context.cpp): + node_class = Nodes.CppRvalueReferenceDeclaratorNode if s.sy == '&&' else Nodes.CReferenceDeclaratorNode + s.next() + base = p_c_declarator(s, ctx, empty=empty, is_type=is_type, + cmethod_flag=cmethod_flag, + assignable=assignable, nonempty=nonempty) + result = node_class(pos, base=base) else: rhs = None if s.sy == 'IDENT': @@ -2913,7 +3036,7 @@ def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag, fatal=False) name += op elif op == 'IDENT': - op = s.systring; + op = s.systring if op not in supported_overloaded_operators: s.error("Overloading operator '%s' not yet supported." % op, fatal=False) @@ -2939,22 +3062,54 @@ def p_with_gil(s): else: return 0 -def p_exception_value_clause(s): +def p_exception_value_clause(s, ctx): + """ + Parse exception value clause. + + Maps clauses to exc_check / exc_value / exc_clause as follows: + ______________________________________________________________________ + | | | | | + | Clause | exc_check | exc_value | exc_clause | + | ___________________________ | ___________ | ___________ | __________ | + | | | | | + | <nothing> (default func.) | True | None | False | + | <nothing> (cdef extern) | False | None | False | + | noexcept | False | None | True | + | except <val> | False | <val> | True | + | except? <val> | True | <val> | True | + | except * | True | None | True | + | except + | '+' | None | True | + | except +* | '+' | '*' | True | + | except +<PyErr> | '+' | <PyErr> | True | + | ___________________________ | ___________ | ___________ | __________ | + + Note that the only reason we need `exc_clause` is to raise a + warning when `'except'` or `'noexcept'` is placed after the + `'nogil'` keyword. + """ + exc_clause = False exc_val = None - exc_check = 0 + if ctx.visibility == 'extern': + exc_check = False + else: + exc_check = True if s.sy == 'IDENT' and s.systring == 'noexcept': + exc_clause = True s.next() - exc_check = False # No-op in Cython 0.29.x + exc_check = False elif s.sy == 'except': + exc_clause = True s.next() if s.sy == '*': - exc_check = 1 + exc_check = True s.next() elif s.sy == '+': exc_check = '+' s.next() - if s.sy == 'IDENT': + if p_nogil(s): + ctx.nogil = True + elif s.sy == 'IDENT': name = s.systring s.next() exc_val = p_name(s, name) @@ -2963,12 +3118,19 @@ def p_exception_value_clause(s): s.next() else: if s.sy == '?': - exc_check = 1 + exc_check = True s.next() + else: + exc_check = False + # exc_val can be non-None even if exc_check is False, c.f. "except -1" exc_val = p_test(s) - return exc_val, exc_check + if not exc_clause and ctx.visibility != 'extern' and s.context.legacy_implicit_noexcept: + exc_check = False + warning(s.position(), "Implicit noexcept declaration is deprecated. Function declaration should contain 'noexcept' keyword.", level=2) + return exc_val, exc_check, exc_clause -c_arg_list_terminators = cython.declare(set, set(['*', '**', '.', ')', ':'])) +c_arg_list_terminators = cython.declare(frozenset, frozenset(( + '*', '**', '...', ')', ':', '/'))) def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0, nonempty_declarators = 0, kw_only = 0, annotated = 1): @@ -2987,7 +3149,7 @@ def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0, return args def p_optional_ellipsis(s): - if s.sy == '.': + if s.sy == '...': expect_ellipsis(s) return 1 else: @@ -3007,7 +3169,7 @@ def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, complex = 0, longness = 0, is_self_arg = cmethod_flag, templates = None) else: - base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty) + base_type = p_c_base_type(s, nonempty=nonempty) declarator = p_c_declarator(s, ctx, nonempty = nonempty) if s.sy in ('not', 'or') and not s.in_python_file: kind = s.sy @@ -3022,7 +3184,7 @@ def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, not_none = kind == 'not' if annotated and s.sy == ':': s.next() - annotation = p_test(s) + annotation = p_annotation(s) if s.sy == '=': s.next() if 'pxd' in ctx.level: @@ -3124,6 +3286,12 @@ def p_cdef_extern_block(s, pos, ctx): def p_c_enum_definition(s, pos, ctx): # s.sy == ident 'enum' s.next() + + scoped = False + if s.context.cpp and (s.sy == 'class' or (s.sy == 'IDENT' and s.systring == 'struct')): + scoped = True + s.next() + if s.sy == 'IDENT': name = s.systring s.next() @@ -3131,24 +3299,51 @@ def p_c_enum_definition(s, pos, ctx): if cname is None and ctx.namespace is not None: cname = ctx.namespace + "::" + name else: - name = None - cname = None - items = None + name = cname = None + if scoped: + s.error("Unnamed scoped enum not allowed") + + if scoped and s.sy == '(': + s.next() + underlying_type = p_c_base_type(s) + s.expect(')') + else: + underlying_type = Nodes.CSimpleBaseTypeNode( + pos, + name="int", + module_path = [], + is_basic_c_type = True, + signed = 1, + complex = 0, + longness = 0 + ) + s.expect(':') items = [] + + doc = None if s.sy != 'NEWLINE': p_c_enum_line(s, ctx, items) else: - s.next() # 'NEWLINE' + s.next() # 'NEWLINE' s.expect_indent() + doc = p_doc_string(s) + while s.sy not in ('DEDENT', 'EOF'): p_c_enum_line(s, ctx, items) + s.expect_dedent() + + if not items and ctx.visibility != "extern": + error(pos, "Empty enum definition not allowed outside a 'cdef extern from' block") + return Nodes.CEnumDefNode( - pos, name = name, cname = cname, items = items, - typedef_flag = ctx.typedef_flag, visibility = ctx.visibility, - create_wrapper = ctx.overridable, - api = ctx.api, in_pxd = ctx.level == 'module_pxd') + pos, name=name, cname=cname, + scoped=scoped, items=items, + underlying_type=underlying_type, + typedef_flag=ctx.typedef_flag, visibility=ctx.visibility, + create_wrapper=ctx.overridable, + api=ctx.api, in_pxd=ctx.level == 'module_pxd', doc=doc) def p_c_enum_line(s, ctx, items): if s.sy != 'pass': @@ -3192,20 +3387,28 @@ def p_c_struct_or_union_definition(s, pos, ctx): attributes = None if s.sy == ':': s.next() - s.expect('NEWLINE') - s.expect_indent() attributes = [] - body_ctx = Ctx() - while s.sy != 'DEDENT': - if s.sy != 'pass': - attributes.append( - p_c_func_or_var_declaration(s, s.position(), body_ctx)) - else: - s.next() - s.expect_newline("Expected a newline") - s.expect_dedent() + if s.sy == 'pass': + s.next() + s.expect_newline("Expected a newline", ignore_semicolon=True) + else: + s.expect('NEWLINE') + s.expect_indent() + body_ctx = Ctx(visibility=ctx.visibility) + while s.sy != 'DEDENT': + if s.sy != 'pass': + attributes.append( + p_c_func_or_var_declaration(s, s.position(), body_ctx)) + else: + s.next() + s.expect_newline("Expected a newline") + s.expect_dedent() + + if not attributes and ctx.visibility != "extern": + error(pos, "Empty struct or union definition not allowed outside a 'cdef extern from' block") else: s.expect_newline("Syntax error in struct or union definition") + return Nodes.CStructOrUnionDefNode(pos, name = name, cname = cname, kind = kind, attributes = attributes, typedef_flag = ctx.typedef_flag, visibility = ctx.visibility, @@ -3232,7 +3435,7 @@ def p_fused_definition(s, pos, ctx): while s.sy != 'DEDENT': if s.sy != 'pass': #types.append(p_c_declarator(s)) - types.append(p_c_base_type(s)) #, nonempty=1)) + types.append(p_c_base_type(s)) #, nonempty=1)) else: s.next() @@ -3363,14 +3566,7 @@ def p_decorators(s): while s.sy == '@': pos = s.position() s.next() - decstring = p_dotted_name(s, as_allowed=0)[2] - names = decstring.split('.') - decorator = ExprNodes.NameNode(pos, name=s.context.intern_ustring(names[0])) - for name in names[1:]: - decorator = ExprNodes.AttributeNode( - pos, attribute=s.context.intern_ustring(name), obj=decorator) - if s.sy == '(': - decorator = p_call(s, decorator) + decorator = p_namedexpr_test(s) decorators.append(Nodes.DecoratorNode(pos, decorator=decorator)) s.expect_newline("Expected a newline after decorator") return decorators @@ -3388,7 +3584,7 @@ def _reject_cdef_modifier_in_py(s, name): def p_def_statement(s, decorators=None, is_async_def=False): # s.sy == 'def' - pos = s.position() + pos = decorators[0].pos if decorators else s.position() # PEP 492 switches the async/await keywords on in "async def" functions if is_async_def: s.enter_async() @@ -3405,7 +3601,7 @@ def p_def_statement(s, decorators=None, is_async_def=False): return_type_annotation = None if s.sy == '->': s.next() - return_type_annotation = p_test(s) + return_type_annotation = p_annotation(s) _reject_cdef_modifier_in_py(s, s.systring) doc, body = p_suite_with_docstring(s, Ctx(level='function')) @@ -3423,6 +3619,20 @@ def p_varargslist(s, terminator=')', annotated=1): annotated = annotated) star_arg = None starstar_arg = None + if s.sy == '/': + if len(args) == 0: + s.error("Got zero positional-only arguments despite presence of " + "positional-only specifier '/'") + s.next() + # Mark all args to the left as pos only + for arg in args: + arg.pos_only = 1 + if s.sy == ',': + s.next() + args.extend(p_c_arg_list(s, in_pyfunc = 1, + nonempty_declarators = 1, annotated = annotated)) + elif s.sy != terminator: + s.error("Syntax error in Python function argument list") if s.sy == '*': s.next() if s.sy == 'IDENT': @@ -3446,7 +3656,7 @@ def p_py_arg_decl(s, annotated = 1): annotation = None if annotated and s.sy == ':': s.next() - annotation = p_test(s) + annotation = p_annotation(s) return Nodes.PyArgDeclNode(pos, name = name, annotation = annotation) @@ -3673,6 +3883,9 @@ def p_compiler_directive_comments(s): for name in new_directives: if name not in result: pass + elif Options.directive_types.get(name) is list: + result[name] += new_directives[name] + new_directives[name] = result[name] elif new_directives[name] == result[name]: warning(pos, "Duplicate directive found: %s" % (name,)) else: @@ -3682,6 +3895,9 @@ def p_compiler_directive_comments(s): if 'language_level' in new_directives: # Make sure we apply the language level already to the first token that follows the comments. s.context.set_language_level(new_directives['language_level']) + if 'legacy_implicit_noexcept' in new_directives: + s.context.legacy_implicit_noexcept = new_directives['legacy_implicit_noexcept'] + result.update(new_directives) @@ -3696,22 +3912,18 @@ def p_module(s, pxd, full_module_name, ctx=Ctx): s.parse_comments = False if s.context.language_level is None: - s.context.set_language_level(2) + s.context.set_language_level('3str') if pos[0].filename: import warnings warnings.warn( - "Cython directive 'language_level' not set, using 2 for now (Py2). " - "This will change in a later release! File: %s" % pos[0].filename, + "Cython directive 'language_level' not set, using '3str' for now (Py3). " + "This has changed from earlier releases! File: %s" % pos[0].filename, FutureWarning, stacklevel=1 if cython.compiled else 2, ) + level = 'module_pxd' if pxd else 'module' doc = p_doc_string(s) - if pxd: - level = 'module_pxd' - else: - level = 'module' - body = p_statement_list(s, ctx(level=level), first_statement = 1) if s.sy != 'EOF': s.error("Syntax error in statement [%s,%s]" % ( @@ -3733,7 +3945,6 @@ def p_template_definition(s): def p_cpp_class_definition(s, pos, ctx): # s.sy == 'cppclass' s.next() - module_path = [] class_name = p_ident(s) cname = p_opt_cname(s) if cname is None and ctx.namespace is not None: @@ -3767,6 +3978,10 @@ def p_cpp_class_definition(s, pos, ctx): s.next() s.expect('NEWLINE') s.expect_indent() + # Allow a cppclass to have docstrings. It will be discarded as comment. + # The goal of this is consistency: we can make docstrings inside cppclass methods, + # so why not on the cppclass itself ? + p_doc_string(s) attributes = [] body_ctx = Ctx(visibility = ctx.visibility, level='cpp_class', nogil=nogil or ctx.nogil) body_ctx.templates = template_names @@ -3850,3 +4065,14 @@ def print_parse_tree(f, node, level, key = None): f.write("%s]\n" % ind) return f.write("%s%s\n" % (ind, node)) + +def p_annotation(s): + """An annotation just has the "test" syntax, but also stores the string it came from + + Note that the string is *allowed* to be changed/processed (although isn't here) + so may not exactly match the string generated by Python, and if it doesn't + then it is not a bug. + """ + pos = s.position() + expr = p_test(s) + return ExprNodes.AnnotationNode(pos, expr=expr) diff --git a/Cython/Compiler/Pipeline.py b/Cython/Compiler/Pipeline.py index 5194c3e49..834eb0e6a 100644 --- a/Cython/Compiler/Pipeline.py +++ b/Cython/Compiler/Pipeline.py @@ -19,7 +19,7 @@ def dumptree(t): def abort_on_errors(node): # Stop the pipeline if there are any errors. - if Errors.num_errors != 0: + if Errors.get_errors_count() != 0: raise AbortError("pipeline break") return node @@ -58,7 +58,7 @@ def generate_pyx_code_stage_factory(options, result): def inject_pxd_code_stage_factory(context): def inject_pxd_code_stage(module_node): for name, (statlistnode, scope) in context.pxds.items(): - module_node.merge_in(statlistnode, scope) + module_node.merge_in(statlistnode, scope, stage="pxd") return module_node return inject_pxd_code_stage @@ -80,56 +80,60 @@ def use_utility_code_definitions(scope, target, seen=None): use_utility_code_definitions(entry.as_module, target, seen) -def sort_utility_codes(utilcodes): +def sorted_utility_codes_and_deps(utilcodes): ranks = {} - def get_rank(utilcode): - if utilcode not in ranks: + get_rank = ranks.get + + def calculate_rank(utilcode): + rank = get_rank(utilcode) + if rank is None: ranks[utilcode] = 0 # prevent infinite recursion on circular dependencies original_order = len(ranks) - ranks[utilcode] = 1 + min([get_rank(dep) for dep in utilcode.requires or ()] or [-1]) + original_order * 1e-8 - return ranks[utilcode] - for utilcode in utilcodes: - get_rank(utilcode) - return [utilcode for utilcode, _ in sorted(ranks.items(), key=lambda kv: kv[1])] - + rank = ranks[utilcode] = 1 + ( + min([calculate_rank(dep) for dep in utilcode.requires]) if utilcode.requires else -1 + ) + original_order * 1e-8 + return rank -def normalize_deps(utilcodes): - deps = {} for utilcode in utilcodes: - deps[utilcode] = utilcode + calculate_rank(utilcode) + + # include all recursively collected dependencies + return sorted(ranks, key=get_rank) - def unify_dep(dep): - if dep in deps: - return deps[dep] - else: - deps[dep] = dep - return dep +def normalize_deps(utilcodes): + deps = {utilcode:utilcode for utilcode in utilcodes} for utilcode in utilcodes: - utilcode.requires = [unify_dep(dep) for dep in utilcode.requires or ()] + utilcode.requires = [deps.setdefault(dep, dep) for dep in utilcode.requires or ()] def inject_utility_code_stage_factory(context): def inject_utility_code_stage(module_node): module_node.prepare_utility_code() use_utility_code_definitions(context.cython_scope, module_node.scope) - module_node.scope.utility_code_list = sort_utility_codes(module_node.scope.utility_code_list) - normalize_deps(module_node.scope.utility_code_list) - added = [] + + utility_code_list = module_node.scope.utility_code_list + utility_code_list[:] = sorted_utility_codes_and_deps(utility_code_list) + normalize_deps(utility_code_list) + + added = set() # Note: the list might be extended inside the loop (if some utility code # pulls in other utility code, explicitly or implicitly) - for utilcode in module_node.scope.utility_code_list: + for utilcode in utility_code_list: if utilcode in added: continue - added.append(utilcode) + added.add(utilcode) if utilcode.requires: for dep in utilcode.requires: - if dep not in added and dep not in module_node.scope.utility_code_list: - module_node.scope.utility_code_list.append(dep) + if dep not in added: + utility_code_list.append(dep) tree = utilcode.get_tree(cython_scope=context.cython_scope) if tree: - module_node.merge_in(tree.body, tree.scope, merge_scope=True) + module_node.merge_in(tree.with_compiler_directives(), + tree.scope, stage="utility", + merge_scope=True) return module_node + return inject_utility_code_stage @@ -148,8 +152,8 @@ def create_pipeline(context, mode, exclude_classes=()): from .ParseTreeTransforms import ExpandInplaceOperators, ParallelRangeTransform from .ParseTreeTransforms import CalculateQualifiedNamesTransform from .TypeInference import MarkParallelAssignments, MarkOverflowingArithmetic - from .ParseTreeTransforms import AdjustDefByDirectives, AlignFunctionDefinitions - from .ParseTreeTransforms import RemoveUnreachableCode, GilCheck + from .ParseTreeTransforms import AdjustDefByDirectives, AlignFunctionDefinitions, AutoCpdefFunctionDefinitions + from .ParseTreeTransforms import RemoveUnreachableCode, GilCheck, CoerceCppTemps from .FlowControl import ControlFlowAnalysis from .AnalysedTreeTransforms import AutoTestDictTransform from .AutoDocTransforms import EmbedSignature @@ -185,10 +189,11 @@ def create_pipeline(context, mode, exclude_classes=()): TrackNumpyAttributes(), InterpretCompilerDirectives(context, context.compiler_directives), ParallelRangeTransform(context), + WithTransform(), AdjustDefByDirectives(context), - WithTransform(context), - MarkClosureVisitor(context), _align_function_definitions, + MarkClosureVisitor(context), + AutoCpdefFunctionDefinitions(context), RemoveUnreachableCode(context), ConstantFolding(), FlattenInListTransform(), @@ -219,26 +224,26 @@ def create_pipeline(context, mode, exclude_classes=()): ConsolidateOverflowCheck(context), DropRefcountingTransform(), FinalOptimizePhase(context), + CoerceCppTemps(context), GilCheck(), ] - filtered_stages = [] - for s in stages: - if s.__class__ not in exclude_classes: - filtered_stages.append(s) - return filtered_stages + if exclude_classes: + stages = [s for s in stages if s.__class__ not in exclude_classes] + return stages def create_pyx_pipeline(context, options, result, py=False, exclude_classes=()): - if py: - mode = 'py' - else: - mode = 'pyx' + mode = 'py' if py else 'pyx' + test_support = [] + ctest_support = [] if options.evaluate_tree_assertions: from ..TestUtils import TreeAssertVisitor - test_support.append(TreeAssertVisitor()) + test_validator = TreeAssertVisitor() + test_support.append(test_validator) + ctest_support.append(test_validator.create_c_file_validator()) if options.gdb_debug: - from ..Debugger import DebugWriter # requires Py2.5+ + from ..Debugger import DebugWriter # requires Py2.5+ from .ParseTreeTransforms import DebugTransform context.gdb_debug_outputwriter = DebugWriter.CythonDebugWriter( options.output_dir) @@ -254,7 +259,9 @@ def create_pyx_pipeline(context, options, result, py=False, exclude_classes=()): inject_utility_code_stage_factory(context), abort_on_errors], debug_transform, - [generate_pyx_code_stage_factory(options, result)])) + [generate_pyx_code_stage_factory(options, result)], + ctest_support, + )) def create_pxd_pipeline(context, scope, module_name): from .CodeGeneration import ExtractPxdCode @@ -284,11 +291,25 @@ def create_pyx_as_pxd_pipeline(context, result): FlattenInListTransform, WithTransform ]) + from .Visitor import VisitorTransform + class SetInPxdTransform(VisitorTransform): + # A number of nodes have an "in_pxd" attribute which affects AnalyseDeclarationsTransform + # (for example controlling pickling generation). Set it, to make sure we don't mix them up with + # the importing main module. + # FIXME: This should be done closer to the parsing step. + def visit_StatNode(self, node): + if hasattr(node, "in_pxd"): + node.in_pxd = True + self.visitchildren(node) + return node + + visit_Node = VisitorTransform.recurse_to_children + for stage in pyx_pipeline: pipeline.append(stage) if isinstance(stage, AnalyseDeclarationsTransform): - # This is the last stage we need. - break + pipeline.insert(-1, SetInPxdTransform()) + break # This is the last stage we need. def fake_pxd(root): for entry in root.scope.entries.values(): if not entry.in_cinclude: @@ -326,11 +347,30 @@ def insert_into_pipeline(pipeline, transform, before=None, after=None): _pipeline_entry_points = {} +try: + from threading import local as _threadlocal +except ImportError: + class _threadlocal(object): pass + +threadlocal = _threadlocal() + + +def get_timings(): + try: + return threadlocal.cython_pipeline_timings + except AttributeError: + return {} + def run_pipeline(pipeline, source, printtree=True): from .Visitor import PrintTree exec_ns = globals().copy() if DebugFlags.debug_verbose_pipeline else None + try: + timings = threadlocal.cython_pipeline_timings + except AttributeError: + timings = threadlocal.cython_pipeline_timings = {} + def run(phase, data): return phase(data) @@ -339,29 +379,39 @@ def run_pipeline(pipeline, source, printtree=True): try: try: for phase in pipeline: - if phase is not None: - if not printtree and isinstance(phase, PrintTree): - continue - if DebugFlags.debug_verbose_pipeline: - t = time() - print("Entering pipeline phase %r" % phase) - # create a new wrapper for each step to show the name in profiles - phase_name = getattr(phase, '__name__', type(phase).__name__) - try: - run = _pipeline_entry_points[phase_name] - except KeyError: - exec("def %s(phase, data): return phase(data)" % phase_name, exec_ns) - run = _pipeline_entry_points[phase_name] = exec_ns[phase_name] - data = run(phase, data) - if DebugFlags.debug_verbose_pipeline: - print(" %.3f seconds" % (time() - t)) + if phase is None: + continue + if not printtree and isinstance(phase, PrintTree): + continue + + phase_name = getattr(phase, '__name__', type(phase).__name__) + if DebugFlags.debug_verbose_pipeline: + print("Entering pipeline phase %r" % phase) + # create a new wrapper for each step to show the name in profiles + try: + run = _pipeline_entry_points[phase_name] + except KeyError: + exec("def %s(phase, data): return phase(data)" % phase_name, exec_ns) + run = _pipeline_entry_points[phase_name] = exec_ns[phase_name] + + t = time() + data = run(phase, data) + t = time() - t + + try: + old_t, count = timings[phase_name] + except KeyError: + old_t, count = 0, 0 + timings[phase_name] = (old_t + int(t * 1000000), count + 1) + if DebugFlags.debug_verbose_pipeline: + print(" %.3f seconds" % t) except CompileError as err: # err is set Errors.report_error(err, use_stack=False) error = err except InternalError as err: # Only raise if there was not an earlier error - if Errors.num_errors == 0: + if Errors.get_errors_count() == 0: raise error = err except AbortError as err: diff --git a/Cython/Compiler/PyrexTypes.py b/Cython/Compiler/PyrexTypes.py index c309bd04b..19d193dcf 100644 --- a/Cython/Compiler/PyrexTypes.py +++ b/Cython/Compiler/PyrexTypes.py @@ -12,13 +12,15 @@ try: reduce except NameError: from functools import reduce +from functools import partial +from itertools import product from Cython.Utils import cached_function from .Code import UtilityCode, LazyUtilityCode, TempitaUtilityCode from . import StringEncoding from . import Naming -from .Errors import error, warning +from .Errors import error, CannotSpecialize class BaseType(object): @@ -46,7 +48,9 @@ class BaseType(object): def cast_code(self, expr_code): return "((%s)%s)" % (self.empty_declaration_code(), expr_code) - def empty_declaration_code(self): + def empty_declaration_code(self, pyrex=False): + if pyrex: + return self.declaration_code('', pyrex=True) if self._empty_declaration is None: self._empty_declaration = self.declaration_code('') return self._empty_declaration @@ -75,7 +79,7 @@ class BaseType(object): """ return self - def get_fused_types(self, result=None, seen=None, subtypes=None): + def get_fused_types(self, result=None, seen=None, subtypes=None, include_function_return_type=False): subtypes = subtypes or self.subtypes if not subtypes: return None @@ -88,10 +92,10 @@ class BaseType(object): list_or_subtype = getattr(self, attr) if list_or_subtype: if isinstance(list_or_subtype, BaseType): - list_or_subtype.get_fused_types(result, seen) + list_or_subtype.get_fused_types(result, seen, include_function_return_type=include_function_return_type) else: for subtype in list_or_subtype: - subtype.get_fused_types(result, seen) + subtype.get_fused_types(result, seen, include_function_return_type=include_function_return_type) return result @@ -115,7 +119,7 @@ class BaseType(object): Deduce any template params in this (argument) type given the actual argument type. - http://en.cppreference.com/w/cpp/language/function_template#Template_argument_deduction + https://en.cppreference.com/w/cpp/language/function_template#Template_argument_deduction """ return {} @@ -176,15 +180,22 @@ class PyrexType(BaseType): # is_ptr boolean Is a C pointer type # is_null_ptr boolean Is the type of NULL # is_reference boolean Is a C reference type - # is_const boolean Is a C const type. + # is_rvalue_reference boolean Is a C++ rvalue reference type + # is_const boolean Is a C const type + # is_volatile boolean Is a C volatile type + # is_cv_qualified boolean Is a C const or volatile type # is_cfunction boolean Is a C function type # is_struct_or_union boolean Is a C struct or union type # is_struct boolean Is a C struct type + # is_cpp_class boolean Is a C++ class + # is_optional_cpp_class boolean Is a C++ class with variable lifetime handled with std::optional # is_enum boolean Is a C enum type + # is_cpp_enum boolean Is a C++ scoped enum type # is_typedef boolean Is a typedef type # is_string boolean Is a C char * type # is_pyunicode_ptr boolean Is a C PyUNICODE * type # is_cpp_string boolean Is a C++ std::string type + # python_type_constructor_name string or None non-None if it is a Python type constructor that can be indexed/"templated" # is_unicode_char boolean Is either Py_UCS4 or Py_UNICODE # is_returncode boolean Is used only to signal exceptions # is_error boolean Is the dummy error type @@ -192,6 +203,10 @@ class PyrexType(BaseType): # is_pythran_expr boolean Is Pythran expr # is_numpy_buffer boolean Is Numpy array buffer # has_attributes boolean Has C dot-selectable attributes + # needs_cpp_construction boolean Needs C++ constructor and destructor when used in a cdef class + # needs_refcounting boolean Needs code to be generated similar to incref/gotref/decref. + # Largely used internally. + # equivalent_type type A C or Python type that is equivalent to this Python or C type. # default_value string Initial value that can be assigned before first user assignment. # declaration_value string The value statically assigned on declaration (if any). # entry Entry The Entry for this type @@ -226,6 +241,7 @@ class PyrexType(BaseType): is_extension_type = 0 is_final_type = 0 is_builtin_type = 0 + is_cython_builtin_type = 0 is_numeric = 0 is_int = 0 is_float = 0 @@ -235,13 +251,20 @@ class PyrexType(BaseType): is_ptr = 0 is_null_ptr = 0 is_reference = 0 + is_fake_reference = 0 + is_rvalue_reference = 0 is_const = 0 + is_volatile = 0 + is_cv_qualified = 0 is_cfunction = 0 is_struct_or_union = 0 is_cpp_class = 0 + is_optional_cpp_class = 0 + python_type_constructor_name = None is_cpp_string = 0 is_struct = 0 is_enum = 0 + is_cpp_enum = False is_typedef = 0 is_string = 0 is_pyunicode_ptr = 0 @@ -254,6 +277,9 @@ class PyrexType(BaseType): is_pythran_expr = 0 is_numpy_buffer = 0 has_attributes = 0 + needs_cpp_construction = 0 + needs_refcounting = 0 + equivalent_type = None default_value = "" declaration_value = "" @@ -262,7 +288,8 @@ class PyrexType(BaseType): return self def specialize(self, values): - # TODO(danilo): Override wherever it makes sense. + # Returns the concrete type if this is a fused type, or otherwise the type itself. + # May raise Errors.CannotSpecialize on failure return self def literal_code(self, value): @@ -317,7 +344,8 @@ class PyrexType(BaseType): return 0 def _assign_from_py_code(self, source_code, result_code, error_pos, code, - from_py_function=None, error_condition=None, extra_args=None): + from_py_function=None, error_condition=None, extra_args=None, + special_none_cvalue=None): args = ', ' + ', '.join('%s' % arg for arg in extra_args) if extra_args else '' convert_call = "%s(%s%s)" % ( from_py_function or self.from_py_function, @@ -326,11 +354,44 @@ class PyrexType(BaseType): ) if self.is_enum: convert_call = typecast(self, c_long_type, convert_call) + if special_none_cvalue: + # NOTE: requires 'source_code' to be simple! + convert_call = "(__Pyx_Py_IsNone(%s) ? (%s) : (%s))" % ( + source_code, special_none_cvalue, convert_call) return '%s = %s; %s' % ( result_code, convert_call, code.error_goto_if(error_condition or self.error_condition(result_code), error_pos)) + def _generate_dummy_refcounting(self, code, *ignored_args, **ignored_kwds): + if self.needs_refcounting: + raise NotImplementedError("Ref-counting operation not yet implemented for type %s" % + self) + + def _generate_dummy_refcounting_assignment(self, code, cname, rhs_cname, *ignored_args, **ignored_kwds): + if self.needs_refcounting: + raise NotImplementedError("Ref-counting operation not yet implemented for type %s" % + self) + code.putln("%s = %s" % (cname, rhs_cname)) + + generate_incref = generate_xincref = generate_decref = generate_xdecref \ + = generate_decref_clear = generate_xdecref_clear \ + = generate_gotref = generate_xgotref = generate_giveref = generate_xgiveref \ + = _generate_dummy_refcounting + + generate_decref_set = generate_xdecref_set = _generate_dummy_refcounting_assignment + + def nullcheck_string(self, code, cname): + if self.needs_refcounting: + raise NotImplementedError("Ref-counting operation not yet implemented for type %s" % + self) + code.putln("1") + + def cpp_optional_declaration_code(self, entity_code, dll_linkage=None): + # declares an std::optional c++ variable + raise NotImplementedError( + "cpp_optional_declaration_code only implemented for c++ classes and not type %s" % self) + def public_decl(base_code, dll_linkage): if dll_linkage: @@ -338,20 +399,15 @@ def public_decl(base_code, dll_linkage): else: return base_code -def create_typedef_type(name, base_type, cname, is_external=0, namespace=None): - is_fused = base_type.is_fused - if base_type.is_complex or is_fused: - if is_external: - if is_fused: - msg = "Fused" - else: - msg = "Complex" - - raise ValueError("%s external typedefs not supported" % msg) +def create_typedef_type(name, base_type, cname, is_external=0, namespace=None): + if is_external: + if base_type.is_complex or base_type.is_fused: + raise ValueError("%s external typedefs not supported" % ( + "Fused" if base_type.is_fused else "Complex")) + if base_type.is_complex or base_type.is_fused: return base_type - else: - return CTypedefType(name, base_type, cname, is_external, namespace) + return CTypedefType(name, base_type, cname, is_external, namespace) class CTypedefType(BaseType): @@ -447,9 +503,9 @@ class CTypedefType(BaseType): "TO_PY_FUNCTION": self.to_py_function})) return True elif base_type.is_float: - pass # XXX implement! + pass # XXX implement! elif base_type.is_complex: - pass # XXX implement! + pass # XXX implement! pass elif base_type.is_cpp_string: cname = "__pyx_convert_PyObject_string_to_py_%s" % type_identifier(self) @@ -476,13 +532,16 @@ class CTypedefType(BaseType): self.from_py_function = "__Pyx_PyInt_As_" + self.specialization_name() env.use_utility_code(TempitaUtilityCode.load_cached( "CIntFromPy", "TypeConversion.c", - context={"TYPE": self.empty_declaration_code(), - "FROM_PY_FUNCTION": self.from_py_function})) + context={ + "TYPE": self.empty_declaration_code(), + "FROM_PY_FUNCTION": self.from_py_function, + "IS_ENUM": base_type.is_enum, + })) return True elif base_type.is_float: - pass # XXX implement! + pass # XXX implement! elif base_type.is_complex: - pass # XXX implement! + pass # XXX implement! elif base_type.is_cpp_string: cname = '__pyx_convert_string_from_py_%s' % type_identifier(self) context = { @@ -507,11 +566,13 @@ class CTypedefType(BaseType): source_code, result_code, result_type, to_py_function) def from_py_call_code(self, source_code, result_code, error_pos, code, - from_py_function=None, error_condition=None): + from_py_function=None, error_condition=None, + special_none_cvalue=None): return self.typedef_base_type.from_py_call_code( source_code, result_code, error_pos, code, from_py_function or self.from_py_function, - error_condition or self.error_condition(result_code) + error_condition or self.error_condition(result_code), + special_none_cvalue=special_none_cvalue, ) def overflow_check_binop(self, binop, env, const_rhs=False): @@ -561,8 +622,13 @@ class CTypedefType(BaseType): class MemoryViewSliceType(PyrexType): is_memoryviewslice = 1 + default_value = "{ 0, 0, { 0 }, { 0 }, { 0 } }" has_attributes = 1 + needs_refcounting = 1 # Ideally this would be true and reference counting for + # memoryview and pyobject code could be generated in the same way. + # However, memoryviews are sufficiently specialized that this doesn't + # seem practical. Implement a limited version of it for now scope = None # These are special cased in Defnode @@ -591,7 +657,7 @@ class MemoryViewSliceType(PyrexType): 'ptr' -- Pointer stored in this dimension. 'full' -- Check along this dimension, don't assume either. - the packing specifiers specify how the array elements are layed-out + the packing specifiers specify how the array elements are laid-out in memory. 'contig' -- The data is contiguous in memory along this dimension. @@ -633,6 +699,10 @@ class MemoryViewSliceType(PyrexType): else: return False + def __ne__(self, other): + # TODO drop when Python2 is dropped + return not (self == other) + def same_as_resolved_type(self, other_type): return ((other_type.is_memoryviewslice and #self.writable_needed == other_type.writable_needed and # FIXME: should be only uni-directional @@ -650,10 +720,10 @@ class MemoryViewSliceType(PyrexType): def declaration_code(self, entity_code, for_display = 0, dll_linkage = None, pyrex = 0): # XXX: we put these guards in for now... - assert not pyrex assert not dll_linkage from . import MemoryView - base_code = str(self) if for_display else MemoryView.memviewslice_cname + base_code = StringEncoding.EncodedString( + str(self) if pyrex or for_display else MemoryView.memviewslice_cname) return self.base_declaration_code( base_code, entity_code) @@ -713,8 +783,8 @@ class MemoryViewSliceType(PyrexType): to_axes_f = contig_dim + follow_dim * (ndim -1) dtype = self.dtype - if dtype.is_const: - dtype = dtype.const_base_type + if dtype.is_cv_qualified: + dtype = dtype.cv_base_type to_memview_c = MemoryViewSliceType(dtype, to_axes_c) to_memview_f = MemoryViewSliceType(dtype, to_axes_f) @@ -791,17 +861,20 @@ class MemoryViewSliceType(PyrexType): # return False src_dtype, dst_dtype = src.dtype, dst.dtype - if dst_dtype.is_const: - # Requesting read-only views is always ok => consider only the non-const base type. - dst_dtype = dst_dtype.const_base_type - if src_dtype.is_const: - # When assigning between read-only views, compare only the non-const base types. - src_dtype = src_dtype.const_base_type - elif copying and src_dtype.is_const: - # Copying by value => ignore const on source. - src_dtype = src_dtype.const_base_type - - if src_dtype != dst_dtype: + # We can add but not remove const/volatile modifiers + # (except if we are copying by value, then anything is fine) + if not copying: + if src_dtype.is_const and not dst_dtype.is_const: + return False + if src_dtype.is_volatile and not dst_dtype.is_volatile: + return False + # const/volatile checks are done, remove those qualifiers + if src_dtype.is_cv_qualified: + src_dtype = src_dtype.cv_base_type + if dst_dtype.is_cv_qualified: + dst_dtype = dst_dtype.cv_base_type + + if not src_dtype.same_as(dst_dtype): return False if src.ndim != dst.ndim: @@ -918,13 +991,16 @@ class MemoryViewSliceType(PyrexType): return True def from_py_call_code(self, source_code, result_code, error_pos, code, - from_py_function=None, error_condition=None): + from_py_function=None, error_condition=None, + special_none_cvalue=None): # NOTE: auto-detection of readonly buffers is disabled: # writable = self.writable_needed or not self.dtype.is_const writable = not self.dtype.is_const return self._assign_from_py_code( source_code, result_code, error_pos, code, from_py_function, error_condition, - extra_args=['PyBUF_WRITABLE' if writable else '0']) + extra_args=['PyBUF_WRITABLE' if writable else '0'], + special_none_cvalue=special_none_cvalue, + ) def create_to_py_utility_code(self, env): self._dtype_to_py_func, self._dtype_from_py_func = self.dtype_object_conversion_funcs(env) @@ -1032,6 +1108,36 @@ class MemoryViewSliceType(PyrexType): def cast_code(self, expr_code): return expr_code + # When memoryviews are increfed currently seems heavily special-cased. + # Therefore, use our own function for now + def generate_incref(self, code, name, **kwds): + pass + + def generate_incref_memoryviewslice(self, code, slice_cname, have_gil): + # TODO ideally would be done separately + code.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil))) + + # decref however did look to always apply for memoryview slices + # with "have_gil" set to True by default + def generate_xdecref(self, code, cname, nanny, have_gil): + code.putln("__PYX_XCLEAR_MEMVIEW(&%s, %d);" % (cname, int(have_gil))) + + def generate_decref(self, code, cname, nanny, have_gil): + # Fall back to xdecref since we don't care to have a separate decref version for this. + self.generate_xdecref(code, cname, nanny, have_gil) + + def generate_xdecref_clear(self, code, cname, clear_before_decref, **kwds): + self.generate_xdecref(code, cname, **kwds) + code.putln("%s.memview = NULL; %s.data = NULL;" % (cname, cname)) + + def generate_decref_clear(self, code, cname, **kwds): + # memoryviews don't currently distinguish between xdecref and decref + self.generate_xdecref_clear(code, cname, **kwds) + + # memoryviews don't participate in giveref/gotref + generate_gotref = generate_xgotref = generate_xgiveref = generate_giveref = lambda *args: None + + class BufferType(BaseType): # @@ -1129,6 +1235,8 @@ class PyObjectType(PyrexType): is_extern = False is_subclassed = False is_gc_simple = False + builtin_trashcan = False # builtin type using trashcan + needs_refcounting = True def __str__(self): return "Python object" @@ -1181,11 +1289,85 @@ class PyObjectType(PyrexType): def check_for_null_code(self, cname): return cname + def generate_incref(self, code, cname, nanny): + if nanny: + code.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname)) + else: + code.putln("Py_INCREF(%s);" % self.as_pyobject(cname)) + + def generate_xincref(self, code, cname, nanny): + if nanny: + code.putln("__Pyx_XINCREF(%s);" % self.as_pyobject(cname)) + else: + code.putln("Py_XINCREF(%s);" % self.as_pyobject(cname)) + + def generate_decref(self, code, cname, nanny, have_gil): + # have_gil is for the benefit of memoryviewslice - it's ignored here + assert have_gil + self._generate_decref(code, cname, nanny, null_check=False, clear=False) + + def generate_xdecref(self, code, cname, nanny, have_gil): + # in this (and other) PyObjectType functions, have_gil is being + # passed to provide a common interface with MemoryviewSlice. + # It's ignored here + self._generate_decref(code, cname, nanny, null_check=True, + clear=False) + + def generate_decref_clear(self, code, cname, clear_before_decref, nanny, have_gil): + self._generate_decref(code, cname, nanny, null_check=False, + clear=True, clear_before_decref=clear_before_decref) + + def generate_xdecref_clear(self, code, cname, clear_before_decref=False, nanny=True, have_gil=None): + self._generate_decref(code, cname, nanny, null_check=True, + clear=True, clear_before_decref=clear_before_decref) + + def generate_gotref(self, code, cname): + code.putln("__Pyx_GOTREF(%s);" % self.as_pyobject(cname)) + + def generate_xgotref(self, code, cname): + code.putln("__Pyx_XGOTREF(%s);" % self.as_pyobject(cname)) + + def generate_giveref(self, code, cname): + code.putln("__Pyx_GIVEREF(%s);" % self.as_pyobject(cname)) + + def generate_xgiveref(self, code, cname): + code.putln("__Pyx_XGIVEREF(%s);" % self.as_pyobject(cname)) + + def generate_decref_set(self, code, cname, rhs_cname): + code.putln("__Pyx_DECREF_SET(%s, %s);" % (cname, rhs_cname)) + + def generate_xdecref_set(self, code, cname, rhs_cname): + code.putln("__Pyx_XDECREF_SET(%s, %s);" % (cname, rhs_cname)) + + def _generate_decref(self, code, cname, nanny, null_check=False, + clear=False, clear_before_decref=False): + prefix = '__Pyx' if nanny else 'Py' + X = 'X' if null_check else '' + + if clear: + if clear_before_decref: + if not nanny: + X = '' # CPython doesn't have a Py_XCLEAR() + code.putln("%s_%sCLEAR(%s);" % (prefix, X, cname)) + else: + code.putln("%s_%sDECREF(%s); %s = 0;" % ( + prefix, X, self.as_pyobject(cname), cname)) + else: + code.putln("%s_%sDECREF(%s);" % ( + prefix, X, self.as_pyobject(cname))) + + def nullcheck_string(self, cname): + return cname + + +builtin_types_that_cannot_create_refcycles = frozenset({ + 'object', 'bool', 'int', 'long', 'float', 'complex', + 'bytearray', 'bytes', 'unicode', 'str', 'basestring', +}) -builtin_types_that_cannot_create_refcycles = set([ - 'bool', 'int', 'long', 'float', 'complex', - 'bytearray', 'bytes', 'unicode', 'str', 'basestring' -]) +builtin_types_with_trashcan = frozenset({ + 'dict', 'list', 'set', 'frozenset', 'tuple', 'type', +}) class BuiltinObjectType(PyObjectType): @@ -1211,6 +1393,7 @@ class BuiltinObjectType(PyObjectType): self.typeptr_cname = "(&%s)" % cname self.objstruct_cname = objstruct_cname self.is_gc_simple = name in builtin_types_that_cannot_create_refcycles + self.builtin_trashcan = name in builtin_types_with_trashcan if name == 'type': # Special case the type type, as many C API calls (and other # libraries) actually expect a PyTypeObject* for type arguments. @@ -1276,6 +1459,12 @@ class BuiltinObjectType(PyObjectType): type_check = 'PyByteArray_Check' elif type_name == 'frozenset': type_check = 'PyFrozenSet_Check' + elif type_name == 'int': + # For backwards compatibility of (Py3) 'x: int' annotations in Py2, we also allow 'long' there. + type_check = '__Pyx_Py3Int_Check' + elif type_name == "memoryview": + # captialize doesn't catch the 'V' + type_check = "PyMemoryView_Check" else: type_check = 'Py%s_Check' % type_name.capitalize() if exact and type_name not in ('bool', 'slice', 'Exception'): @@ -1292,14 +1481,9 @@ class BuiltinObjectType(PyObjectType): check += '||((%s) == Py_None)' % arg if self.name == 'basestring': name = '(PY_MAJOR_VERSION < 3 ? "basestring" : "str")' - space_for_name = 16 else: name = '"%s"' % self.name - # avoid wasting too much space but limit number of different format strings - space_for_name = (len(self.name) // 16 + 1) * 16 - error = '((void)PyErr_Format(PyExc_TypeError, "Expected %%.%ds, got %%.200s", %s, Py_TYPE(%s)->tp_name), 0)' % ( - space_for_name, name, arg) - return check + '||' + error + return check + ' || __Pyx_RaiseUnexpectedTypeError(%s, %s)' % (name, arg) def declaration_code(self, entity_code, for_display = 0, dll_linkage = None, pyrex = 0): @@ -1318,7 +1502,7 @@ class BuiltinObjectType(PyObjectType): def cast_code(self, expr_code, to_object_struct = False): return "((%s*)%s)" % ( - to_object_struct and self.objstruct_cname or self.decl_type, # self.objstruct_cname may be None + to_object_struct and self.objstruct_cname or self.decl_type, # self.objstruct_cname may be None expr_code) def py_type_name(self): @@ -1332,7 +1516,6 @@ class PyExtensionType(PyObjectType): # # name string # scope CClassScope Attribute namespace - # visibility string # typedef_flag boolean # base_type PyExtensionType or None # module_name string or None Qualified name of defining module @@ -1346,13 +1529,20 @@ class PyExtensionType(PyObjectType): # vtable_cname string Name of C method table definition # early_init boolean Whether to initialize early (as opposed to during module execution). # defered_declarations [thunk] Used to declare class hierarchies in order + # is_external boolean Defined in a extern block # check_size 'warn', 'error', 'ignore' What to do if tp_basicsize does not match + # dataclass_fields OrderedDict nor None Used for inheriting from dataclasses + # multiple_bases boolean Does this class have multiple bases + # has_sequence_flag boolean Set Py_TPFLAGS_SEQUENCE is_extension_type = 1 has_attributes = 1 early_init = 1 objtypedef_cname = None + dataclass_fields = None + multiple_bases = False + has_sequence_flag = False def __init__(self, name, typedef_flag, base_type, is_external=0, check_size=None): self.name = name @@ -1510,9 +1700,11 @@ class CType(PyrexType): source_code or 'NULL') def from_py_call_code(self, source_code, result_code, error_pos, code, - from_py_function=None, error_condition=None): + from_py_function=None, error_condition=None, + special_none_cvalue=None): return self._assign_from_py_code( - source_code, result_code, error_pos, code, from_py_function, error_condition) + source_code, result_code, error_pos, code, from_py_function, error_condition, + special_none_cvalue=special_none_cvalue) @@ -1543,8 +1735,14 @@ class PythranExpr(CType): self.scope = scope = Symtab.CClassScope('', None, visibility="extern") scope.parent_type = self scope.directives = {} - scope.declare_var("shape", CPtrType(c_long_type), None, cname="_shape", is_cdef=True) - scope.declare_var("ndim", c_long_type, None, cname="value", is_cdef=True) + + scope.declare_var("ndim", c_long_type, pos=None, cname="value", is_cdef=True) + scope.declare_cproperty( + "shape", c_ptr_type(c_long_type), "__Pyx_PythranShapeAccessor", + doc="Pythran array shape", + visibility="extern", + nogil=True, + ) return True @@ -1558,59 +1756,76 @@ class PythranExpr(CType): return hash(self.pythran_type) -class CConstType(BaseType): +class CConstOrVolatileType(BaseType): + "A C const or volatile type" - is_const = 1 - subtypes = ['const_base_type'] + subtypes = ['cv_base_type'] - def __init__(self, const_base_type): - self.const_base_type = const_base_type - if const_base_type.has_attributes and const_base_type.scope is not None: - from . import Symtab - self.scope = Symtab.CConstScope(const_base_type.scope) + is_cv_qualified = 1 + + def __init__(self, base_type, is_const=0, is_volatile=0): + self.cv_base_type = base_type + self.is_const = is_const + self.is_volatile = is_volatile + if base_type.has_attributes and base_type.scope is not None: + from .Symtab import CConstOrVolatileScope + self.scope = CConstOrVolatileScope(base_type.scope, is_const, is_volatile) + + def cv_string(self): + cvstring = "" + if self.is_const: + cvstring = "const " + cvstring + if self.is_volatile: + cvstring = "volatile " + cvstring + return cvstring def __repr__(self): - return "<CConstType %s>" % repr(self.const_base_type) + return "<CConstOrVolatileType %s%r>" % (self.cv_string(), self.cv_base_type) def __str__(self): return self.declaration_code("", for_display=1) def declaration_code(self, entity_code, for_display = 0, dll_linkage = None, pyrex = 0): + cv = self.cv_string() if for_display or pyrex: - return "const " + self.const_base_type.declaration_code(entity_code, for_display, dll_linkage, pyrex) + return cv + self.cv_base_type.declaration_code(entity_code, for_display, dll_linkage, pyrex) else: - return self.const_base_type.declaration_code("const %s" % entity_code, for_display, dll_linkage, pyrex) + return self.cv_base_type.declaration_code(cv + entity_code, for_display, dll_linkage, pyrex) def specialize(self, values): - base_type = self.const_base_type.specialize(values) - if base_type == self.const_base_type: + base_type = self.cv_base_type.specialize(values) + if base_type == self.cv_base_type: return self - else: - return CConstType(base_type) + return CConstOrVolatileType(base_type, + self.is_const, self.is_volatile) def deduce_template_params(self, actual): - return self.const_base_type.deduce_template_params(actual) + return self.cv_base_type.deduce_template_params(actual) def can_coerce_to_pyobject(self, env): - return self.const_base_type.can_coerce_to_pyobject(env) + return self.cv_base_type.can_coerce_to_pyobject(env) def can_coerce_from_pyobject(self, env): - return self.const_base_type.can_coerce_from_pyobject(env) + return self.cv_base_type.can_coerce_from_pyobject(env) def create_to_py_utility_code(self, env): - if self.const_base_type.create_to_py_utility_code(env): - self.to_py_function = self.const_base_type.to_py_function + if self.cv_base_type.create_to_py_utility_code(env): + self.to_py_function = self.cv_base_type.to_py_function return True def same_as_resolved_type(self, other_type): - if other_type.is_const: - return self.const_base_type.same_as_resolved_type(other_type.const_base_type) - # Accept const LHS <- non-const RHS. - return self.const_base_type.same_as_resolved_type(other_type) + if other_type.is_cv_qualified: + return self.cv_base_type.same_as_resolved_type(other_type.cv_base_type) + # Accept cv LHS <- non-cv RHS. + return self.cv_base_type.same_as_resolved_type(other_type) def __getattr__(self, name): - return getattr(self.const_base_type, name) + return getattr(self.cv_base_type, name) + + +def CConstType(base_type): + return CConstOrVolatileType(base_type, is_const=1) class FusedType(CType): @@ -1634,7 +1849,27 @@ class FusedType(CType): for t in types: if t.is_fused: # recursively merge in subtypes - for subtype in t.types: + if isinstance(t, FusedType): + t_types = t.types + else: + # handle types that aren't a fused type themselves but contain fused types + # for example a C++ template where the template type is fused. + t_fused_types = t.get_fused_types() + t_types = [] + for substitution in product( + *[fused_type.types for fused_type in t_fused_types] + ): + t_types.append( + t.specialize( + { + fused_type: sub + for fused_type, sub in zip( + t_fused_types, substitution + ) + } + ) + ) + for subtype in t_types: if subtype not in flattened_types: flattened_types.append(subtype) elif t not in flattened_types: @@ -1653,9 +1888,12 @@ class FusedType(CType): return 'FusedType(name=%r)' % self.name def specialize(self, values): - return values[self] + if self in values: + return values[self] + else: + raise CannotSpecialize() - def get_fused_types(self, result=None, seen=None): + def get_fused_types(self, result=None, seen=None, include_function_return_type=False): if result is None: return [self] @@ -1738,6 +1976,7 @@ class CNumericType(CType): base_code = type_name.replace('PY_LONG_LONG', 'long long') else: base_code = public_decl(type_name, dll_linkage) + base_code = StringEncoding.EncodedString(base_code) return self.base_declaration_code(base_code, entity_code) def attributes_known(self): @@ -1807,8 +2046,11 @@ class CIntLike(object): self.from_py_function = "__Pyx_PyInt_As_" + self.specialization_name() env.use_utility_code(TempitaUtilityCode.load_cached( "CIntFromPy", "TypeConversion.c", - context={"TYPE": self.empty_declaration_code(), - "FROM_PY_FUNCTION": self.from_py_function})) + context={ + "TYPE": self.empty_declaration_code(), + "FROM_PY_FUNCTION": self.from_py_function, + "IS_ENUM": self.is_enum, + })) return True @staticmethod @@ -2012,7 +2254,7 @@ class CPyUCS4IntType(CIntType): # is 0..1114111, which is checked when converting from an integer # value. - to_py_function = "PyUnicode_FromOrdinal" + to_py_function = "__Pyx_PyUnicode_FromOrdinal" from_py_function = "__Pyx_PyObject_AsPy_UCS4" def can_coerce_to_pystring(self, env, format_spec=None): @@ -2036,7 +2278,7 @@ class CPyUnicodeIntType(CIntType): # Py_UNICODE is 0..1114111, which is checked when converting from # an integer value. - to_py_function = "PyUnicode_FromOrdinal" + to_py_function = "__Pyx_PyUnicode_FromOrdinal" from_py_function = "__Pyx_PyObject_AsPy_UNICODE" def can_coerce_to_pystring(self, env, format_spec=None): @@ -2110,14 +2352,27 @@ class CFloatType(CNumericType): class CComplexType(CNumericType): is_complex = 1 - to_py_function = "__pyx_PyComplex_FromComplex" has_attributes = 1 scope = None + @property + def to_py_function(self): + return "__pyx_PyComplex_FromComplex%s" % self.implementation_suffix + def __init__(self, real_type): while real_type.is_typedef and not real_type.typedef_is_external: real_type = real_type.typedef_base_type self.funcsuffix = "_%s" % real_type.specialization_name() + if not real_type.is_float: + # neither C nor C++ supports non-floating complex numbers, + # so fall back the on Cython implementation. + self.implementation_suffix = "_Cy" + elif real_type.is_typedef and real_type.typedef_is_external: + # C can't handle typedefs in complex numbers, + # so in this case also fall back on the Cython implementation. + self.implementation_suffix = "_CyTypedef" + else: + self.implementation_suffix = "" if real_type.is_float: self.math_h_modifier = real_type.math_h_modifier else: @@ -2170,8 +2425,8 @@ class CComplexType(CNumericType): def assignable_from(self, src_type): # Temporary hack/feature disabling, see #441 if (not src_type.is_complex and src_type.is_numeric and src_type.is_typedef - and src_type.typedef_is_external): - return False + and src_type.typedef_is_external): + return False elif src_type.is_pyobject: return True else: @@ -2179,8 +2434,8 @@ class CComplexType(CNumericType): def assignable_from_resolved_type(self, src_type): return (src_type.is_complex and self.real_type.assignable_from_resolved_type(src_type.real_type) - or src_type.is_numeric and self.real_type.assignable_from_resolved_type(src_type) - or src_type is error_type) + or src_type.is_numeric and self.real_type.assignable_from_resolved_type(src_type) + or src_type is error_type) def attributes_known(self): if self.scope is None: @@ -2209,18 +2464,23 @@ class CComplexType(CNumericType): 'real_type': self.real_type.empty_declaration_code(), 'func_suffix': self.funcsuffix, 'm': self.math_h_modifier, - 'is_float': int(self.real_type.is_float) + 'is_float': int(self.real_type.is_float), + 'is_extern_float_typedef': int( + self.real_type.is_float and self.real_type.is_typedef and self.real_type.typedef_is_external) } def create_declaration_utility_code(self, env): # This must always be run, because a single CComplexType instance can be shared # across multiple compilations (the one created in the module scope) - env.use_utility_code(UtilityCode.load_cached('Header', 'Complex.c')) - env.use_utility_code(UtilityCode.load_cached('RealImag', 'Complex.c')) + if self.real_type.is_float: + env.use_utility_code(UtilityCode.load_cached('Header', 'Complex.c')) + utility_code_context = self._utility_code_context() + env.use_utility_code(UtilityCode.load_cached( + 'RealImag' + self.implementation_suffix, 'Complex.c')) env.use_utility_code(TempitaUtilityCode.load_cached( - 'Declarations', 'Complex.c', self._utility_code_context())) + 'Declarations', 'Complex.c', utility_code_context)) env.use_utility_code(TempitaUtilityCode.load_cached( - 'Arithmetic', 'Complex.c', self._utility_code_context())) + 'Arithmetic', 'Complex.c', utility_code_context)) return True def can_coerce_to_pyobject(self, env): @@ -2230,7 +2490,8 @@ class CComplexType(CNumericType): return True def create_to_py_utility_code(self, env): - env.use_utility_code(UtilityCode.load_cached('ToPy', 'Complex.c')) + env.use_utility_code(TempitaUtilityCode.load_cached( + 'ToPy', 'Complex.c', self._utility_code_context())) return True def create_from_py_utility_code(self, env): @@ -2263,6 +2524,12 @@ class CComplexType(CNumericType): def cast_code(self, expr_code): return expr_code + def real_code(self, expr_code): + return "__Pyx_CREAL%s(%s)" % (self.implementation_suffix, expr_code) + + def imag_code(self, expr_code): + return "__Pyx_CIMAG%s(%s)" % (self.implementation_suffix, expr_code) + complex_ops = { (1, '-'): 'neg', (1, 'zero'): 'is_zero', @@ -2275,6 +2542,42 @@ complex_ops = { } +class SoftCComplexType(CComplexType): + """ + a**b in Python can return either a complex or a float + depending on the sign of a. This "soft complex" type is + stored as a C complex (and so is a little slower than a + direct C double) but it prints/coerces to a float if + the imaginary part is 0. Therefore it provides a C + representation of the Python behaviour. + """ + + to_py_function = "__pyx_Py_FromSoftComplex" + + def __init__(self): + super(SoftCComplexType, self).__init__(c_double_type) + + def declaration_code(self, entity_code, for_display=0, dll_linkage=None, pyrex=0): + base_result = super(SoftCComplexType, self).declaration_code( + entity_code, + for_display=for_display, + dll_linkage=dll_linkage, + pyrex=pyrex, + ) + if for_display: + return "soft %s" % base_result + else: + return base_result + + def create_to_py_utility_code(self, env): + env.use_utility_code(UtilityCode.load_cached('SoftComplexToPy', 'Complex.c')) + return True + + def __repr__(self): + result = super(SoftCComplexType, self).__repr__() + assert result[-1] == ">" + return "%s (soft)%s" % (result[:-1], result[-1]) + class CPyTSSTType(CType): # # PEP-539 "Py_tss_t" type @@ -2303,8 +2606,8 @@ class CPointerBaseType(CType): def __init__(self, base_type): self.base_type = base_type - if base_type.is_const: - base_type = base_type.const_base_type + if base_type.is_cv_qualified: + base_type = base_type.cv_base_type for char_type in (c_char_type, c_uchar_type, c_schar_type): if base_type.same_as(char_type): self.is_string = 1 @@ -2347,6 +2650,7 @@ class CPointerBaseType(CType): if self.is_string: assert isinstance(value, str) return '"%s"' % StringEncoding.escape_byte_string(value) + return str(value) class CArrayType(CPointerBaseType): @@ -2366,7 +2670,7 @@ class CArrayType(CPointerBaseType): return False def __hash__(self): - return hash(self.base_type) + 28 # arbitrarily chosen offset + return hash(self.base_type) + 28 # arbitrarily chosen offset def __repr__(self): return "<CArrayType %s %s>" % (self.size, repr(self.base_type)) @@ -2483,13 +2787,21 @@ class CArrayType(CPointerBaseType): return True def from_py_call_code(self, source_code, result_code, error_pos, code, - from_py_function=None, error_condition=None): + from_py_function=None, error_condition=None, + special_none_cvalue=None): assert not error_condition, '%s: %s' % (error_pos, error_condition) + assert not special_none_cvalue, '%s: %s' % (error_pos, special_none_cvalue) # not currently supported call_code = "%s(%s, %s, %s)" % ( from_py_function or self.from_py_function, source_code, result_code, self.size) return code.error_goto_if_neg(call_code, error_pos) + def error_condition(self, result_code): + # It isn't possible to use CArrays as return type so the error_condition + # is irrelevant. Returning a falsy value does avoid an error when getting + # from_py_call_code from a typedef. + return "" + class CPtrType(CPointerBaseType): # base_type CType Reference type @@ -2498,7 +2810,7 @@ class CPtrType(CPointerBaseType): default_value = "0" def __hash__(self): - return hash(self.base_type) + 27 # arbitrarily chosen offset + return hash(self.base_type) + 27 # arbitrarily chosen offset def __eq__(self, other): if isinstance(other, CType) and other.is_ptr: @@ -2528,8 +2840,8 @@ class CPtrType(CPointerBaseType): return 1 if other_type.is_null_ptr: return 1 - if self.base_type.is_const: - self = CPtrType(self.base_type.const_base_type) + if self.base_type.is_cv_qualified: + self = CPtrType(self.base_type.cv_base_type) if self.base_type.is_cfunction: if other_type.is_ptr: other_type = other_type.base_type.resolve() @@ -2565,32 +2877,30 @@ class CPtrType(CPointerBaseType): return self.base_type.find_cpp_operation_type(operator, operand_type) return None + def get_fused_types(self, result=None, seen=None, include_function_return_type=False): + # For function pointers, include the return type - unlike for fused functions themselves, + # where the return type cannot be an independent fused type (i.e. is derived or non-fused). + return super(CPointerBaseType, self).get_fused_types(result, seen, include_function_return_type=True) + class CNullPtrType(CPtrType): is_null_ptr = 1 -class CReferenceType(BaseType): +class CReferenceBaseType(BaseType): - is_reference = 1 is_fake_reference = 0 + # Common base type for C reference and C++ rvalue reference types. + + subtypes = ['ref_base_type'] + def __init__(self, base_type): self.ref_base_type = base_type def __repr__(self): - return "<CReferenceType %s>" % repr(self.ref_base_type) - - def __str__(self): - return "%s &" % self.ref_base_type - - def declaration_code(self, entity_code, - for_display = 0, dll_linkage = None, pyrex = 0): - #print "CReferenceType.declaration_code: pointer to", self.base_type ### - return self.ref_base_type.declaration_code( - "&%s" % entity_code, - for_display, dll_linkage, pyrex) + return "<%r %s>" % (self.__class__.__name__, self.ref_base_type) def specialize(self, values): base_type = self.ref_base_type.specialize(values) @@ -2606,13 +2916,25 @@ class CReferenceType(BaseType): return getattr(self.ref_base_type, name) +class CReferenceType(CReferenceBaseType): + + is_reference = 1 + + def __str__(self): + return "%s &" % self.ref_base_type + + def declaration_code(self, entity_code, + for_display = 0, dll_linkage = None, pyrex = 0): + #print "CReferenceType.declaration_code: pointer to", self.base_type ### + return self.ref_base_type.declaration_code( + "&%s" % entity_code, + for_display, dll_linkage, pyrex) + + class CFakeReferenceType(CReferenceType): is_fake_reference = 1 - def __repr__(self): - return "<CFakeReferenceType %s>" % repr(self.ref_base_type) - def __str__(self): return "%s [&]" % self.ref_base_type @@ -2622,6 +2944,20 @@ class CFakeReferenceType(CReferenceType): return "__Pyx_FakeReference<%s> %s" % (self.ref_base_type.empty_declaration_code(), entity_code) +class CppRvalueReferenceType(CReferenceBaseType): + + is_rvalue_reference = 1 + + def __str__(self): + return "%s &&" % self.ref_base_type + + def declaration_code(self, entity_code, + for_display = 0, dll_linkage = None, pyrex = 0): + return self.ref_base_type.declaration_code( + "&&%s" % entity_code, + for_display, dll_linkage, pyrex) + + class CFuncType(CType): # return_type CType # args [CFuncTypeArg] @@ -2639,12 +2975,14 @@ class CFuncType(CType): # (used for optimisation overrides) # is_const_method boolean # is_static_method boolean + # op_arg_struct CPtrType Pointer to optional argument struct is_cfunction = 1 original_sig = None cached_specialized_types = None from_fused = False is_const_method = False + op_arg_struct = None subtypes = ['return_type', 'args'] @@ -2793,8 +3131,8 @@ class CFuncType(CType): # is performed elsewhere). for i in range(as_cmethod, len(other_type.args)): if not self.args[i].type.same_as( - other_type.args[i].type): - return 0 + other_type.args[i].type): + return 0 if self.has_varargs != other_type.has_varargs: return 0 if not self.return_type.subtype_of_resolved_type(other_type.return_type): @@ -2814,6 +3152,9 @@ class CFuncType(CType): # must catch C++ exceptions if we raise them return 0 if not other_type.exception_check or other_type.exception_value is not None: + # There's no problem if this type doesn't emit exceptions but the other type checks + if other_type.exception_check and not (self.exception_check or self.exception_value): + return 1 # if other does not *always* check exceptions, self must comply if not self._same_exception_value(other_type.exception_value): return 0 @@ -2899,8 +3240,10 @@ class CFuncType(CType): if (pyrex or for_display) and not self.return_type.is_pyobject: if self.exception_value and self.exception_check: trailer = " except? %s" % self.exception_value - elif self.exception_value: + elif self.exception_value and not self.exception_check: trailer = " except %s" % self.exception_value + elif not self.exception_value and not self.exception_check: + trailer = " noexcept" elif self.exception_check == '+': trailer = " except +" elif self.exception_check and for_display: @@ -3021,10 +3364,13 @@ class CFuncType(CType): return result - def get_fused_types(self, result=None, seen=None, subtypes=None): + def get_fused_types(self, result=None, seen=None, subtypes=None, include_function_return_type=False): """Return fused types in the order they appear as parameter types""" - return super(CFuncType, self).get_fused_types(result, seen, - subtypes=['args']) + return super(CFuncType, self).get_fused_types( + result, seen, + # for function pointer types, we consider the result type; for plain function + # types we don't (because it must be derivable from the arguments) + subtypes=self.subtypes if include_function_return_type else ['args']) def specialize_entry(self, entry, cname): assert not self.is_fused @@ -3050,8 +3396,16 @@ class CFuncType(CType): if not self.can_coerce_to_pyobject(env): return False from .UtilityCode import CythonUtilityCode - safe_typename = re.sub('[^a-zA-Z0-9]', '__', self.declaration_code("", pyrex=1)) - to_py_function = "__Pyx_CFunc_%s_to_py" % safe_typename + + # include argument names into the c function name to ensure cname is unique + # between functions with identical types but different argument names + from .Symtab import punycodify_name + def arg_name_part(arg): + return "%s%s" % (len(arg.name), punycodify_name(arg.name)) if arg.name else "0" + arg_names = [ arg_name_part(arg) for arg in self.args ] + arg_names = "_".join(arg_names) + safe_typename = type_identifier(self, pyrex=True) + to_py_function = "__Pyx_CFunc_%s_to_py_%s" % (safe_typename, arg_names) for arg in self.args: if not arg.type.is_pyobject and not arg.type.create_from_py_utility_code(env): @@ -3243,7 +3597,7 @@ class CFuncTypeArg(BaseType): self.annotation = annotation self.type = type self.pos = pos - self.needs_type_test = False # TODO: should these defaults be set in analyse_types()? + self.needs_type_test = False # TODO: should these defaults be set in analyse_types()? def __repr__(self): return "%s:%s" % (self.name, repr(self.type)) @@ -3254,6 +3608,12 @@ class CFuncTypeArg(BaseType): def specialize(self, values): return CFuncTypeArg(self.name, self.type.specialize(values), self.pos, self.cname) + def is_forwarding_reference(self): + if self.type.is_rvalue_reference: + if (isinstance(self.type.ref_base_type, TemplatePlaceholderType) + and not self.type.ref_base_type.is_cv_qualified): + return True + return False class ToPyStructUtilityCode(object): @@ -3321,7 +3681,7 @@ class CStructOrUnionType(CType): has_attributes = 1 exception_check = True - def __init__(self, name, kind, scope, typedef_flag, cname, packed=False): + def __init__(self, name, kind, scope, typedef_flag, cname, packed=False, in_cpp=False): self.name = name self.cname = cname self.kind = kind @@ -3336,6 +3696,7 @@ class CStructOrUnionType(CType): self._convert_to_py_code = None self._convert_from_py_code = None self.packed = packed + self.needs_cpp_construction = self.is_struct and in_cpp def can_coerce_to_pyobject(self, env): if self._convert_to_py_code is False: @@ -3413,6 +3774,7 @@ class CStructOrUnionType(CType): var_entries=self.scope.var_entries, funcname=self.from_py_function, ) + env.use_utility_code(UtilityCode.load_cached("RaiseUnexpectedTypeError", "ObjectHandling.c")) from .UtilityCode import CythonUtilityCode self._convert_from_py_code = CythonUtilityCode.load( "FromPyStructUtility" if self.is_struct else "FromPyUnionUtility", @@ -3505,6 +3867,7 @@ class CppClassType(CType): is_cpp_class = 1 has_attributes = 1 + needs_cpp_construction = 1 exception_check = True namespace = None @@ -3578,10 +3941,12 @@ class CppClassType(CType): 'maybe_unordered': self.maybe_unordered(), 'type': self.cname, }) + # Override directives that should not be inherited from user code. from .UtilityCode import CythonUtilityCode + directives = CythonUtilityCode.filter_inherited_directives(env.directives) env.use_utility_code(CythonUtilityCode.load( cls.replace('unordered_', '') + ".from_py", "CppConvert.pyx", - context=context, compiler_directives=env.directives)) + context=context, compiler_directives=directives)) self.from_py_function = cname return True @@ -3624,16 +3989,18 @@ class CppClassType(CType): 'type': self.cname, }) from .UtilityCode import CythonUtilityCode + # Override directives that should not be inherited from user code. + directives = CythonUtilityCode.filter_inherited_directives(env.directives) env.use_utility_code(CythonUtilityCode.load( cls.replace('unordered_', '') + ".to_py", "CppConvert.pyx", - context=context, compiler_directives=env.directives)) + context=context, compiler_directives=directives)) self.to_py_function = cname return True def is_template_type(self): return self.templates is not None and self.template_type is None - def get_fused_types(self, result=None, seen=None): + def get_fused_types(self, result=None, seen=None, include_function_return_type=False): if result is None: result = [] seen = set() @@ -3644,7 +4011,7 @@ class CppClassType(CType): T.get_fused_types(result, seen) return result - def specialize_here(self, pos, template_values=None): + def specialize_here(self, pos, env, template_values=None): if not self.is_template_type(): error(pos, "'%s' type is not a template" % self) return error_type @@ -3669,10 +4036,12 @@ class CppClassType(CType): return error_type has_object_template_param = False for value in template_values: - if value.is_pyobject: + if value.is_pyobject or value.needs_refcounting: has_object_template_param = True + type_description = "Python object" if value.is_pyobject else "Reference-counted" error(pos, - "Python object type '%s' cannot be used as a template argument" % value) + "%s type '%s' cannot be used as a template argument" % ( + type_description, value)) if has_object_template_param: return error_type return self.specialize(dict(zip(self.templates, template_values))) @@ -3695,23 +4064,23 @@ class CppClassType(CType): specialized.namespace = self.namespace.specialize(values) specialized.scope = self.scope.specialize(values, specialized) if self.cname == 'std::vector': - # vector<bool> is special cased in the C++ standard, and its - # accessors do not necessarily return references to the underlying - # elements (which may be bit-packed). - # http://www.cplusplus.com/reference/vector/vector-bool/ - # Here we pretend that the various methods return bool values - # (as the actual returned values are coercable to such, and - # we don't support call expressions as lvalues). - T = values.get(self.templates[0], None) - if T and not T.is_fused and T.empty_declaration_code() == 'bool': - for bit_ref_returner in ('at', 'back', 'front'): - if bit_ref_returner in specialized.scope.entries: - specialized.scope.entries[bit_ref_returner].type.return_type = T + # vector<bool> is special cased in the C++ standard, and its + # accessors do not necessarily return references to the underlying + # elements (which may be bit-packed). + # http://www.cplusplus.com/reference/vector/vector-bool/ + # Here we pretend that the various methods return bool values + # (as the actual returned values are coercable to such, and + # we don't support call expressions as lvalues). + T = values.get(self.templates[0], None) + if T and not T.is_fused and T.empty_declaration_code() == 'bool': + for bit_ref_returner in ('at', 'back', 'front'): + if bit_ref_returner in specialized.scope.entries: + specialized.scope.entries[bit_ref_returner].type.return_type = T return specialized def deduce_template_params(self, actual): - if actual.is_const: - actual = actual.const_base_type + if actual.is_cv_qualified: + actual = actual.cv_base_type if actual.is_reference: actual = actual.ref_base_type if self == actual: @@ -3765,6 +4134,12 @@ class CppClassType(CType): base_code = public_decl(base_code, dll_linkage) return self.base_declaration_code(base_code, entity_code) + def cpp_optional_declaration_code(self, entity_code, dll_linkage=None, template_params=None): + return "__Pyx_Optional_Type<%s> %s" % ( + self.declaration_code("", False, dll_linkage, False, + template_params), + entity_code) + def is_subclass(self, other_type): if self.same_as_resolved_type(other_type): return 1 @@ -3847,6 +4222,101 @@ class CppClassType(CType): if constructor is not None and best_match([], constructor.all_alternatives()) is None: error(pos, "C++ class must have a nullary constructor to be %s" % msg) + def cpp_optional_check_for_null_code(self, cname): + # only applies to c++ classes that are being declared as std::optional + return "(%s.has_value())" % cname + + +class CppScopedEnumType(CType): + # name string + # doc string or None + # cname string + + is_cpp_enum = True + + def __init__(self, name, cname, underlying_type, namespace=None, doc=None): + self.name = name + self.doc = doc + self.cname = cname + self.values = [] + self.underlying_type = underlying_type + self.namespace = namespace + + def __str__(self): + return self.name + + def declaration_code(self, entity_code, + for_display=0, dll_linkage=None, pyrex=0): + if pyrex or for_display: + type_name = self.name + else: + if self.namespace: + type_name = "%s::%s" % ( + self.namespace.empty_declaration_code(), + self.cname + ) + else: + type_name = "__PYX_ENUM_CLASS_DECL %s" % self.cname + type_name = public_decl(type_name, dll_linkage) + return self.base_declaration_code(type_name, entity_code) + + def create_from_py_utility_code(self, env): + if self.from_py_function: + return True + if self.underlying_type.create_from_py_utility_code(env): + self.from_py_function = '(%s)%s' % ( + self.cname, self.underlying_type.from_py_function + ) + return True + + def create_to_py_utility_code(self, env): + if self.to_py_function is not None: + return True + if self.entry.create_wrapper: + from .UtilityCode import CythonUtilityCode + self.to_py_function = "__Pyx_Enum_%s_to_py" % self.name + if self.entry.scope != env.global_scope(): + module_name = self.entry.scope.qualified_name + else: + module_name = None + env.use_utility_code(CythonUtilityCode.load( + "EnumTypeToPy", "CpdefEnums.pyx", + context={"funcname": self.to_py_function, + "name": self.name, + "items": tuple(self.values), + "underlying_type": self.underlying_type.empty_declaration_code(), + "module_name": module_name, + "is_flag": False, + }, + outer_module_scope=self.entry.scope # ensure that "name" is findable + )) + return True + if self.underlying_type.create_to_py_utility_code(env): + # Using a C++11 lambda here, which is fine since + # scoped enums are a C++11 feature + self.to_py_function = '[](const %s& x){return %s((%s)x);}' % ( + self.cname, + self.underlying_type.to_py_function, + self.underlying_type.empty_declaration_code() + ) + return True + + def create_type_wrapper(self, env): + from .UtilityCode import CythonUtilityCode + rst = CythonUtilityCode.load( + "CppScopedEnumType", "CpdefEnums.pyx", + context={ + "name": self.name, + "cname": self.cname.split("::")[-1], + "items": tuple(self.values), + "underlying_type": self.underlying_type.empty_declaration_code(), + "enum_doc": self.doc, + "static_modname": env.qualified_name, + }, + outer_module_scope=env.global_scope()) + + env.use_utility_code(rst) + class TemplatePlaceholderType(CType): @@ -3897,16 +4367,18 @@ def is_optional_template_param(type): class CEnumType(CIntLike, CType): # name string + # doc string or None # cname string or None # typedef_flag boolean # values [string], populated during declaration analysis is_enum = 1 signed = 1 - rank = -1 # Ranks below any integer type + rank = -1 # Ranks below any integer type - def __init__(self, name, cname, typedef_flag, namespace=None): + def __init__(self, name, cname, typedef_flag, namespace=None, doc=None): self.name = name + self.doc = doc self.cname = cname self.values = [] self.typedef_flag = typedef_flag @@ -3945,12 +4417,47 @@ class CEnumType(CIntLike, CType): def create_type_wrapper(self, env): from .UtilityCode import CythonUtilityCode + # Generate "int"-like conversion function + old_to_py_function = self.to_py_function + self.to_py_function = None + CIntLike.create_to_py_utility_code(self, env) + enum_to_pyint_func = self.to_py_function + self.to_py_function = old_to_py_function # we don't actually want to overwrite this + env.use_utility_code(CythonUtilityCode.load( "EnumType", "CpdefEnums.pyx", context={"name": self.name, - "items": tuple(self.values)}, + "items": tuple(self.values), + "enum_doc": self.doc, + "enum_to_pyint_func": enum_to_pyint_func, + "static_modname": env.qualified_name, + }, outer_module_scope=env.global_scope())) + def create_to_py_utility_code(self, env): + if self.to_py_function is not None: + return self.to_py_function + if not self.entry.create_wrapper: + return super(CEnumType, self).create_to_py_utility_code(env) + from .UtilityCode import CythonUtilityCode + self.to_py_function = "__Pyx_Enum_%s_to_py" % self.name + if self.entry.scope != env.global_scope(): + module_name = self.entry.scope.qualified_name + else: + module_name = None + env.use_utility_code(CythonUtilityCode.load( + "EnumTypeToPy", "CpdefEnums.pyx", + context={"funcname": self.to_py_function, + "name": self.name, + "items": tuple(self.values), + "underlying_type": "int", + "module_name": module_name, + "is_flag": True, + }, + outer_module_scope=self.entry.scope # ensure that "name" is findable + )) + return True + class CTupleType(CType): # components [PyrexType] @@ -3966,6 +4473,9 @@ class CTupleType(CType): self.exception_check = True self._convert_to_py_code = None self._convert_from_py_code = None + # equivalent_type must be set now because it isn't available at import time + from .Builtin import tuple_type + self.equivalent_type = tuple_type def __str__(self): return "(%s)" % ", ".join(str(c) for c in self.components) @@ -3973,7 +4483,7 @@ class CTupleType(CType): def declaration_code(self, entity_code, for_display = 0, dll_linkage = None, pyrex = 0): if pyrex or for_display: - return str(self) + return "%s %s" % (str(self), entity_code) else: return self.base_declaration_code(self.cname, entity_code) @@ -4085,21 +4595,91 @@ class ErrorType(PyrexType): return "dummy" +class PythonTypeConstructorMixin(object): + """Used to help Cython interpret indexed types from the typing module (or similar) + """ + modifier_name = None + + def set_python_type_constructor_name(self, name): + self.python_type_constructor_name = name + + def specialize_here(self, pos, env, template_values=None): + # for a lot of the typing classes it doesn't really matter what the template is + # (i.e. typing.Dict[int] is really just a dict) + return self + + def __repr__(self): + if self.base_type: + return "%s[%r]" % (self.name, self.base_type) + else: + return self.name + + def is_template_type(self): + return True + + +class BuiltinTypeConstructorObjectType(BuiltinObjectType, PythonTypeConstructorMixin): + """ + builtin types like list, dict etc which can be subscripted in annotations + """ + def __init__(self, name, cname, objstruct_cname=None): + super(BuiltinTypeConstructorObjectType, self).__init__( + name, cname, objstruct_cname=objstruct_cname) + self.set_python_type_constructor_name(name) + + +class PythonTupleTypeConstructor(BuiltinTypeConstructorObjectType): + def specialize_here(self, pos, env, template_values=None): + if (template_values and None not in template_values and + not any(v.is_pyobject for v in template_values)): + entry = env.declare_tuple_type(pos, template_values) + if entry: + entry.used = True + return entry.type + return super(PythonTupleTypeConstructor, self).specialize_here(pos, env, template_values) + + +class SpecialPythonTypeConstructor(PyObjectType, PythonTypeConstructorMixin): + """ + For things like ClassVar, Optional, etc, which are not types and disappear during type analysis. + """ + + def __init__(self, name): + super(SpecialPythonTypeConstructor, self).__init__() + self.set_python_type_constructor_name(name) + self.modifier_name = name + + def __repr__(self): + return self.name + + def resolve(self): + return self + + def specialize_here(self, pos, env, template_values=None): + if len(template_values) != 1: + error(pos, "'%s' takes exactly one template argument." % self.name) + return error_type + if template_values[0] is None: + # FIXME: allowing unknown types for now since we don't recognise all Python types. + return None + # Replace this type with the actual 'template' argument. + return template_values[0].resolve() + + rank_to_type_name = ( - "char", # 0 - "short", # 1 - "int", # 2 - "long", # 3 - "PY_LONG_LONG", # 4 - "float", # 5 - "double", # 6 - "long double", # 7 + "char", # 0 + "short", # 1 + "int", # 2 + "long", # 3 + "PY_LONG_LONG", # 4 + "float", # 5 + "double", # 6 + "long double", # 7 ) -_rank_to_type_name = list(rank_to_type_name) -RANK_INT = _rank_to_type_name.index('int') -RANK_LONG = _rank_to_type_name.index('long') -RANK_FLOAT = _rank_to_type_name.index('float') +RANK_INT = rank_to_type_name.index('int') +RANK_LONG = rank_to_type_name.index('long') +RANK_FLOAT = rank_to_type_name.index('float') UNSIGNED = 0 SIGNED = 2 @@ -4136,6 +4716,8 @@ c_float_complex_type = CComplexType(c_float_type) c_double_complex_type = CComplexType(c_double_type) c_longdouble_complex_type = CComplexType(c_longdouble_type) +soft_complex_type = SoftCComplexType() + c_anon_enum_type = CAnonEnumType(-1) c_returncode_type = CReturnCodeType(RANK_INT) c_bint_type = CBIntType(RANK_INT) @@ -4303,8 +4885,7 @@ def best_match(arg_types, functions, pos=None, env=None, args=None): # Check no. of args max_nargs = len(func_type.args) min_nargs = max_nargs - func_type.optional_arg_count - if actual_nargs < min_nargs or \ - (not func_type.has_varargs and actual_nargs > max_nargs): + if actual_nargs < min_nargs or (not func_type.has_varargs and actual_nargs > max_nargs): if max_nargs == min_nargs and not func_type.has_varargs: expectation = max_nargs elif actual_nargs < min_nargs: @@ -4316,12 +4897,23 @@ def best_match(arg_types, functions, pos=None, env=None, args=None): errors.append((func, error_mesg)) continue if func_type.templates: + # For any argument/parameter pair A/P, if P is a forwarding reference, + # use lvalue-reference-to-A for deduction in place of A when the + # function call argument is an lvalue. See: + # https://en.cppreference.com/w/cpp/language/template_argument_deduction#Deduction_from_a_function_call + arg_types_for_deduction = list(arg_types) + if func.type.is_cfunction and args: + for i, formal_arg in enumerate(func.type.args): + if formal_arg.is_forwarding_reference(): + if args[i].is_lvalue(): + arg_types_for_deduction[i] = c_ref_type(arg_types[i]) deductions = reduce( merge_template_deductions, - [pattern.type.deduce_template_params(actual) for (pattern, actual) in zip(func_type.args, arg_types)], + [pattern.type.deduce_template_params(actual) for (pattern, actual) in zip(func_type.args, arg_types_for_deduction)], {}) if deductions is None: - errors.append((func, "Unable to deduce type parameters for %s given (%s)" % (func_type, ', '.join(map(str, arg_types))))) + errors.append((func, "Unable to deduce type parameters for %s given (%s)" % ( + func_type, ', '.join(map(str, arg_types_for_deduction))))) elif len(deductions) < len(func_type.templates): errors.append((func, "Unable to deduce type parameter %s" % ( ", ".join([param.name for param in set(func_type.templates) - set(deductions.keys())])))) @@ -4456,10 +5048,10 @@ def widest_numeric_type(type1, type2): type1 = type1.ref_base_type if type2.is_reference: type2 = type2.ref_base_type - if type1.is_const: - type1 = type1.const_base_type - if type2.is_const: - type2 = type2.const_base_type + if type1.is_cv_qualified: + type1 = type1.cv_base_type + if type2.is_cv_qualified: + type2 = type2.cv_base_type if type1 == type2: widest_type = type1 elif type1.is_complex or type2.is_complex: @@ -4471,6 +5063,14 @@ def widest_numeric_type(type1, type2): widest_numeric_type( real_type(type1), real_type(type2))) + if type1 is soft_complex_type or type2 is soft_complex_type: + type1_is_other_complex = type1 is not soft_complex_type and type1.is_complex + type2_is_other_complex = type2 is not soft_complex_type and type2.is_complex + if (not type1_is_other_complex and not type2_is_other_complex and + widest_type.real_type == soft_complex_type.real_type): + # ensure we can do an actual "is" comparison + # (this possibly goes slightly wrong when mixing long double and soft complex) + widest_type = soft_complex_type elif type1.is_enum and type2.is_enum: widest_type = c_int_type elif type1.rank < type2.rank: @@ -4505,14 +5105,19 @@ def independent_spanning_type(type1, type2): type1 = type1.ref_base_type else: type2 = type2.ref_base_type - if type1 == type2: + + resolved_type1 = type1.resolve() + resolved_type2 = type2.resolve() + if resolved_type1 == resolved_type2: return type1 - elif (type1 is c_bint_type or type2 is c_bint_type) and (type1.is_numeric and type2.is_numeric): + elif ((resolved_type1 is c_bint_type or resolved_type2 is c_bint_type) + and (type1.is_numeric and type2.is_numeric)): # special case: if one of the results is a bint and the other # is another C integer, we must prevent returning a numeric # type so that we do not lose the ability to coerce to a # Python bool if we have to. return py_object_type + span_type = _spanning_type(type1, type2) if span_type is None: return error_type @@ -4649,35 +5254,38 @@ def parse_basic_type(name): name = 'int' return simple_c_type(signed, longness, name) -def c_array_type(base_type, size): - # Construct a C array type. + +def _construct_type_from_base(cls, base_type, *args): if base_type is error_type: return error_type - else: - return CArrayType(base_type, size) + return cls(base_type, *args) + +def c_array_type(base_type, size): + # Construct a C array type. + return _construct_type_from_base(CArrayType, base_type, size) def c_ptr_type(base_type): # Construct a C pointer type. - if base_type is error_type: - return error_type - elif base_type.is_reference: - return CPtrType(base_type.ref_base_type) - else: - return CPtrType(base_type) + if base_type.is_reference: + base_type = base_type.ref_base_type + return _construct_type_from_base(CPtrType, base_type) def c_ref_type(base_type): # Construct a C reference type - if base_type is error_type: - return error_type - else: - return CReferenceType(base_type) + return _construct_type_from_base(CReferenceType, base_type) + +def cpp_rvalue_ref_type(base_type): + # Construct a C++ rvalue reference type + return _construct_type_from_base(CppRvalueReferenceType, base_type) def c_const_type(base_type): # Construct a C const type. - if base_type is error_type: - return error_type - else: - return CConstType(base_type) + return _construct_type_from_base(CConstType, base_type) + +def c_const_or_volatile_type(base_type, is_const, is_volatile): + # Construct a C const/volatile type. + return _construct_type_from_base(CConstOrVolatileType, base_type, is_const, is_volatile) + def same_type(type1, type2): return type1.same_as(type2) @@ -4703,28 +5311,42 @@ def typecast(to_type, from_type, expr_code): def type_list_identifier(types): return cap_length('__and_'.join(type_identifier(type) for type in types)) +_special_type_characters = { + '__': '__dunder', + 'const ': '__const_', + ' ': '__space_', + '*': '__ptr', + '&': '__ref', + '&&': '__fwref', + '[': '__lArr', + ']': '__rArr', + '<': '__lAng', + '>': '__rAng', + '(': '__lParen', + ')': '__rParen', + ',': '__comma_', + '...': '__EL', + '::': '__in_', + ':': '__D', +} + +_escape_special_type_characters = partial(re.compile( + # join substrings in reverse order to put longer matches first, e.g. "::" before ":" + " ?(%s) ?" % "|".join(re.escape(s) for s in sorted(_special_type_characters, reverse=True)) +).sub, lambda match: _special_type_characters[match.group(1)]) + +def type_identifier(type, pyrex=False): + decl = type.empty_declaration_code(pyrex=pyrex) + return type_identifier_from_declaration(decl) + _type_identifier_cache = {} -def type_identifier(type): - decl = type.empty_declaration_code() +def type_identifier_from_declaration(decl): safe = _type_identifier_cache.get(decl) if safe is None: safe = decl safe = re.sub(' +', ' ', safe) - safe = re.sub(' ([^a-zA-Z0-9_])', r'\1', safe) - safe = re.sub('([^a-zA-Z0-9_]) ', r'\1', safe) - safe = (safe.replace('__', '__dunder') - .replace('const ', '__const_') - .replace(' ', '__space_') - .replace('*', '__ptr') - .replace('&', '__ref') - .replace('[', '__lArr') - .replace(']', '__rArr') - .replace('<', '__lAng') - .replace('>', '__rAng') - .replace('(', '__lParen') - .replace(')', '__rParen') - .replace(',', '__comma_') - .replace('::', '__in_')) + safe = re.sub(' ?([^a-zA-Z0-9_]) ?', r'\1', safe) + safe = _escape_special_type_characters(safe) safe = cap_length(re.sub('[^a-zA-Z0-9_]', lambda x: '__%X' % ord(x.group(0)), safe)) _type_identifier_cache[decl] = safe return safe diff --git a/Cython/Compiler/Scanning.pxd b/Cython/Compiler/Scanning.pxd index 59593f88a..2d64565c0 100644 --- a/Cython/Compiler/Scanning.pxd +++ b/Cython/Compiler/Scanning.pxd @@ -1,4 +1,4 @@ -from __future__ import absolute_import +# cython: language_level=3 import cython @@ -9,11 +9,6 @@ cdef unicode any_string_prefix, IDENT cdef get_lexicon() cdef initial_compile_time_env() -cdef class Method: - cdef object name - cdef dict kwargs - cdef readonly object __name__ # for tracing the scanner - ## methods commented with '##' out are used by Parsing.py when compiled. @cython.final @@ -39,10 +34,11 @@ cdef class PyrexScanner(Scanner): cdef public indentation_char cdef public int bracket_nesting_level cdef readonly bint async_enabled - cdef public sy - cdef public systring + cdef public unicode sy + cdef public systring # EncodedString + cdef public list put_back_on_failure - cdef long current_level(self) + cdef Py_ssize_t current_level(self) #cpdef commentline(self, text) #cpdef open_bracket_action(self, text) #cpdef close_bracket_action(self, text) @@ -50,13 +46,12 @@ cdef class PyrexScanner(Scanner): #cpdef begin_string_action(self, text) #cpdef end_string_action(self, text) #cpdef unclosed_string_action(self, text) - @cython.locals(current_level=cython.long, new_level=cython.long) + @cython.locals(current_level=Py_ssize_t, new_level=Py_ssize_t) cpdef indentation_action(self, text) #cpdef eof_action(self, text) ##cdef next(self) ##cdef peek(self) #cpdef put_back(self, sy, systring) - #cdef unread(self, token, value) ##cdef bint expect(self, what, message = *) except -2 ##cdef expect_keyword(self, what, message = *) ##cdef expected(self, what, message = *) @@ -65,3 +60,4 @@ cdef class PyrexScanner(Scanner): ##cdef expect_newline(self, message=*, bint ignore_semicolon=*) ##cdef int enter_async(self) except -1 ##cdef int exit_async(self) except -1 + cdef void error_at_scanpos(self, str message) except * diff --git a/Cython/Compiler/Scanning.py b/Cython/Compiler/Scanning.py index f61144033..d12d9d305 100644 --- a/Cython/Compiler/Scanning.py +++ b/Cython/Compiler/Scanning.py @@ -1,4 +1,4 @@ -# cython: infer_types=True, language_level=3, py2_import=True, auto_pickle=False +# cython: infer_types=True, language_level=3, auto_pickle=False # # Cython Scanner # @@ -12,11 +12,13 @@ cython.declare(make_lexicon=object, lexicon=object, import os import platform +from unicodedata import normalize +from contextlib import contextmanager from .. import Utils from ..Plex.Scanners import Scanner from ..Plex.Errors import UnrecognizedInput -from .Errors import error, warning +from .Errors import error, warning, hold_errors, release_errors, CompileError from .Lexicon import any_string_prefix, make_lexicon, IDENT from .Future import print_function @@ -51,25 +53,6 @@ pyx_reserved_words = py_reserved_words + [ ] -class Method(object): - - def __init__(self, name, **kwargs): - self.name = name - self.kwargs = kwargs or None - self.__name__ = name # for Plex tracing - - def __call__(self, stream, text): - method = getattr(stream, self.name) - # self.kwargs is almost always unused => avoid call overhead - return method(text, **self.kwargs) if self.kwargs is not None else method(text) - - def __copy__(self): - return self # immutable, no need to copy - - def __deepcopy__(self, memo): - return self # immutable, no need to copy - - #------------------------------------------------------------------ class CompileTimeScope(object): @@ -154,7 +137,7 @@ class SourceDescriptor(object): _escaped_description = None _cmp_name = '' def __str__(self): - assert False # To catch all places where a descriptor is used directly as a filename + assert False # To catch all places where a descriptor is used directly as a filename def set_file_type_from_name(self, filename): name, ext = os.path.splitext(filename) @@ -295,7 +278,7 @@ class StringSourceDescriptor(SourceDescriptor): get_error_description = get_description def get_filenametable_entry(self): - return "stringsource" + return "<stringsource>" def __hash__(self): return id(self) @@ -318,6 +301,8 @@ class PyrexScanner(Scanner): # compile_time_env dict Environment for conditional compilation # compile_time_eval boolean In a true conditional compilation context # compile_time_expr boolean In a compile-time expression context + # put_back_on_failure list or None If set, this records states so the tentatively_scan + # contextmanager can restore it def __init__(self, file, filename, parent_scanner=None, scope=None, context=None, source_encoding=None, parse_comments=True, initial_pos=None): @@ -356,10 +341,19 @@ class PyrexScanner(Scanner): self.indentation_char = None self.bracket_nesting_level = 0 + self.put_back_on_failure = None + self.begin('INDENT') self.sy = '' self.next() + def normalize_ident(self, text): + try: + text.encode('ascii') # really just name.isascii but supports Python 2 and 3 + except UnicodeEncodeError: + text = normalize('NFKC', text) + self.produce(IDENT, text) + def commentline(self, text): if self.parse_comments: self.produce('commentline', text) @@ -402,7 +396,7 @@ class PyrexScanner(Scanner): def unclosed_string_action(self, text): self.end_string_action(text) - self.error("Unclosed string literal") + self.error_at_scanpos("Unclosed string literal") def indentation_action(self, text): self.begin('') @@ -418,9 +412,9 @@ class PyrexScanner(Scanner): #print "Scanner.indentation_action: setting indent_char to", repr(c) else: if self.indentation_char != c: - self.error("Mixed use of tabs and spaces") + self.error_at_scanpos("Mixed use of tabs and spaces") if text.replace(c, "") != "": - self.error("Mixed use of tabs and spaces") + self.error_at_scanpos("Mixed use of tabs and spaces") # Figure out how many indents/dedents to do current_level = self.current_level() new_level = len(text) @@ -438,7 +432,7 @@ class PyrexScanner(Scanner): self.produce('DEDENT', '') #print "...current level now", self.current_level() ### if new_level != self.current_level(): - self.error("Inconsistent indentation") + self.error_at_scanpos("Inconsistent indentation") def eof_action(self, text): while len(self.indentation_stack) > 1: @@ -450,7 +444,7 @@ class PyrexScanner(Scanner): try: sy, systring = self.read() except UnrecognizedInput: - self.error("Unrecognized character") + self.error_at_scanpos("Unrecognized character") return # just a marker, error() always raises if sy == IDENT: if systring in self.keywords: @@ -461,9 +455,11 @@ class PyrexScanner(Scanner): else: sy = systring systring = self.context.intern_ustring(systring) + if self.put_back_on_failure is not None: + self.put_back_on_failure.append((sy, systring, self.position())) self.sy = sy self.systring = systring - if False: # debug_scanner: + if False: # debug_scanner: _, line, col = self.position() if not self.systring or self.sy == self.systring: t = self.sy @@ -473,20 +469,20 @@ class PyrexScanner(Scanner): def peek(self): saved = self.sy, self.systring + saved_pos = self.position() self.next() next = self.sy, self.systring - self.unread(*next) + self.unread(self.sy, self.systring, self.position()) self.sy, self.systring = saved + self.last_token_position_tuple = saved_pos return next - def put_back(self, sy, systring): - self.unread(self.sy, self.systring) + def put_back(self, sy, systring, pos): + self.unread(self.sy, self.systring, self.last_token_position_tuple) self.sy = sy self.systring = systring + self.last_token_position_tuple = pos - def unread(self, token, value): - # This method should be added to Plex - self.queue.insert(0, (token, value)) def error(self, message, pos=None, fatal=True): if pos is None: @@ -496,6 +492,12 @@ class PyrexScanner(Scanner): err = error(pos, message) if fatal: raise err + def error_at_scanpos(self, message): + # Like error(fatal=True), but gets the current scanning position rather than + # the position of the last token read. + pos = self.get_current_scan_pos() + self.error(message, pos, True) + def expect(self, what, message=None): if self.sy == what: self.next() @@ -549,3 +551,30 @@ class PyrexScanner(Scanner): self.keywords.discard('async') if self.sy in ('async', 'await'): self.sy, self.systring = IDENT, self.context.intern_ustring(self.sy) + +@contextmanager +@cython.locals(scanner=Scanner) +def tentatively_scan(scanner): + errors = hold_errors() + try: + put_back_on_failure = scanner.put_back_on_failure + scanner.put_back_on_failure = [] + initial_state = (scanner.sy, scanner.systring, scanner.position()) + try: + yield errors + except CompileError as e: + pass + finally: + if errors: + if scanner.put_back_on_failure: + for put_back in reversed(scanner.put_back_on_failure[:-1]): + scanner.put_back(*put_back) + # we need to restore the initial state too + scanner.put_back(*initial_state) + elif put_back_on_failure is not None: + # the outer "tentatively_scan" block that we're in might still + # want to undo this block + put_back_on_failure.extend(scanner.put_back_on_failure) + scanner.put_back_on_failure = put_back_on_failure + finally: + release_errors(ignore=True) diff --git a/Cython/Compiler/StringEncoding.py b/Cython/Compiler/StringEncoding.py index c37e8aab7..192fc3de3 100644 --- a/Cython/Compiler/StringEncoding.py +++ b/Cython/Compiler/StringEncoding.py @@ -138,6 +138,24 @@ class EncodedString(_unicode): def as_utf8_string(self): return bytes_literal(self.utf8encode(), 'utf8') + def as_c_string_literal(self): + # first encodes the string then produces a c string literal + if self.encoding is None: + s = self.as_utf8_string() + else: + s = bytes_literal(self.byteencode(), self.encoding) + return s.as_c_string_literal() + + if not hasattr(_unicode, "isascii"): + def isascii(self): + # not defined for Python3.7+ since the class already has it + try: + self.encode("ascii") + except UnicodeEncodeError: + return False + else: + return True + def string_contains_surrogates(ustring): """ @@ -211,6 +229,11 @@ class BytesLiteral(_bytes): value = split_string_literal(escape_byte_string(self)) return '"%s"' % value + if not hasattr(_bytes, "isascii"): + def isascii(self): + # already defined for Python3.7+ + return True + def bytes_literal(s, encoding): assert isinstance(s, bytes) @@ -226,6 +249,12 @@ def encoded_string(s, encoding): s.encoding = encoding return s +def encoded_string_or_bytes_literal(s, encoding): + if isinstance(s, bytes): + return bytes_literal(s, encoding) + else: + return encoded_string(s, encoding) + char_from_escape_sequence = { r'\a' : u'\a', @@ -291,7 +320,7 @@ def escape_byte_string(s): """ s = _replace_specials(s) try: - return s.decode("ASCII") # trial decoding: plain ASCII => done + return s.decode("ASCII") # trial decoding: plain ASCII => done except UnicodeDecodeError: pass if IS_PYTHON3: @@ -324,7 +353,7 @@ def split_string_literal(s, limit=2000): while start < len(s): end = start + limit if len(s) > end-4 and '\\' in s[end-4:end]: - end -= 4 - s[end-4:end].find('\\') # just before the backslash + end -= 4 - s[end-4:end].find('\\') # just before the backslash while s[end-1] == '\\': end -= 1 if end == start: diff --git a/Cython/Compiler/Symtab.py b/Cython/Compiler/Symtab.py index 7361a55ae..608d68c22 100644 --- a/Cython/Compiler/Symtab.py +++ b/Cython/Compiler/Symtab.py @@ -13,6 +13,7 @@ try: except ImportError: # Py3 import builtins +from ..Utils import try_finally_contextmanager from .Errors import warning, error, InternalError from .StringEncoding import EncodedString from . import Options, Naming @@ -20,18 +21,19 @@ from . import PyrexTypes from .PyrexTypes import py_object_type, unspecified_type from .TypeSlots import ( pyfunction_signature, pymethod_signature, richcmp_special_methods, - get_special_method_signature, get_property_accessor_signature) + get_slot_table, get_property_accessor_signature) from . import Future from . import Code -iso_c99_keywords = set( -['auto', 'break', 'case', 'char', 'const', 'continue', 'default', 'do', +iso_c99_keywords = { + 'auto', 'break', 'case', 'char', 'const', 'continue', 'default', 'do', 'double', 'else', 'enum', 'extern', 'float', 'for', 'goto', 'if', 'int', 'long', 'register', 'return', 'short', 'signed', 'sizeof', 'static', 'struct', 'switch', 'typedef', 'union', 'unsigned', 'void', 'volatile', 'while', - '_Bool', '_Complex'', _Imaginary', 'inline', 'restrict']) + '_Bool', '_Complex'', _Imaginary', 'inline', 'restrict', +} def c_safe_identifier(cname): @@ -42,6 +44,28 @@ def c_safe_identifier(cname): cname = Naming.pyrex_prefix + cname return cname +def punycodify_name(cname, mangle_with=None): + # if passed the mangle_with should be a byte string + # modified from PEP489 + try: + cname.encode('ascii') + except UnicodeEncodeError: + cname = cname.encode('punycode').replace(b'-', b'_').decode('ascii') + if mangle_with: + # sometimes it necessary to mangle unicode names alone where + # they'll be inserted directly into C, because the punycode + # transformation can turn them into invalid identifiers + cname = "%s_%s" % (mangle_with, cname) + elif cname.startswith(Naming.pyrex_prefix): + # a punycode name could also be a valid ascii variable name so + # change the prefix to distinguish + cname = cname.replace(Naming.pyrex_prefix, + Naming.pyunicode_identifier_prefix, 1) + + return cname + + + class BufferAux(object): writable_needed = False @@ -87,6 +111,7 @@ class Entry(object): # doc_cname string or None C const holding the docstring # getter_cname string C func for getting property # setter_cname string C func for setting or deleting property + # is_cproperty boolean Is an inline property of an external type # is_self_arg boolean Is the "self" arg of an exttype method # is_arg boolean Is the arg of a method # is_local boolean Is a local variable @@ -134,6 +159,12 @@ class Entry(object): # cf_used boolean Entry is used # is_fused_specialized boolean Whether this entry of a cdef or def function # is a specialization + # is_cgetter boolean Is a c-level getter function + # is_cpp_optional boolean Entry should be declared as std::optional (cpp_locals directive) + # known_standard_library_import Either None (default), an empty string (definitely can't be determined) + # or a string of "modulename.something.attribute" + # Used for identifying imports from typing/dataclasses etc + # pytyping_modifiers Python type modifiers like "typing.ClassVar" but also "dataclasses.InitVar" # TODO: utility_code and utility_code_definition serves the same purpose... @@ -141,6 +172,7 @@ class Entry(object): borrowed = 0 init = "" annotation = None + pep563_annotation = None visibility = 'private' is_builtin = 0 is_cglobal = 0 @@ -160,6 +192,7 @@ class Entry(object): is_cpp_class = 0 is_const = 0 is_property = 0 + is_cproperty = 0 doc_cname = None getter_cname = None setter_cname = None @@ -203,6 +236,10 @@ class Entry(object): error_on_uninitialized = False cf_used = True outer_entry = None + is_cgetter = False + is_cpp_optional = False + known_standard_library_import = None + pytyping_modifiers = None def __init__(self, name, cname, type, pos = None, init = None): self.name = name @@ -238,6 +275,19 @@ class Entry(object): else: return NotImplemented + @property + def cf_is_reassigned(self): + return len(self.cf_assignments) > 1 + + def make_cpp_optional(self): + assert self.type.is_cpp_class + self.is_cpp_optional = True + assert not self.utility_code # we're not overwriting anything? + self.utility_code_definition = Code.UtilityCode.load_cached("OptionalLocals", "CppSupport.cpp") + + def declared_with_pytyping_modifier(self, modifier_name): + return modifier_name in self.pytyping_modifiers if self.pytyping_modifiers else False + class InnerEntry(Entry): """ @@ -262,6 +312,7 @@ class InnerEntry(Entry): self.cf_assignments = outermost_entry.cf_assignments self.cf_references = outermost_entry.cf_references self.overloaded_alternatives = outermost_entry.overloaded_alternatives + self.is_cpp_optional = outermost_entry.is_cpp_optional self.inner_entries.append(self) def __getattr__(self, name): @@ -291,10 +342,13 @@ class Scope(object): # is_builtin_scope boolean Is the builtin scope of Python/Cython # is_py_class_scope boolean Is a Python class scope # is_c_class_scope boolean Is an extension type scope + # is_local_scope boolean Is a local (i.e. function/method/generator) scope # is_closure_scope boolean Is a closure scope + # is_generator_expression_scope boolean A subset of closure scope used for generator expressions # is_passthrough boolean Outer scope is passed directly # is_cpp_class_scope boolean Is a C++ class scope # is_property_scope boolean Is a extension type property scope + # is_c_dataclass_scope boolean or "frozen" is a cython.dataclasses.dataclass # scope_prefix string Disambiguator for C names # in_cinclude boolean Suppress C declaration code # qualified_name string "modname" or "modname.classname" @@ -308,17 +362,22 @@ class Scope(object): is_py_class_scope = 0 is_c_class_scope = 0 is_closure_scope = 0 - is_genexpr_scope = 0 + is_local_scope = False + is_generator_expression_scope = 0 + is_comprehension_scope = 0 is_passthrough = 0 is_cpp_class_scope = 0 is_property_scope = 0 is_module_scope = 0 + is_c_dataclass_scope = False is_internal = 0 scope_prefix = "" in_cinclude = 0 nogil = 0 fused_to_specific = None return_type = None + # Do ambiguous type names like 'int' and 'float' refer to the C types? (Otherwise, Python types.) + in_c_type_context = True def __init__(self, name, outer_scope, parent_scope): # The outer_scope is the next scope in the lookup chain. @@ -347,7 +406,6 @@ class Scope(object): self.defined_c_classes = [] self.imported_c_classes = {} self.cname_to_entry = {} - self.string_to_entry = {} self.identifier_to_entry = {} self.num_to_entry = {} self.obj_to_entry = {} @@ -358,11 +416,11 @@ class Scope(object): def __deepcopy__(self, memo): return self - def merge_in(self, other, merge_unused=True, whitelist=None): + def merge_in(self, other, merge_unused=True, allowlist=None): # Use with care... entries = [] for name, entry in other.entries.items(): - if not whitelist or name in whitelist: + if not allowlist or name in allowlist: if entry.used or merge_unused: entries.append((name, entry)) @@ -390,7 +448,7 @@ class Scope(object): def mangle(self, prefix, name = None): if name: - return "%s%s%s" % (prefix, self.scope_prefix, name) + return punycodify_name("%s%s%s" % (prefix, self.scope_prefix, name)) else: return self.parent_scope.mangle(prefix, self.name) @@ -436,17 +494,26 @@ class Scope(object): for scope in sorted(self.subscopes, key=operator.attrgetter('scope_prefix')): yield scope + @try_finally_contextmanager + def new_c_type_context(self, in_c_type_context=None): + old_c_type_context = self.in_c_type_context + if in_c_type_context is not None: + self.in_c_type_context = in_c_type_context + yield + self.in_c_type_context = old_c_type_context + def declare(self, name, cname, type, pos, visibility, shadow = 0, is_type = 0, create_wrapper = 0): # Create new entry, and add to dictionary if # name is not None. Reports a warning if already # declared. - if type.is_buffer and not isinstance(self, LocalScope): # and not is_type: + if type.is_buffer and not isinstance(self, LocalScope): # and not is_type: error(pos, 'Buffer types only allowed as function local variables') if not self.in_cinclude and cname and re.match("^_[_A-Z]+$", cname): - # See http://www.gnu.org/software/libc/manual/html_node/Reserved-Names.html#Reserved-Names + # See https://www.gnu.org/software/libc/manual/html_node/Reserved-Names.html#Reserved-Names warning(pos, "'%s' is a reserved name in C." % cname, -1) + entries = self.entries - if name and name in entries and not shadow: + if name and name in entries and not shadow and not self.is_builtin_scope: old_entry = entries[name] # Reject redeclared C++ functions only if they have the same type signature. @@ -487,8 +554,7 @@ class Scope(object): entries[name] = entry if type.is_memoryviewslice: - from . import MemoryView - entry.init = MemoryView.memslice_entry_init + entry.init = type.default_value entry.scope = self entry.visibility = visibility @@ -522,7 +588,8 @@ class Scope(object): if defining: self.type_entries.append(entry) - if not template: + # don't replace an entry that's already set + if not template and getattr(type, "entry", None) is None: type.entry = entry # here we would set as_variable to an object representing this type @@ -563,8 +630,10 @@ class Scope(object): cname = self.mangle(Naming.type_prefix, name) entry = self.lookup_here(name) if not entry: + in_cpp = self.is_cpp() type = PyrexTypes.CStructOrUnionType( - name, kind, scope, typedef_flag, cname, packed) + name, kind, scope, typedef_flag, cname, packed, + in_cpp = in_cpp) entry = self.declare_type(name, type, pos, cname, visibility = visibility, api = api, defining = scope is not None) @@ -650,12 +719,12 @@ class Scope(object): error(pos, "'%s' previously declared as '%s'" % ( entry.name, entry.visibility)) - def declare_enum(self, name, pos, cname, typedef_flag, - visibility = 'private', api = 0, create_wrapper = 0): + def declare_enum(self, name, pos, cname, scoped, typedef_flag, + visibility='private', api=0, create_wrapper=0, doc=None): if name: if not cname: if (self.in_cinclude or visibility == 'public' - or visibility == 'extern' or api): + or visibility == 'extern' or api): cname = name else: cname = self.mangle(Naming.type_prefix, name) @@ -663,13 +732,21 @@ class Scope(object): namespace = self.outer_scope.lookup(self.name).type else: namespace = None - type = PyrexTypes.CEnumType(name, cname, typedef_flag, namespace) + + if scoped: + type = PyrexTypes.CppScopedEnumType(name, cname, namespace, doc=doc) + else: + type = PyrexTypes.CEnumType(name, cname, typedef_flag, namespace, doc=doc) else: type = PyrexTypes.c_anon_enum_type entry = self.declare_type(name, type, pos, cname = cname, visibility = visibility, api = api) + if scoped: + entry.utility_code = Code.UtilityCode.load_cached("EnumClassDecl", "CppSupport.cpp") + self.use_entry_utility_code(entry) entry.create_wrapper = create_wrapper entry.enum_values = [] + self.sue_entries.append(entry) return entry @@ -677,27 +754,45 @@ class Scope(object): return self.outer_scope.declare_tuple_type(pos, components) def declare_var(self, name, type, pos, - cname = None, visibility = 'private', - api = 0, in_pxd = 0, is_cdef = 0): + cname=None, visibility='private', + api=False, in_pxd=False, is_cdef=False, pytyping_modifiers=None): # Add an entry for a variable. if not cname: if visibility != 'private' or api: cname = name else: cname = self.mangle(Naming.var_prefix, name) - if type.is_cpp_class and visibility != 'extern': - type.check_nullary_constructor(pos) entry = self.declare(name, cname, type, pos, visibility) entry.is_variable = 1 + if type.is_cpp_class and visibility != 'extern': + if self.directives['cpp_locals']: + entry.make_cpp_optional() + else: + type.check_nullary_constructor(pos) if in_pxd and visibility != 'extern': entry.defined_in_pxd = 1 entry.used = 1 if api: entry.api = 1 entry.used = 1 + if pytyping_modifiers: + entry.pytyping_modifiers = pytyping_modifiers return entry + def _reject_pytyping_modifiers(self, pos, modifiers, allowed=()): + if not modifiers: + return + for modifier in modifiers: + if modifier not in allowed: + error(pos, "Modifier '%s' is not allowed here." % modifier) + + def declare_assignment_expression_target(self, name, type, pos): + # In most cases declares the variable as normal. + # For generator expressions and comprehensions the variable is declared in their parent + return self.declare_var(name, type, pos) + def declare_builtin(self, name, pos): + name = self.mangle_class_private_name(name) return self.outer_scope.declare_builtin(name, pos) def _declare_pyfunction(self, name, pos, visibility='extern', entry=None): @@ -719,7 +814,7 @@ class Scope(object): entry.type = py_object_type elif entry.type is not py_object_type: return self._declare_pyfunction(name, pos, visibility=visibility, entry=entry) - else: # declare entry stub + else: # declare entry stub self.declare_var(name, py_object_type, pos, visibility=visibility) entry = self.declare_var(None, py_object_type, pos, cname=name, visibility='private') @@ -736,7 +831,7 @@ class Scope(object): qualified_name = self.qualify_name(lambda_name) entry = self.declare(None, func_cname, py_object_type, pos, 'private') - entry.name = lambda_name + entry.name = EncodedString(lambda_name) entry.qualified_name = qualified_name entry.pymethdef_cname = pymethdef_cname entry.func_cname = func_cname @@ -769,7 +864,8 @@ class Scope(object): entry.cname = cname entry.func_cname = cname if visibility != 'private' and visibility != entry.visibility: - warning(pos, "Function '%s' previously declared as '%s', now as '%s'" % (name, entry.visibility, visibility), 1) + warning(pos, "Function '%s' previously declared as '%s', now as '%s'" % ( + name, entry.visibility, visibility), 1) if overridable != entry.is_overridable: warning(pos, "Function '%s' previously declared as '%s'" % ( name, 'cpdef' if overridable else 'cdef'), 1) @@ -778,15 +874,15 @@ class Scope(object): entry.type = entry.type.with_with_gil(type.with_gil) else: if visibility == 'extern' and entry.visibility == 'extern': - can_override = False + can_override = self.is_builtin_scope if self.is_cpp(): can_override = True - elif cname: + elif cname and not can_override: # if all alternatives have different cnames, # it's safe to allow signature overrides for alt_entry in entry.all_alternatives(): if not alt_entry.cname or cname == alt_entry.cname: - break # cname not unique! + break # cname not unique! else: can_override = True if can_override: @@ -830,6 +926,23 @@ class Scope(object): type.entry = entry return entry + def declare_cgetter(self, name, return_type, pos=None, cname=None, + visibility="private", modifiers=(), defining=False, **cfunc_type_config): + assert all( + k in ('exception_value', 'exception_check', 'nogil', 'with_gil', 'is_const_method', 'is_static_method') + for k in cfunc_type_config + ) + cfunc_type = PyrexTypes.CFuncType( + return_type, + [PyrexTypes.CFuncTypeArg("self", self.parent_type, None)], + **cfunc_type_config) + entry = self.declare_cfunction( + name, cfunc_type, pos, cname=None, visibility=visibility, modifiers=modifiers, defining=defining) + entry.is_cgetter = True + if cname is not None: + entry.func_cname = cname + return entry + def add_cfunction(self, name, type, pos, cname, visibility, modifiers, inherited=False): # Add a C function entry without giving it a func_cname. entry = self.declare(name, cname, type, pos, visibility) @@ -875,37 +988,81 @@ class Scope(object): def lookup(self, name): # Look up name in this scope or an enclosing one. # Return None if not found. - return (self.lookup_here(name) - or (self.outer_scope and self.outer_scope.lookup(name)) - or None) + + mangled_name = self.mangle_class_private_name(name) + entry = (self.lookup_here(name) # lookup here also does mangling + or (self.outer_scope and self.outer_scope.lookup(mangled_name)) + or None) + if entry: + return entry + + # look up the original name in the outer scope + # Not strictly Python behaviour but see https://github.com/cython/cython/issues/3544 + entry = (self.outer_scope and self.outer_scope.lookup(name)) or None + if entry and entry.is_pyglobal: + self._emit_class_private_warning(entry.pos, name) + return entry def lookup_here(self, name): # Look up in this scope only, return None if not found. + + entry = self.entries.get(self.mangle_class_private_name(name), None) + if entry: + return entry + # Also check the unmangled name in the current scope + # (even if mangling should give us something else). + # This is to support things like global __foo which makes a declaration for __foo + return self.entries.get(name, None) + + def lookup_here_unmangled(self, name): return self.entries.get(name, None) + def lookup_assignment_expression_target(self, name): + # For most cases behaves like "lookup_here". + # However, it does look outwards for comprehension and generator expression scopes + return self.lookup_here(name) + def lookup_target(self, name): # Look up name in this scope only. Declare as Python # variable if not found. entry = self.lookup_here(name) if not entry: + entry = self.lookup_here_unmangled(name) + if entry and entry.is_pyglobal: + self._emit_class_private_warning(entry.pos, name) + if not entry: entry = self.declare_var(name, py_object_type, None) return entry - def lookup_type(self, name): - entry = self.lookup(name) + def _type_or_specialized_type_from_entry(self, entry): if entry and entry.is_type: if entry.type.is_fused and self.fused_to_specific: return entry.type.specialize(self.fused_to_specific) return entry.type + def lookup_type(self, name): + entry = self.lookup(name) + # The logic here is: + # 1. if entry is a type then return it (and maybe specialize it) + # 2. if the entry comes from a known standard library import then follow that + # 3. repeat step 1 with the (possibly) updated entry + + tp = self._type_or_specialized_type_from_entry(entry) + if tp: + return tp + # allow us to find types from the "typing" module and similar + if entry and entry.known_standard_library_import: + from .Builtin import get_known_standard_library_entry + entry = get_known_standard_library_entry(entry.known_standard_library_import) + return self._type_or_specialized_type_from_entry(entry) + def lookup_operator(self, operator, operands): if operands[0].type.is_cpp_class: obj_type = operands[0].type method = obj_type.scope.lookup("operator%s" % operator) if method is not None: arg_types = [arg.type for arg in operands[1:]] - res = PyrexTypes.best_match([arg.type for arg in operands[1:]], - method.all_alternatives()) + res = PyrexTypes.best_match(arg_types, method.all_alternatives()) if res is not None: return res function = self.lookup("operator%s" % operator) @@ -915,7 +1072,7 @@ class Scope(object): # look-up nonmember methods listed within a class method_alternatives = [] - if len(operands)==2: # binary operators only + if len(operands) == 2: # binary operators only for n in range(2): if operands[n].type.is_cpp_class: obj_type = operands[n].type @@ -939,6 +1096,11 @@ class Scope(object): operands = [FakeOperand(pos, type=type) for type in types] return self.lookup_operator(operator, operands) + def _emit_class_private_warning(self, pos, name): + warning(pos, "Global name %s matched from within class scope " + "in contradiction to to Python 'class private name' rules. " + "This may change in a future release." % name, 1) + def use_utility_code(self, new_code): self.global_scope().use_utility_code(new_code) @@ -1000,9 +1162,9 @@ class BuiltinScope(Scope): Scope.__init__(self, "__builtin__", PreImportScope(), None) self.type_names = {} - for name, definition in sorted(self.builtin_entries.items()): - cname, type = definition - self.declare_var(name, type, None, cname) + # Most entries are initialized in init_builtins, except for "bool" + # which is apparently a special case because it conflicts with C++ bool + self.declare_var("bool", py_object_type, None, "((PyObject*)&PyBool_Type)") def lookup(self, name, language_level=None, str_is_str=None): # 'language_level' and 'str_is_str' are passed by ModuleScope @@ -1027,8 +1189,7 @@ class BuiltinScope(Scope): # If python_equiv == "*", the Python equivalent has the same name # as the entry, otherwise it has the name specified by python_equiv. name = EncodedString(name) - entry = self.declare_cfunction(name, type, None, cname, visibility='extern', - utility_code=utility_code) + entry = self.declare_cfunction(name, type, None, cname, visibility='extern', utility_code=utility_code) if python_equiv: if python_equiv == "*": python_equiv = name @@ -1043,9 +1204,10 @@ class BuiltinScope(Scope): entry.as_variable = var_entry return entry - def declare_builtin_type(self, name, cname, utility_code = None, objstruct_cname = None): + def declare_builtin_type(self, name, cname, utility_code=None, + objstruct_cname=None, type_class=PyrexTypes.BuiltinObjectType): name = EncodedString(name) - type = PyrexTypes.BuiltinObjectType(name, cname, objstruct_cname) + type = type_class(name, cname, objstruct_cname) scope = CClassScope(name, outer_scope=None, visibility='extern') scope.directives = {} if name == 'bool': @@ -1055,10 +1217,12 @@ class BuiltinScope(Scope): entry = self.declare_type(name, type, None, visibility='extern') entry.utility_code = utility_code - var_entry = Entry(name = entry.name, - type = self.lookup('type').type, # make sure "type" is the first type declared... - pos = entry.pos, - cname = entry.type.typeptr_cname) + var_entry = Entry( + name=entry.name, + type=self.lookup('type').type, # make sure "type" is the first type declared... + pos=entry.pos, + cname=entry.type.typeptr_cname, + ) var_entry.qualified_name = self.qualify_name(name) var_entry.is_variable = 1 var_entry.is_cglobal = 1 @@ -1075,36 +1239,8 @@ class BuiltinScope(Scope): def builtin_scope(self): return self - builtin_entries = { - - "type": ["((PyObject*)&PyType_Type)", py_object_type], - - "bool": ["((PyObject*)&PyBool_Type)", py_object_type], - "int": ["((PyObject*)&PyInt_Type)", py_object_type], - "long": ["((PyObject*)&PyLong_Type)", py_object_type], - "float": ["((PyObject*)&PyFloat_Type)", py_object_type], - "complex":["((PyObject*)&PyComplex_Type)", py_object_type], - - "bytes": ["((PyObject*)&PyBytes_Type)", py_object_type], - "bytearray": ["((PyObject*)&PyByteArray_Type)", py_object_type], - "str": ["((PyObject*)&PyString_Type)", py_object_type], - "unicode":["((PyObject*)&PyUnicode_Type)", py_object_type], - - "tuple": ["((PyObject*)&PyTuple_Type)", py_object_type], - "list": ["((PyObject*)&PyList_Type)", py_object_type], - "dict": ["((PyObject*)&PyDict_Type)", py_object_type], - "set": ["((PyObject*)&PySet_Type)", py_object_type], - "frozenset": ["((PyObject*)&PyFrozenSet_Type)", py_object_type], - - "slice": ["((PyObject*)&PySlice_Type)", py_object_type], -# "file": ["((PyObject*)&PyFile_Type)", py_object_type], # not in Py3 - - "None": ["Py_None", py_object_type], - "False": ["Py_False", py_object_type], - "True": ["Py_True", py_object_type], - } -const_counter = 1 # As a temporary solution for compiling code in pxds +const_counter = 1 # As a temporary solution for compiling code in pxds class ModuleScope(Scope): # module_name string Python name of the module @@ -1116,7 +1252,6 @@ class ModuleScope(Scope): # utility_code_list [UtilityCode] Queuing utility codes for forwarding to Code.py # c_includes {key: IncludeCode} C headers or verbatim code to be generated # See process_include() for more documentation - # string_to_entry {string : Entry} Map string const to entry # identifier_to_entry {string : Entry} Map identifier string const to entry # context Context # parent_module Scope Parent in the import namespace @@ -1136,19 +1271,13 @@ class ModuleScope(Scope): is_cython_builtin = 0 old_style_globals = 0 - def __init__(self, name, parent_module, context): + def __init__(self, name, parent_module, context, is_package=False): from . import Builtin self.parent_module = parent_module outer_scope = Builtin.builtin_scope Scope.__init__(self, name, outer_scope, parent_module) - if name == "__init__": - # Treat Spam/__init__.pyx specially, so that when Python loads - # Spam/__init__.so, initSpam() is defined. - self.module_name = parent_module.module_name - self.is_package = True - else: - self.module_name = name - self.is_package = False + self.is_package = is_package + self.module_name = name self.module_name = EncodedString(self.module_name) self.context = context self.module_cname = Naming.module_cname @@ -1239,7 +1368,7 @@ class ModuleScope(Scope): entry = self.declare(None, None, py_object_type, pos, 'private') if Options.cache_builtins and name not in Code.uncachable_builtins: entry.is_builtin = 1 - entry.is_const = 1 # cached + entry.is_const = 1 # cached entry.name = name entry.cname = Naming.builtin_prefix + name self.cached_builtins.append(entry) @@ -1261,9 +1390,16 @@ class ModuleScope(Scope): # explicit relative cimport # error of going beyond top-level is handled in cimport node relative_to = self - while relative_level > 0 and relative_to: + + top_level = 1 if self.is_package else 0 + # * top_level == 1 when file is __init__.pyx, current package (relative_to) is the current module + # i.e. dot in `from . import ...` points to the current package + # * top_level == 0 when file is regular module, current package (relative_to) is parent module + # i.e. dot in `from . import ...` points to the package where module is placed + while relative_level > top_level and relative_to: relative_to = relative_to.parent_module relative_level -= 1 + elif relative_level != 0: # -1 or None: try relative cimport first, then absolute relative_to = self.parent_module @@ -1273,7 +1409,7 @@ class ModuleScope(Scope): return module_scope.context.find_module( module_name, relative_to=relative_to, pos=pos, absolute_fallback=absolute_fallback) - def find_submodule(self, name): + def find_submodule(self, name, as_package=False): # Find and return scope for a submodule of this module, # creating a new empty one if necessary. Doesn't parse .pxd. if '.' in name: @@ -1282,10 +1418,10 @@ class ModuleScope(Scope): submodule = None scope = self.lookup_submodule(name) if not scope: - scope = ModuleScope(name, parent_module=self, context=self.context) + scope = ModuleScope(name, parent_module=self, context=self.context, is_package=True if submodule else as_package) self.module_entries[name] = scope if submodule: - scope = scope.find_submodule(submodule) + scope = scope.find_submodule(submodule, as_package=as_package) return scope def lookup_submodule(self, name): @@ -1364,7 +1500,7 @@ class ModuleScope(Scope): entry = self.lookup_here(name) if entry: if entry.is_pyglobal and entry.as_module is scope: - return entry # Already declared as the same module + return entry # Already declared as the same module if not (entry.is_pyglobal and not entry.as_module): # SAGE -- I put this here so Pyrex # cimport's work across directories. @@ -1382,14 +1518,15 @@ class ModuleScope(Scope): return entry def declare_var(self, name, type, pos, - cname = None, visibility = 'private', - api = 0, in_pxd = 0, is_cdef = 0): + cname=None, visibility='private', + api=False, in_pxd=False, is_cdef=False, pytyping_modifiers=None): # Add an entry for a global variable. If it is a Python # object type, and not declared with cdef, it will live # in the module dictionary, otherwise it will be a C # global variable. - if not visibility in ('private', 'public', 'extern'): + if visibility not in ('private', 'public', 'extern'): error(pos, "Module-level variable cannot be declared %s" % visibility) + self._reject_pytyping_modifiers(pos, pytyping_modifiers, ('typing.Optional',)) # let's allow at least this one if not is_cdef: if type is unspecified_type: type = py_object_type @@ -1425,7 +1562,7 @@ class ModuleScope(Scope): entry = Scope.declare_var(self, name, type, pos, cname=cname, visibility=visibility, - api=api, in_pxd=in_pxd, is_cdef=is_cdef) + api=api, in_pxd=in_pxd, is_cdef=is_cdef, pytyping_modifiers=pytyping_modifiers) if is_cdef: entry.is_cglobal = 1 if entry.type.declaration_value: @@ -1454,7 +1591,7 @@ class ModuleScope(Scope): entry = self.lookup_here(name) if entry and entry.defined_in_pxd: if entry.visibility != "private": - mangled_cname = self.mangle(Naming.var_prefix, name) + mangled_cname = self.mangle(Naming.func_prefix, name) if entry.cname == mangled_cname: cname = name entry.cname = cname @@ -1505,7 +1642,7 @@ class ModuleScope(Scope): if entry and not shadow: type = entry.type if not (entry.is_type and type.is_extension_type): - entry = None # Will cause redeclaration and produce an error + entry = None # Will cause redeclaration and produce an error else: scope = type.scope if typedef_flag and (not scope or scope.defined): @@ -1585,6 +1722,15 @@ class ModuleScope(Scope): if self.directives.get('final'): entry.type.is_final_type = True + collection_type = self.directives.get('collection_type') + if collection_type: + from .UtilityCode import NonManglingModuleScope + if not isinstance(self, NonManglingModuleScope): + # TODO - DW would like to make it public, but I'm making it internal-only + # for now to avoid adding new features without consensus + error(pos, "'collection_type' is not a public cython directive") + if collection_type == 'sequence': + entry.type.has_sequence_flag = True # cdef classes are always exported, but we need to set it to # distinguish between unused Cython utility code extension classes @@ -1727,6 +1873,7 @@ class ModuleScope(Scope): class LocalScope(Scope): + is_local_scope = True # Does the function have a 'with gil:' block? has_with_gil_block = False @@ -1740,10 +1887,11 @@ class LocalScope(Scope): Scope.__init__(self, name, outer_scope, parent_scope) def mangle(self, prefix, name): - return prefix + name + return punycodify_name(prefix + name) def declare_arg(self, name, type, pos): # Add an entry for an argument of a function. + name = self.mangle_class_private_name(name) cname = self.mangle(Naming.var_prefix, name) entry = self.declare(name, cname, type, pos, 'private') entry.is_variable = 1 @@ -1755,14 +1903,15 @@ class LocalScope(Scope): return entry def declare_var(self, name, type, pos, - cname = None, visibility = 'private', - api = 0, in_pxd = 0, is_cdef = 0): + cname=None, visibility='private', + api=False, in_pxd=False, is_cdef=False, pytyping_modifiers=None): + name = self.mangle_class_private_name(name) # Add an entry for a local variable. if visibility in ('public', 'readonly'): error(pos, "Local variable cannot be declared %s" % visibility) entry = Scope.declare_var(self, name, type, pos, cname=cname, visibility=visibility, - api=api, in_pxd=in_pxd, is_cdef=is_cdef) + api=api, in_pxd=in_pxd, is_cdef=is_cdef, pytyping_modifiers=pytyping_modifiers) if entry.type.declaration_value: entry.init = entry.type.declaration_value entry.is_local = 1 @@ -1790,24 +1939,28 @@ class LocalScope(Scope): if entry is None or not entry.from_closure: error(pos, "no binding for nonlocal '%s' found" % name) + def _create_inner_entry_for_closure(self, name, entry): + entry.in_closure = True + inner_entry = InnerEntry(entry, self) + inner_entry.is_variable = True + self.entries[name] = inner_entry + return inner_entry + def lookup(self, name): # Look up name in this scope or an enclosing one. # Return None if not found. + entry = Scope.lookup(self, name) if entry is not None: entry_scope = entry.scope - while entry_scope.is_genexpr_scope: + while entry_scope.is_comprehension_scope: entry_scope = entry_scope.outer_scope if entry_scope is not self and entry_scope.is_closure_scope: if hasattr(entry.scope, "scope_class"): raise InternalError("lookup() after scope class created.") # The actual c fragment for the different scopes differs # on the outside and inside, so we make a new entry - entry.in_closure = True - inner_entry = InnerEntry(entry, self) - inner_entry.is_variable = True - self.entries[name] = inner_entry - return inner_entry + return self._create_inner_entry_for_closure(name, entry) return entry def mangle_closure_cnames(self, outer_scope_cname): @@ -1824,19 +1977,21 @@ class LocalScope(Scope): elif entry.in_closure: entry.original_cname = entry.cname entry.cname = "%s->%s" % (Naming.cur_scope_cname, entry.cname) + if entry.type.is_cpp_class and entry.scope.directives['cpp_locals']: + entry.make_cpp_optional() -class GeneratorExpressionScope(Scope): - """Scope for generator expressions and comprehensions. As opposed - to generators, these can be easily inlined in some cases, so all +class ComprehensionScope(Scope): + """Scope for comprehensions (but not generator expressions, which use ClosureScope). + As opposed to generators, these can be easily inlined in some cases, so all we really need is a scope that holds the loop variable(s). """ - is_genexpr_scope = True + is_comprehension_scope = True def __init__(self, outer_scope): parent_scope = outer_scope # TODO: also ignore class scopes? - while parent_scope.is_genexpr_scope: + while parent_scope.is_comprehension_scope: parent_scope = parent_scope.parent_scope name = parent_scope.global_scope().next_id(Naming.genexpr_id_ref) Scope.__init__(self, name, outer_scope, parent_scope) @@ -1845,7 +2000,7 @@ class GeneratorExpressionScope(Scope): # Class/ExtType scopes are filled at class creation time, i.e. from the # module init function or surrounding function. - while outer_scope.is_genexpr_scope or outer_scope.is_c_class_scope or outer_scope.is_py_class_scope: + while outer_scope.is_comprehension_scope or outer_scope.is_c_class_scope or outer_scope.is_py_class_scope: outer_scope = outer_scope.outer_scope self.var_entries = outer_scope.var_entries # keep declarations outside outer_scope.subscopes.add(self) @@ -1854,13 +2009,14 @@ class GeneratorExpressionScope(Scope): return '%s%s' % (self.genexp_prefix, self.parent_scope.mangle(prefix, name)) def declare_var(self, name, type, pos, - cname = None, visibility = 'private', - api = 0, in_pxd = 0, is_cdef = True): + cname=None, visibility='private', + api=False, in_pxd=False, is_cdef=True, pytyping_modifiers=None): if type is unspecified_type: # if the outer scope defines a type for this variable, inherit it outer_entry = self.outer_scope.lookup(name) if outer_entry and outer_entry.is_variable: - type = outer_entry.type # may still be 'unspecified_type' ! + type = outer_entry.type # may still be 'unspecified_type' ! + self._reject_pytyping_modifiers(pos, pytyping_modifiers) # the parent scope needs to generate code for the variable, but # this scope must hold its name exclusively cname = '%s%s' % (self.genexp_prefix, self.parent_scope.mangle(Naming.var_prefix, name or self.next_id())) @@ -1875,6 +2031,10 @@ class GeneratorExpressionScope(Scope): self.entries[name] = entry return entry + def declare_assignment_expression_target(self, name, type, pos): + # should be declared in the parent scope instead + return self.parent_scope.declare_var(name, type, pos) + def declare_pyfunction(self, name, pos, allow_redefine=False): return self.outer_scope.declare_pyfunction( name, pos, allow_redefine) @@ -1885,6 +2045,12 @@ class GeneratorExpressionScope(Scope): def add_lambda_def(self, def_node): return self.outer_scope.add_lambda_def(def_node) + def lookup_assignment_expression_target(self, name): + entry = self.lookup_here(name) + if not entry: + entry = self.parent_scope.lookup_assignment_expression_target(name) + return entry + class ClosureScope(LocalScope): @@ -1906,17 +2072,36 @@ class ClosureScope(LocalScope): def declare_pyfunction(self, name, pos, allow_redefine=False): return LocalScope.declare_pyfunction(self, name, pos, allow_redefine, visibility='private') + def declare_assignment_expression_target(self, name, type, pos): + return self.declare_var(name, type, pos) + + +class GeneratorExpressionScope(ClosureScope): + is_generator_expression_scope = True + + def declare_assignment_expression_target(self, name, type, pos): + entry = self.parent_scope.declare_var(name, type, pos) + return self._create_inner_entry_for_closure(name, entry) + + def lookup_assignment_expression_target(self, name): + entry = self.lookup_here(name) + if not entry: + entry = self.parent_scope.lookup_assignment_expression_target(name) + if entry: + return self._create_inner_entry_for_closure(name, entry) + return entry + class StructOrUnionScope(Scope): # Namespace of a C struct or union. def __init__(self, name="?"): - Scope.__init__(self, name, None, None) + Scope.__init__(self, name, outer_scope=None, parent_scope=None) def declare_var(self, name, type, pos, - cname = None, visibility = 'private', - api = 0, in_pxd = 0, is_cdef = 0, - allow_pyobject=False, allow_memoryview=False): + cname=None, visibility='private', + api=False, in_pxd=False, is_cdef=False, pytyping_modifiers=None, + allow_pyobject=False, allow_memoryview=False, allow_refcounted=False): # Add an entry for an attribute. if not cname: cname = name @@ -1924,16 +2109,20 @@ class StructOrUnionScope(Scope): cname = c_safe_identifier(cname) if type.is_cfunction: type = PyrexTypes.CPtrType(type) + self._reject_pytyping_modifiers(pos, pytyping_modifiers) entry = self.declare(name, cname, type, pos, visibility) entry.is_variable = 1 self.var_entries.append(entry) - if type.is_pyobject and not allow_pyobject: - error(pos, "C struct/union member cannot be a Python object") - elif type.is_memoryviewslice and not allow_memoryview: - # Memory views wrap their buffer owner as a Python object. - error(pos, "C struct/union member cannot be a memory view") - if visibility != 'private': - error(pos, "C struct/union member cannot be declared %s" % visibility) + if type.is_pyobject: + if not allow_pyobject: + error(pos, "C struct/union member cannot be a Python object") + elif type.is_memoryviewslice: + if not allow_memoryview: + # Memory views wrap their buffer owner as a Python object. + error(pos, "C struct/union member cannot be a memory view") + elif type.needs_refcounting: + if not allow_refcounted: + error(pos, "C struct/union member cannot be reference-counted type '%s'" % type) return entry def declare_cfunction(self, name, type, pos, @@ -1954,6 +2143,14 @@ class ClassScope(Scope): # declared in the class # doc string or None Doc string + def mangle_class_private_name(self, name): + # a few utilitycode names need to specifically be ignored + if name and name.lower().startswith("__pyx_"): + return name + if name and name.startswith('__') and not name.endswith('__'): + name = EncodedString('_%s%s' % (self.class_name.lstrip('_'), name)) + return name + def __init__(self, name, outer_scope): Scope.__init__(self, name, outer_scope, outer_scope) self.class_name = name @@ -1987,28 +2184,16 @@ class PyClassScope(ClassScope): is_py_class_scope = 1 - def mangle_class_private_name(self, name): - return self.mangle_special_name(name) - - def mangle_special_name(self, name): - if name and name.startswith('__') and not name.endswith('__'): - name = EncodedString('_%s%s' % (self.class_name.lstrip('_'), name)) - return name - - def lookup_here(self, name): - name = self.mangle_special_name(name) - return ClassScope.lookup_here(self, name) - def declare_var(self, name, type, pos, - cname = None, visibility = 'private', - api = 0, in_pxd = 0, is_cdef = 0): - name = self.mangle_special_name(name) + cname=None, visibility='private', + api=False, in_pxd=False, is_cdef=False, pytyping_modifiers=None): + name = self.mangle_class_private_name(name) if type is unspecified_type: type = py_object_type # Add an entry for a class attribute. entry = Scope.declare_var(self, name, type, pos, cname=cname, visibility=visibility, - api=api, in_pxd=in_pxd, is_cdef=is_cdef) + api=api, in_pxd=in_pxd, is_cdef=is_cdef, pytyping_modifiers=pytyping_modifiers) entry.is_pyglobal = 1 entry.is_pyclass_attr = 1 return entry @@ -2043,7 +2228,7 @@ class PyClassScope(ClassScope): class CClassScope(ClassScope): # Namespace of an extension type. # - # parent_type CClassType + # parent_type PyExtensionType # #typeobj_cname string or None # #objstruct_cname string # method_table_cname string @@ -2062,7 +2247,7 @@ class CClassScope(ClassScope): has_pyobject_attrs = False has_memoryview_attrs = False - has_cpp_class_attrs = False + has_cpp_constructable_attrs = False has_cyclic_pyobject_attrs = False defined = False implemented = False @@ -2087,6 +2272,22 @@ class CClassScope(ClassScope): return not self.parent_type.is_gc_simple return False + def needs_trashcan(self): + # If the trashcan directive is explicitly set to False, + # unconditionally disable the trashcan. + directive = self.directives.get('trashcan') + if directive is False: + return False + # If the directive is set to True and the class has Python-valued + # C attributes, then it should use the trashcan in tp_dealloc. + if directive and self.has_cyclic_pyobject_attrs: + return True + # Use the trashcan if the base class uses it + base_type = self.parent_type.base_type + if base_type and base_type.scope is not None: + return base_type.scope.needs_trashcan() + return self.parent_type.builtin_trashcan + def needs_tp_clear(self): """ Do we need to generate an implementation for the tp_clear slot? Can @@ -2094,6 +2295,25 @@ class CClassScope(ClassScope): """ return self.needs_gc() and not self.directives.get('no_gc_clear', False) + def may_have_finalize(self): + """ + This covers cases where we definitely have a __del__ function + and also cases where one of the base classes could have a __del__ + function but we don't know. + """ + current_type_scope = self + while current_type_scope: + del_entry = current_type_scope.lookup_here("__del__") + if del_entry and del_entry.is_special: + return True + if (current_type_scope.parent_type.is_extern or not current_type_scope.implemented or + current_type_scope.parent_type.multiple_bases): + # we don't know if we have __del__, so assume we do and call it + return True + current_base_type = current_type_scope.parent_type.base_type + current_type_scope = current_base_type.scope if current_base_type else None + return False + def get_refcounted_entries(self, include_weakref=False, include_gc_simple=True): py_attrs = [] @@ -2114,15 +2334,30 @@ class CClassScope(ClassScope): return have_entries, (py_attrs, py_buffers, memoryview_slices) def declare_var(self, name, type, pos, - cname = None, visibility = 'private', - api = 0, in_pxd = 0, is_cdef = 0): + cname=None, visibility='private', + api=False, in_pxd=False, is_cdef=False, pytyping_modifiers=None): + name = self.mangle_class_private_name(name) + + if pytyping_modifiers: + if "typing.ClassVar" in pytyping_modifiers: + is_cdef = 0 + if not type.is_pyobject: + if not type.equivalent_type: + warning(pos, "ClassVar[] requires the type to be a Python object type. Found '%s', using object instead." % type) + type = py_object_type + else: + type = type.equivalent_type + if "dataclasses.InitVar" in pytyping_modifiers and not self.is_c_dataclass_scope: + error(pos, "Use of cython.dataclasses.InitVar does not make sense outside a dataclass") + if is_cdef: # Add an entry for an attribute. if self.defined: error(pos, "C attributes cannot be added in implementation part of" " extension type defined in a pxd") - if not self.is_closure_class_scope and get_special_method_signature(name): + if (not self.is_closure_class_scope and + get_slot_table(self.directives).get_special_method_signature(name)): error(pos, "The name '%s' is reserved for a special method." % name) @@ -2130,16 +2365,21 @@ class CClassScope(ClassScope): cname = name if visibility == 'private': cname = c_safe_identifier(cname) - if type.is_cpp_class and visibility != 'extern': - type.check_nullary_constructor(pos) - self.use_utility_code(Code.UtilityCode("#include <new>")) + cname = punycodify_name(cname, Naming.unicode_structmember_prefix) entry = self.declare(name, cname, type, pos, visibility) entry.is_variable = 1 self.var_entries.append(entry) + entry.pytyping_modifiers = pytyping_modifiers + if type.is_cpp_class and visibility != 'extern': + if self.directives['cpp_locals']: + entry.make_cpp_optional() + else: + type.check_nullary_constructor(pos) if type.is_memoryviewslice: self.has_memoryview_attrs = True - elif type.is_cpp_class: - self.has_cpp_class_attrs = True + elif type.needs_cpp_construction: + self.use_utility_code(Code.UtilityCode("#include <new>")) + self.has_cpp_constructable_attrs = True elif type.is_pyobject and (self.is_closure_class_scope or name != '__weakref__'): self.has_pyobject_attrs = True if (not type.is_builtin_type @@ -2167,12 +2407,13 @@ class CClassScope(ClassScope): # Add an entry for a class attribute. entry = Scope.declare_var(self, name, type, pos, cname=cname, visibility=visibility, - api=api, in_pxd=in_pxd, is_cdef=is_cdef) + api=api, in_pxd=in_pxd, is_cdef=is_cdef, pytyping_modifiers=pytyping_modifiers) entry.is_member = 1 - entry.is_pyglobal = 1 # xxx: is_pyglobal changes behaviour in so many places that - # I keep it in for now. is_member should be enough - # later on + # xxx: is_pyglobal changes behaviour in so many places that I keep it in for now. + # is_member should be enough later on + entry.is_pyglobal = 1 self.namespace_cname = "(PyObject *)%s" % self.parent_type.typeptr_cname + return entry def declare_pyfunction(self, name, pos, allow_redefine=False): @@ -2189,7 +2430,7 @@ class CClassScope(ClassScope): "in a future version of Pyrex and Cython. Use __cinit__ instead.") entry = self.declare_var(name, py_object_type, pos, visibility='extern') - special_sig = get_special_method_signature(name) + special_sig = get_slot_table(self.directives).get_special_method_signature(name) if special_sig: # Special methods get put in the method table with a particular # signature declared in advance. @@ -2220,7 +2461,9 @@ class CClassScope(ClassScope): def declare_cfunction(self, name, type, pos, cname=None, visibility='private', api=0, in_pxd=0, defining=0, modifiers=(), utility_code=None, overridable=False): - if get_special_method_signature(name) and not self.parent_type.is_builtin_type: + name = self.mangle_class_private_name(name) + if (get_slot_table(self.directives).get_special_method_signature(name) + and not self.parent_type.is_builtin_type): error(pos, "Special methods must be declared with 'def', not 'cdef'") args = type.args if not type.is_static_method: @@ -2231,10 +2474,11 @@ class CClassScope(ClassScope): (args[0].type, name, self.parent_type)) entry = self.lookup_here(name) if cname is None: - cname = c_safe_identifier(name) + cname = punycodify_name(c_safe_identifier(name), Naming.unicode_vtabentry_prefix) if entry: if not entry.is_cfunction: - warning(pos, "'%s' redeclared " % name, 0) + error(pos, "'%s' redeclared " % name) + entry.already_declared_here() else: if defining and entry.func_cname: error(pos, "'%s' already defined" % name) @@ -2246,13 +2490,14 @@ class CClassScope(ClassScope): entry.type = entry.type.with_with_gil(type.with_gil) elif type.compatible_signature_with(entry.type, as_cmethod = 1) and type.nogil == entry.type.nogil: if (self.defined and not in_pxd - and not type.same_c_signature_as_resolved_type(entry.type, as_cmethod = 1, as_pxd_definition = 1)): + and not type.same_c_signature_as_resolved_type( + entry.type, as_cmethod=1, as_pxd_definition=1)): # TODO(robertwb): Make this an error. warning(pos, "Compatible but non-identical C method '%s' not redeclared " "in definition part of extension type '%s'. " "This may cause incorrect vtables to be generated." % ( - name, self.class_name), 2) + name, self.class_name), 2) warning(entry.pos, "Previous declaration is here", 2) entry = self.add_cfunction(name, type, pos, cname, visibility='ignore', modifiers=modifiers) else: @@ -2272,8 +2517,7 @@ class CClassScope(ClassScope): if u'inline' in modifiers: entry.is_inline_cmethod = True - if (self.parent_type.is_final_type or entry.is_inline_cmethod or - self.directives.get('final')): + if self.parent_type.is_final_type or entry.is_inline_cmethod or self.directives.get('final'): entry.is_final_cmethod = True entry.final_func_cname = entry.func_cname @@ -2282,8 +2526,8 @@ class CClassScope(ClassScope): def add_cfunction(self, name, type, pos, cname, visibility, modifiers, inherited=False): # Add a cfunction entry without giving it a func_cname. prev_entry = self.lookup_here(name) - entry = ClassScope.add_cfunction(self, name, type, pos, cname, - visibility, modifiers, inherited=inherited) + entry = ClassScope.add_cfunction( + self, name, type, pos, cname, visibility, modifiers, inherited=inherited) entry.is_cmethod = 1 entry.prev_entry = prev_entry return entry @@ -2292,8 +2536,8 @@ class CClassScope(ClassScope): # overridden methods of builtin types still have their Python # equivalent that must be accessible to support bound methods name = EncodedString(name) - entry = self.declare_cfunction(name, type, None, cname, visibility='extern', - utility_code=utility_code) + entry = self.declare_cfunction( + name, type, pos=None, cname=cname, visibility='extern', utility_code=utility_code) var_entry = Entry(name, name, py_object_type) var_entry.qualified_name = name var_entry.is_variable = 1 @@ -2303,18 +2547,44 @@ class CClassScope(ClassScope): entry.as_variable = var_entry return entry - def declare_property(self, name, doc, pos): + def declare_property(self, name, doc, pos, ctype=None, property_scope=None): entry = self.lookup_here(name) if entry is None: - entry = self.declare(name, name, py_object_type, pos, 'private') - entry.is_property = 1 + entry = self.declare(name, name, py_object_type if ctype is None else ctype, pos, 'private') + entry.is_property = True + if ctype is not None: + entry.is_cproperty = True entry.doc = doc - entry.scope = PropertyScope(name, - outer_scope = self.global_scope(), parent_scope = self) - entry.scope.parent_type = self.parent_type + if property_scope is None: + entry.scope = PropertyScope(name, class_scope=self) + else: + entry.scope = property_scope self.property_entries.append(entry) return entry + def declare_cproperty(self, name, type, cfunc_name, doc=None, pos=None, visibility='extern', + nogil=False, with_gil=False, exception_value=None, exception_check=False, + utility_code=None): + """Internal convenience method to declare a C property function in one go. + """ + property_entry = self.declare_property(name, doc=doc, ctype=type, pos=pos) + cfunc_entry = property_entry.scope.declare_cfunction( + name=name, + type=PyrexTypes.CFuncType( + type, + [PyrexTypes.CFuncTypeArg("self", self.parent_type, pos=None)], + nogil=nogil, + with_gil=with_gil, + exception_value=exception_value, + exception_check=exception_check, + ), + cname=cfunc_name, + utility_code=utility_code, + visibility=visibility, + pos=pos, + ) + return property_entry, cfunc_entry + def declare_inherited_c_attributes(self, base_scope): # Declare entries for all the C attributes of an # inherited type, with cnames modified appropriately @@ -2328,6 +2598,8 @@ class CClassScope(ClassScope): base_entry.name, adapt(base_entry.cname), base_entry.type, None, 'private') entry.is_variable = 1 + entry.is_inherited = True + entry.annotation = base_entry.annotation self.inherited_var_entries.append(entry) # If the class defined in a pxd, specific entries have not been added. @@ -2343,9 +2615,9 @@ class CClassScope(ClassScope): is_builtin = var_entry and var_entry.is_builtin if not is_builtin: cname = adapt(cname) - entry = self.add_cfunction(base_entry.name, base_entry.type, - base_entry.pos, cname, - base_entry.visibility, base_entry.func_modifiers, inherited=True) + entry = self.add_cfunction( + base_entry.name, base_entry.type, base_entry.pos, cname, + base_entry.visibility, base_entry.func_modifiers, inherited=True) entry.is_inherited = 1 if base_entry.is_final_cmethod: entry.is_final_cmethod = True @@ -2379,11 +2651,12 @@ class CppClassScope(Scope): template_entry.is_type = 1 def declare_var(self, name, type, pos, - cname = None, visibility = 'extern', - api = 0, in_pxd = 0, is_cdef = 0, defining = 0): + cname=None, visibility='extern', + api=False, in_pxd=False, is_cdef=False, defining=False, pytyping_modifiers=None): # Add an entry for an attribute. if not cname: cname = name + self._reject_pytyping_modifiers(pos, pytyping_modifiers) entry = self.lookup_here(name) if defining and entry is not None: if entry.type.same_as(type): @@ -2409,7 +2682,7 @@ class CppClassScope(Scope): class_name = self.name.split('::')[-1] if name in (class_name, '__init__') and cname is None: cname = "%s__init__%s" % (Naming.func_prefix, class_name) - name = '<init>' + name = EncodedString('<init>') type.return_type = PyrexTypes.CVoidType() # This is called by the actual constructor, but need to support # arguments that cannot by called by value. @@ -2423,7 +2696,7 @@ class CppClassScope(Scope): type.args = [maybe_ref(arg) for arg in type.args] elif name == '__dealloc__' and cname is None: cname = "%s__dealloc__%s" % (Naming.func_prefix, class_name) - name = '<del>' + name = EncodedString('<del>') type.return_type = PyrexTypes.CVoidType() if name in ('<init>', '<del>') and type.nogil: for base in self.type.base_classes: @@ -2453,19 +2726,18 @@ class CppClassScope(Scope): # Declare entries for all the C++ attributes of an # inherited type, with cnames modified appropriately # to work with this type. - for base_entry in \ - base_scope.inherited_var_entries + base_scope.var_entries: - #constructor/destructor is not inherited - if base_entry.name in ("<init>", "<del>"): - continue - #print base_entry.name, self.entries - if base_entry.name in self.entries: - base_entry.name # FIXME: is there anything to do in this case? - entry = self.declare(base_entry.name, base_entry.cname, - base_entry.type, None, 'extern') - entry.is_variable = 1 - entry.is_inherited = 1 - self.inherited_var_entries.append(entry) + for base_entry in base_scope.inherited_var_entries + base_scope.var_entries: + #constructor/destructor is not inherited + if base_entry.name in ("<init>", "<del>"): + continue + #print base_entry.name, self.entries + if base_entry.name in self.entries: + base_entry.name # FIXME: is there anything to do in this case? + entry = self.declare(base_entry.name, base_entry.cname, + base_entry.type, None, 'extern') + entry.is_variable = 1 + entry.is_inherited = 1 + self.inherited_var_entries.append(entry) for base_entry in base_scope.cfunc_entries: entry = self.declare_cfunction(base_entry.name, base_entry.type, base_entry.pos, base_entry.cname, @@ -2477,7 +2749,7 @@ class CppClassScope(Scope): if base_entry.name not in base_templates: entry = self.declare_type(base_entry.name, base_entry.type, base_entry.pos, base_entry.cname, - base_entry.visibility) + base_entry.visibility, defining=False) entry.is_inherited = 1 def specialize(self, values, type_entry): @@ -2507,6 +2779,23 @@ class CppClassScope(Scope): return scope +class CppScopedEnumScope(Scope): + # Namespace of a ScopedEnum + + def __init__(self, name, outer_scope): + Scope.__init__(self, name, outer_scope, None) + + def declare_var(self, name, type, pos, + cname=None, visibility='extern', pytyping_modifiers=None): + # Add an entry for an attribute. + if not cname: + cname = name + self._reject_pytyping_modifiers(pos, pytyping_modifiers) + entry = self.declare(name, cname, type, pos, visibility) + entry.is_variable = True + return entry + + class PropertyScope(Scope): # Scope holding the __get__, __set__ and __del__ methods for # a property of an extension type. @@ -2515,6 +2804,31 @@ class PropertyScope(Scope): is_property_scope = 1 + def __init__(self, name, class_scope): + # outer scope is None for some internal properties + outer_scope = class_scope.global_scope() if class_scope.outer_scope else None + Scope.__init__(self, name, outer_scope, parent_scope=class_scope) + self.parent_type = class_scope.parent_type + self.directives = class_scope.directives + + def declare_cfunction(self, name, type, pos, *args, **kwargs): + """Declare a C property function. + """ + if type.return_type.is_void: + error(pos, "C property method cannot return 'void'") + + if type.args and type.args[0].type is py_object_type: + # Set 'self' argument type to extension type. + type.args[0].type = self.parent_scope.parent_type + elif len(type.args) != 1: + error(pos, "C property method must have a single (self) argument") + elif not (type.args[0].type.is_pyobject or type.args[0].type is self.parent_scope.parent_type): + error(pos, "C property method must have a single (object) argument") + + entry = Scope.declare_cfunction(self, name, type, pos, *args, **kwargs) + entry.is_cproperty = True + return entry + def declare_pyfunction(self, name, pos, allow_redefine=False): # Add an entry for a method. signature = get_property_accessor_signature(name) @@ -2529,23 +2843,27 @@ class PropertyScope(Scope): return None -class CConstScope(Scope): +class CConstOrVolatileScope(Scope): - def __init__(self, const_base_type_scope): + def __init__(self, base_type_scope, is_const=0, is_volatile=0): Scope.__init__( self, - 'const_' + const_base_type_scope.name, - const_base_type_scope.outer_scope, - const_base_type_scope.parent_scope) - self.const_base_type_scope = const_base_type_scope + 'cv_' + base_type_scope.name, + base_type_scope.outer_scope, + base_type_scope.parent_scope) + self.base_type_scope = base_type_scope + self.is_const = is_const + self.is_volatile = is_volatile def lookup_here(self, name): - entry = self.const_base_type_scope.lookup_here(name) + entry = self.base_type_scope.lookup_here(name) if entry is not None: entry = copy.copy(entry) - entry.type = PyrexTypes.c_const_type(entry.type) + entry.type = PyrexTypes.c_const_or_volatile_type( + entry.type, self.is_const, self.is_volatile) return entry + class TemplateScope(Scope): def __init__(self, name, outer_scope): Scope.__init__(self, name, outer_scope, None) diff --git a/Cython/Compiler/Tests/TestBuffer.py b/Cython/Compiler/Tests/TestBuffer.py index 1f69d9652..2f653d0ff 100644 --- a/Cython/Compiler/Tests/TestBuffer.py +++ b/Cython/Compiler/Tests/TestBuffer.py @@ -55,7 +55,7 @@ class TestBufferOptions(CythonTest): root = self.fragment(s, pipeline=[NormalizeTree(self), PostParse(self)]).root if not expect_error: vardef = root.stats[0].body.stats[0] - assert isinstance(vardef, CVarDefNode) # use normal assert as this is to validate the test code + assert isinstance(vardef, CVarDefNode) # use normal assert as this is to validate the test code buftype = vardef.base_type self.assertTrue(isinstance(buftype, TemplatedTypeNode)) self.assertTrue(isinstance(buftype.base_type_node, CSimpleBaseTypeNode)) @@ -99,7 +99,7 @@ class TestBufferOptions(CythonTest): # add exotic and impossible combinations as they come along... + if __name__ == '__main__': import unittest unittest.main() - diff --git a/Cython/Compiler/Tests/TestCmdLine.py b/Cython/Compiler/Tests/TestCmdLine.py index bd31da000..290efd1d7 100644 --- a/Cython/Compiler/Tests/TestCmdLine.py +++ b/Cython/Compiler/Tests/TestCmdLine.py @@ -1,8 +1,12 @@ - +import os import sys import re from unittest import TestCase try: + from unittest.mock import patch, Mock +except ImportError: # Py2 + from mock import patch, Mock +try: from StringIO import StringIO except ImportError: from io import StringIO # doesn't accept 'str' in Py2 @@ -10,38 +14,39 @@ except ImportError: from .. import Options from ..CmdLine import parse_command_line +from .Utils import backup_Options, restore_Options, check_global_options -def check_global_options(expected_options, white_list=[]): - """ - returns error message of "" if check Ok - """ - no_value = object() - for name, orig_value in expected_options.items(): - if name not in white_list: - if getattr(Options, name, no_value) != orig_value: - return "error in option " + name - return "" +unpatched_exists = os.path.exists +def patched_exists(path): + # avoid the Cython command raising a file not found error + if path in ( + 'source.pyx', + os.path.join('/work/dir', 'source.pyx'), + os.path.join('my_working_path', 'source.pyx'), + 'file.pyx', + 'file1.pyx', + 'file2.pyx', + 'file3.pyx', + 'foo.pyx', + 'bar.pyx', + ): + return True + return unpatched_exists(path) +@patch('os.path.exists', new=Mock(side_effect=patched_exists)) class CmdLineParserTest(TestCase): def setUp(self): - backup = {} - for name, value in vars(Options).items(): - backup[name] = value - self._options_backup = backup + self._options_backup = backup_Options() def tearDown(self): - no_value = object() - for name, orig_value in self._options_backup.items(): - if getattr(Options, name, no_value) != orig_value: - setattr(Options, name, orig_value) + restore_Options(self._options_backup) def check_default_global_options(self, white_list=[]): self.assertEqual(check_global_options(self._options_backup, white_list), "") def check_default_options(self, options, white_list=[]): - from ..Main import CompilationOptions, default_options - default_options = CompilationOptions(default_options) + default_options = Options.CompilationOptions(Options.default_options) no_value = object() for name in default_options.__dict__.keys(): if name not in white_list: @@ -121,6 +126,397 @@ class CmdLineParserTest(TestCase): self.assertEqual(Options.annotate_coverage_xml, 'cov.xml') self.assertTrue(options.gdb_debug) self.assertEqual(options.output_dir, '/gdb/outdir') + self.assertEqual(options.compiler_directives['wraparound'], False) + + def test_embed_before_positional(self): + options, sources = parse_command_line([ + '--embed', + 'source.pyx', + ]) + self.assertEqual(sources, ['source.pyx']) + self.assertEqual(Options.embed, 'main') + + def test_two_embeds(self): + options, sources = parse_command_line([ + '--embed', '--embed=huhu', + 'source.pyx', + ]) + self.assertEqual(sources, ['source.pyx']) + self.assertEqual(Options.embed, 'huhu') + + def test_two_embeds2(self): + options, sources = parse_command_line([ + '--embed=huhu', '--embed', + 'source.pyx', + ]) + self.assertEqual(sources, ['source.pyx']) + self.assertEqual(Options.embed, 'main') + + def test_no_annotate(self): + options, sources = parse_command_line([ + '--embed=huhu', 'source.pyx' + ]) + self.assertFalse(Options.annotate) + + def test_annotate_short(self): + options, sources = parse_command_line([ + '-a', + 'source.pyx', + ]) + self.assertEqual(Options.annotate, 'default') + + def test_annotate_long(self): + options, sources = parse_command_line([ + '--annotate', + 'source.pyx', + ]) + self.assertEqual(Options.annotate, 'default') + + def test_annotate_fullc(self): + options, sources = parse_command_line([ + '--annotate-fullc', + 'source.pyx', + ]) + self.assertEqual(Options.annotate, 'fullc') + + def test_short_w(self): + options, sources = parse_command_line([ + '-w', 'my_working_path', + 'source.pyx' + ]) + self.assertEqual(options.working_path, 'my_working_path') + self.check_default_global_options() + self.check_default_options(options, ['working_path']) + + def test_short_o(self): + options, sources = parse_command_line([ + '-o', 'my_output', + 'source.pyx' + ]) + self.assertEqual(options.output_file, 'my_output') + self.check_default_global_options() + self.check_default_options(options, ['output_file']) + + def test_short_z(self): + options, sources = parse_command_line([ + '-z', 'my_preimport', + 'source.pyx' + ]) + self.assertEqual(Options.pre_import, 'my_preimport') + self.check_default_global_options(['pre_import']) + self.check_default_options(options) + + def test_convert_range(self): + options, sources = parse_command_line([ + '--convert-range', + 'source.pyx' + ]) + self.assertEqual(Options.convert_range, True) + self.check_default_global_options(['convert_range']) + self.check_default_options(options) + + def test_line_directives(self): + options, sources = parse_command_line([ + '--line-directives', + 'source.pyx' + ]) + self.assertEqual(options.emit_linenums, True) + self.check_default_global_options() + self.check_default_options(options, ['emit_linenums']) + + def test_no_c_in_traceback(self): + options, sources = parse_command_line([ + '--no-c-in-traceback', + 'source.pyx' + ]) + self.assertEqual(options.c_line_in_traceback, False) + self.check_default_global_options() + self.check_default_options(options, ['c_line_in_traceback']) + + def test_gdb(self): + options, sources = parse_command_line([ + '--gdb', + 'source.pyx' + ]) + self.assertEqual(options.gdb_debug, True) + self.assertEqual(options.output_dir, os.curdir) + self.check_default_global_options() + self.check_default_options(options, ['gdb_debug', 'output_dir']) + + def test_3str(self): + options, sources = parse_command_line([ + '--3str', + 'source.pyx' + ]) + self.assertEqual(options.language_level, '3str') + self.check_default_global_options() + self.check_default_options(options, ['language_level']) + + def test_capi_reexport_cincludes(self): + options, sources = parse_command_line([ + '--capi-reexport-cincludes', + 'source.pyx' + ]) + self.assertEqual(options.capi_reexport_cincludes, True) + self.check_default_global_options() + self.check_default_options(options, ['capi_reexport_cincludes']) + + def test_fast_fail(self): + options, sources = parse_command_line([ + '--fast-fail', + 'source.pyx' + ]) + self.assertEqual(Options.fast_fail, True) + self.check_default_global_options(['fast_fail']) + self.check_default_options(options) + + def test_cimport_from_pyx(self): + options, sources = parse_command_line([ + '--cimport-from-pyx', + 'source.pyx' + ]) + self.assertEqual(Options.cimport_from_pyx, True) + self.check_default_global_options(['cimport_from_pyx']) + self.check_default_options(options) + + def test_Werror(self): + options, sources = parse_command_line([ + '-Werror', + 'source.pyx' + ]) + self.assertEqual(Options.warning_errors, True) + self.check_default_global_options(['warning_errors']) + self.check_default_options(options) + + def test_warning_errors(self): + options, sources = parse_command_line([ + '--warning-errors', + 'source.pyx' + ]) + self.assertEqual(Options.warning_errors, True) + self.check_default_global_options(['warning_errors']) + self.check_default_options(options) + + def test_Wextra(self): + options, sources = parse_command_line([ + '-Wextra', + 'source.pyx' + ]) + self.assertEqual(options.compiler_directives, Options.extra_warnings) + self.check_default_global_options() + self.check_default_options(options, ['compiler_directives']) + + def test_warning_extra(self): + options, sources = parse_command_line([ + '--warning-extra', + 'source.pyx' + ]) + self.assertEqual(options.compiler_directives, Options.extra_warnings) + self.check_default_global_options() + self.check_default_options(options, ['compiler_directives']) + + def test_old_style_globals(self): + options, sources = parse_command_line([ + '--old-style-globals', + 'source.pyx' + ]) + self.assertEqual(Options.old_style_globals, True) + self.check_default_global_options(['old_style_globals']) + self.check_default_options(options) + + def test_directive_multiple(self): + options, source = parse_command_line([ + '-X', 'cdivision=True', + '-X', 'c_string_type=bytes', + 'source.pyx' + ]) + self.assertEqual(options.compiler_directives['cdivision'], True) + self.assertEqual(options.compiler_directives['c_string_type'], 'bytes') + self.check_default_global_options() + self.check_default_options(options, ['compiler_directives']) + + def test_directive_multiple_v2(self): + options, source = parse_command_line([ + '-X', 'cdivision=True,c_string_type=bytes', + 'source.pyx' + ]) + self.assertEqual(options.compiler_directives['cdivision'], True) + self.assertEqual(options.compiler_directives['c_string_type'], 'bytes') + self.check_default_global_options() + self.check_default_options(options, ['compiler_directives']) + + def test_directive_value_yes(self): + options, source = parse_command_line([ + '-X', 'cdivision=YeS', + 'source.pyx' + ]) + self.assertEqual(options.compiler_directives['cdivision'], True) + self.check_default_global_options() + self.check_default_options(options, ['compiler_directives']) + + def test_directive_value_no(self): + options, source = parse_command_line([ + '-X', 'cdivision=no', + 'source.pyx' + ]) + self.assertEqual(options.compiler_directives['cdivision'], False) + self.check_default_global_options() + self.check_default_options(options, ['compiler_directives']) + + def test_directive_value_invalid(self): + self.assertRaises(ValueError, parse_command_line, [ + '-X', 'cdivision=sadfasd', + 'source.pyx' + ]) + + def test_directive_key_invalid(self): + self.assertRaises(ValueError, parse_command_line, [ + '-X', 'abracadabra', + 'source.pyx' + ]) + + def test_directive_no_value(self): + self.assertRaises(ValueError, parse_command_line, [ + '-X', 'cdivision', + 'source.pyx' + ]) + + def test_compile_time_env_short(self): + options, source = parse_command_line([ + '-E', 'MYSIZE=10', + 'source.pyx' + ]) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + self.check_default_global_options() + self.check_default_options(options, ['compile_time_env']) + + def test_compile_time_env_long(self): + options, source = parse_command_line([ + '--compile-time-env', 'MYSIZE=10', + 'source.pyx' + ]) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + self.check_default_global_options() + self.check_default_options(options, ['compile_time_env']) + + def test_compile_time_env_multiple(self): + options, source = parse_command_line([ + '-E', 'MYSIZE=10', '-E', 'ARRSIZE=11', + 'source.pyx' + ]) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + self.assertEqual(options.compile_time_env['ARRSIZE'], 11) + self.check_default_global_options() + self.check_default_options(options, ['compile_time_env']) + + def test_compile_time_env_multiple_v2(self): + options, source = parse_command_line([ + '-E', 'MYSIZE=10,ARRSIZE=11', + 'source.pyx' + ]) + self.assertEqual(options.compile_time_env['MYSIZE'], 10) + self.assertEqual(options.compile_time_env['ARRSIZE'], 11) + self.check_default_global_options() + self.check_default_options(options, ['compile_time_env']) + + def test_option_first(self): + options, sources = parse_command_line(['-V', 'file.pyx']) + self.assertEqual(sources, ['file.pyx']) + + def test_file_inbetween(self): + options, sources = parse_command_line(['-V', 'file.pyx', '-a']) + self.assertEqual(sources, ['file.pyx']) + + def test_option_trailing(self): + options, sources = parse_command_line(['file.pyx', '-V']) + self.assertEqual(sources, ['file.pyx']) + + def test_multiple_files(self): + options, sources = parse_command_line([ + 'file1.pyx', '-V', + 'file2.pyx', '-a', + 'file3.pyx' + ]) + self.assertEqual(sources, ['file1.pyx', 'file2.pyx', 'file3.pyx']) + + def test_debug_flags(self): + options, sources = parse_command_line([ + '--debug-disposal-code', '--debug-coercion', + 'file3.pyx' + ]) + from Cython.Compiler import DebugFlags + for name in ['debug_disposal_code', 'debug_temp_alloc', 'debug_coercion']: + self.assertEqual(getattr(DebugFlags, name), name in ['debug_disposal_code', 'debug_coercion']) + setattr(DebugFlags, name, 0) # restore original value + + def test_gdb_overwrites_gdb_outdir(self): + options, sources = parse_command_line([ + '--gdb-outdir=my_dir', '--gdb', + 'file3.pyx' + ]) + self.assertEqual(options.gdb_debug, True) + self.assertEqual(options.output_dir, os.curdir) + self.check_default_global_options() + self.check_default_options(options, ['gdb_debug', 'output_dir']) + + def test_gdb_first(self): + options, sources = parse_command_line([ + '--gdb', '--gdb-outdir=my_dir', + 'file3.pyx' + ]) + self.assertEqual(options.gdb_debug, True) + self.assertEqual(options.output_dir, 'my_dir') + self.check_default_global_options() + self.check_default_options(options, ['gdb_debug', 'output_dir']) + + def test_coverage_overwrites_annotation(self): + options, sources = parse_command_line([ + '--annotate-fullc', '--annotate-coverage=my.xml', + 'file3.pyx' + ]) + self.assertEqual(Options.annotate, True) + self.assertEqual(Options.annotate_coverage_xml, 'my.xml') + self.check_default_global_options(['annotate', 'annotate_coverage_xml']) + self.check_default_options(options) + + def test_coverage_first(self): + options, sources = parse_command_line([ + '--annotate-coverage=my.xml', '--annotate-fullc', + 'file3.pyx' + ]) + self.assertEqual(Options.annotate, 'fullc') + self.assertEqual(Options.annotate_coverage_xml, 'my.xml') + self.check_default_global_options(['annotate', 'annotate_coverage_xml']) + self.check_default_options(options) + + def test_annotate_first_fullc_second(self): + options, sources = parse_command_line([ + '--annotate', '--annotate-fullc', + 'file3.pyx' + ]) + self.assertEqual(Options.annotate, 'fullc') + self.check_default_global_options(['annotate']) + self.check_default_options(options) + + def test_annotate_fullc_first(self): + options, sources = parse_command_line([ + '--annotate-fullc', '--annotate', + 'file3.pyx' + ]) + self.assertEqual(Options.annotate, 'default') + self.check_default_global_options(['annotate']) + self.check_default_options(options) + + def test_warning_extra_dont_overwrite(self): + options, sources = parse_command_line([ + '-X', 'cdivision=True', + '--warning-extra', + '-X', 'c_string_type=bytes', + 'source.pyx' + ]) + self.assertTrue(len(options.compiler_directives), len(Options.extra_warnings) + 1) + self.check_default_global_options() + self.check_default_options(options, ['compiler_directives']) def test_module_name(self): options, sources = parse_command_line([ @@ -145,25 +541,38 @@ class CmdLineParserTest(TestCase): self.assertRaises(SystemExit, parse_command_line, list(args)) finally: sys.stderr = old_stderr - msg = stderr.getvalue().strip() - self.assertTrue(msg) + msg = stderr.getvalue() + err_msg = 'Message "{}"'.format(msg.strip()) + self.assertTrue(msg.startswith('usage: '), + '%s does not start with "usage :"' % err_msg) + self.assertTrue(': error: ' in msg, + '%s does not contain ": error :"' % err_msg) if regex: self.assertTrue(re.search(regex, msg), - '"%s" does not match search "%s"' % - (msg, regex)) + '%s does not match search "%s"' % + (err_msg, regex)) error(['-1'], - 'Unknown compiler flag: -1') - error(['-I']) - error(['--version=-a']) - error(['--version=--annotate=true']) - error(['--working']) - error(['--verbose=1']) - error(['--cleanup']) + 'unknown option -1') + error(['-I'], + 'argument -I/--include-dir: expected one argument') + error(['--version=-a'], + "argument -V/--version: ignored explicit argument '-a'") + error(['--version=--annotate=true'], + "argument -V/--version: ignored explicit argument " + "'--annotate=true'") + error(['--working'], + "argument -w/--working: expected one argument") + error(['--verbose=1'], + "argument -v/--verbose: ignored explicit argument '1'") + error(['--cleanup'], + "argument --cleanup: expected one argument") error(['--debug-disposal-code-wrong-name', 'file3.pyx'], - "Unknown debug flag: debug_disposal_code_wrong_name") - error(['--module-name', 'foo.pyx']) - error(['--module-name', 'foo.bar']) + "unknown option --debug-disposal-code-wrong-name") + error(['--module-name', 'foo.pyx'], + "Need at least one source file") + error(['--module-name', 'foo.bar'], + "Need at least one source file") error(['--module-name', 'foo.bar', 'foo.pyx', 'bar.pyx'], "Only one source file allowed when using --module-name") error(['--module-name', 'foo.bar', '--timestamps', 'foo.pyx'], diff --git a/Cython/Compiler/Tests/TestGrammar.py b/Cython/Compiler/Tests/TestGrammar.py index 3dddc960b..852b48c33 100644 --- a/Cython/Compiler/Tests/TestGrammar.py +++ b/Cython/Compiler/Tests/TestGrammar.py @@ -7,9 +7,12 @@ Uses TreeFragment to test invalid syntax. from __future__ import absolute_import +import ast +import textwrap + from ...TestUtils import CythonTest -from ..Errors import CompileError from .. import ExprNodes +from ..Errors import CompileError # Copied from CPython's test_grammar.py VALID_UNDERSCORE_LITERALS = [ @@ -27,7 +30,15 @@ VALID_UNDERSCORE_LITERALS = [ '1e1_0', '.1_4', '.1_4e1', + '0b_0', + '0x_f', + '0o_5', + '1_00_00j', + '1_00_00.5j', + '1_00_00e5_1j', '.1_4j', + '(1_2.5+3_3j)', + '(.5_6j)', ] # Copied from CPython's test_grammar.py @@ -36,22 +47,29 @@ INVALID_UNDERSCORE_LITERALS = [ '0_', '42_', '1.4j_', + '0x_', '0b1_', '0xf_', '0o5_', + '0 if 1_Else 1', # Underscores in the base selector: '0_b0', '0_xf', '0_o5', - # Underscore right after the base selector: - '0b_0', - '0x_f', - '0o_5', # Old-style octal, still disallowed: - #'0_7', - #'09_99', - # Special case with exponent: - '0 if 1_Else 1', + # FIXME: still need to support PY_VERSION_HEX < 3 + '0_7', + '09_99', + # Multiple consecutive underscores: + '4_______2', + '0.1__4', + '0.1__4j', + '0b1001__0100', + '0xffff__ffff', + '0x___', + '0o5__77', + '1e1__0', + '1e1__0j', # Underscore right before a dot: '1_.4', '1_.4j', @@ -59,24 +77,24 @@ INVALID_UNDERSCORE_LITERALS = [ '1._4', '1._4j', '._5', + '._5j', # Underscore right after a sign: '1.0e+_1', - # Multiple consecutive underscores: - '4_______2', - '0.1__4', - '0b1001__0100', - '0xffff__ffff', - '0o5__77', - '1e1__0', + '1.0e+_1j', # Underscore right before j: '1.4_j', '1.4e5_j', # Underscore right before e: '1_e1', '1.4_e1', + '1.4_e1j', # Underscore right after e: '1e_1', '1.4e_1', + '1.4e_1j', + # Complex cases with parens: + '(1+1.5_j_)', + '(1+1.5_j)', # Whitespace in literals '1_ 2', '1 _2', @@ -88,6 +106,39 @@ INVALID_UNDERSCORE_LITERALS = [ ] +INVALID_ELLIPSIS = [ + (". . .", 2, 0), + (". ..", 2, 0), + (".. .", 2, 0), + (". ...", 2, 0), + (". ... .", 2, 0), + (".. ... .", 2, 0), + (". ... ..", 2, 0), + (""" + ( + . + .. + ) + """, 3, 4), + (""" + [ + .. + ., + None + ] + """, 3, 4), + (""" + { + None, + . + . + + . + } + """, 4, 4) +] + + class TestGrammar(CythonTest): def test_invalid_number_literals(self): @@ -117,11 +168,34 @@ class TestGrammar(CythonTest): # Add/MulNode() -> literal is first or second operand literal_node = literal_node.operand2 if i % 2 else literal_node.operand1 if 'j' in literal or 'J' in literal: - assert isinstance(literal_node, ExprNodes.ImagNode) + if '+' in literal: + # FIXME: tighten this test + assert isinstance(literal_node, ExprNodes.AddNode), (literal, literal_node) + else: + assert isinstance(literal_node, ExprNodes.ImagNode), (literal, literal_node) elif '.' in literal or 'e' in literal or 'E' in literal and not ('0x' in literal or '0X' in literal): - assert isinstance(literal_node, ExprNodes.FloatNode) + assert isinstance(literal_node, ExprNodes.FloatNode), (literal, literal_node) else: - assert isinstance(literal_node, ExprNodes.IntNode) + assert isinstance(literal_node, ExprNodes.IntNode), (literal, literal_node) + + def test_invalid_ellipsis(self): + ERR = ":{0}:{1}: Expected an identifier or literal" + for code, line, col in INVALID_ELLIPSIS: + try: + ast.parse(textwrap.dedent(code)) + except SyntaxError as exc: + assert True + else: + assert False, "Invalid Python code '%s' failed to raise an exception" % code + + try: + self.fragment(u'''\ + # cython: language_level=3 + ''' + code) + except CompileError as exc: + assert ERR.format(line, col) in str(exc), str(exc) + else: + assert False, "Invalid Cython code '%s' failed to raise an exception" % code if __name__ == "__main__": diff --git a/Cython/Compiler/Tests/TestMemView.py b/Cython/Compiler/Tests/TestMemView.py index 3792f26e9..1d04a17fc 100644 --- a/Cython/Compiler/Tests/TestMemView.py +++ b/Cython/Compiler/Tests/TestMemView.py @@ -53,11 +53,11 @@ class TestMemviewParsing(CythonTest): # we also test other similar declarations (buffers, anonymous C arrays) # since the parsing has to distinguish between them. - def disable_test_no_buf_arg(self): # TODO + def disable_test_no_buf_arg(self): # TODO self.not_parseable(u"Expected ']'", u"cdef extern foo(object[int, ndim=2])") - def disable_test_parse_sizeof(self): # TODO + def disable_test_parse_sizeof(self): # TODO self.parse(u"sizeof(int[NN])") self.parse(u"sizeof(int[])") self.parse(u"sizeof(int[][NN])") diff --git a/Cython/Compiler/Tests/TestParseTreeTransforms.py b/Cython/Compiler/Tests/TestParseTreeTransforms.py index 8a16f98cc..6e29263e5 100644 --- a/Cython/Compiler/Tests/TestParseTreeTransforms.py +++ b/Cython/Compiler/Tests/TestParseTreeTransforms.py @@ -5,7 +5,7 @@ from Cython.TestUtils import TransformTest from Cython.Compiler.ParseTreeTransforms import * from Cython.Compiler.ParseTreeTransforms import _calculate_pickle_checksums from Cython.Compiler.Nodes import * -from Cython.Compiler import Main, Symtab +from Cython.Compiler import Main, Symtab, Options class TestNormalizeTree(TransformTest): @@ -91,7 +91,7 @@ class TestNormalizeTree(TransformTest): t = self.run_pipeline([NormalizeTree(None)], u"pass") self.assertTrue(len(t.stats) == 0) -class TestWithTransform(object): # (TransformTest): # Disabled! +class TestWithTransform(object): # (TransformTest): # Disabled! def test_simplified(self): t = self.run_pipeline([WithTransform(None)], u""" @@ -179,8 +179,8 @@ class TestInterpretCompilerDirectives(TransformTest): def setUp(self): super(TestInterpretCompilerDirectives, self).setUp() - compilation_options = Main.CompilationOptions(Main.default_options) - ctx = compilation_options.create_context() + compilation_options = Options.CompilationOptions(Options.default_options) + ctx = Main.Context.from_options(compilation_options) transform = InterpretCompilerDirectives(ctx, ctx.compiler_directives) transform.module_scope = Symtab.ModuleScope('__main__', None, ctx) diff --git a/Cython/Compiler/Tests/TestScanning.py b/Cython/Compiler/Tests/TestScanning.py new file mode 100644 index 000000000..e9cac1b47 --- /dev/null +++ b/Cython/Compiler/Tests/TestScanning.py @@ -0,0 +1,136 @@ +from __future__ import unicode_literals + +import unittest +from io import StringIO +import string + +from .. import Scanning +from ..Symtab import ModuleScope +from ..TreeFragment import StringParseContext +from ..Errors import init_thread + +# generate some fake code - just a bunch of lines of the form "a0 a1 ..." +code = [] +for ch in string.ascii_lowercase: + line = " ".join(["%s%s" % (ch, n) for n in range(10)]) + code.append(line) +code = "\n".join(code) + +init_thread() + + +class TestScanning(unittest.TestCase): + def make_scanner(self): + source = Scanning.StringSourceDescriptor("fake code", code) + buf = StringIO(code) + context = StringParseContext("fake context") + scope = ModuleScope("fake_module", None, None) + + return Scanning.PyrexScanner(buf, source, scope=scope, context=context) + + def test_put_back_positions(self): + scanner = self.make_scanner() + + self.assertEqual(scanner.sy, "IDENT") + self.assertEqual(scanner.systring, "a0") + scanner.next() + self.assertEqual(scanner.sy, "IDENT") + self.assertEqual(scanner.systring, "a1") + a1pos = scanner.position() + self.assertEqual(a1pos[1:], (1, 3)) + a2peek = scanner.peek() # shouldn't mess up the position + self.assertEqual(a1pos, scanner.position()) + scanner.next() + self.assertEqual(a2peek, (scanner.sy, scanner.systring)) + + # find next line + while scanner.sy != "NEWLINE": + scanner.next() + + line_sy = [] + line_systring = [] + line_pos = [] + + scanner.next() + while scanner.sy != "NEWLINE": + line_sy.append(scanner.sy) + line_systring.append(scanner.systring) + line_pos.append(scanner.position()) + scanner.next() + + for sy, systring, pos in zip( + line_sy[::-1], line_systring[::-1], line_pos[::-1] + ): + scanner.put_back(sy, systring, pos) + + n = 0 + while scanner.sy != "NEWLINE": + self.assertEqual(scanner.sy, line_sy[n]) + self.assertEqual(scanner.systring, line_systring[n]) + self.assertEqual(scanner.position(), line_pos[n]) + scanner.next() + n += 1 + + self.assertEqual(n, len(line_pos)) + + def test_tentatively_scan(self): + scanner = self.make_scanner() + with Scanning.tentatively_scan(scanner) as errors: + while scanner.sy != "NEWLINE": + scanner.next() + self.assertFalse(errors) + + scanner.next() + self.assertEqual(scanner.systring, "b0") + pos = scanner.position() + with Scanning.tentatively_scan(scanner) as errors: + while scanner.sy != "NEWLINE": + scanner.next() + if scanner.systring == "b7": + scanner.error("Oh no not b7!") + break + self.assertTrue(errors) + self.assertEqual(scanner.systring, "b0") # state has been restored + self.assertEqual(scanner.position(), pos) + scanner.next() + self.assertEqual(scanner.systring, "b1") # and we can keep going again + scanner.next() + self.assertEqual(scanner.systring, "b2") # and we can keep going again + + with Scanning.tentatively_scan(scanner) as error: + scanner.error("Something has gone wrong with the current symbol") + self.assertEqual(scanner.systring, "b2") + scanner.next() + self.assertEqual(scanner.systring, "b3") + + # test a few combinations of nested scanning + sy1, systring1 = scanner.sy, scanner.systring + pos1 = scanner.position() + with Scanning.tentatively_scan(scanner): + scanner.next() + sy2, systring2 = scanner.sy, scanner.systring + pos2 = scanner.position() + with Scanning.tentatively_scan(scanner): + with Scanning.tentatively_scan(scanner): + scanner.next() + scanner.next() + scanner.error("Ooops") + self.assertEqual((scanner.sy, scanner.systring), (sy2, systring2)) + self.assertEqual((scanner.sy, scanner.systring), (sy2, systring2)) + scanner.error("eee") + self.assertEqual((scanner.sy, scanner.systring), (sy1, systring1)) + with Scanning.tentatively_scan(scanner): + scanner.next() + scanner.next() + with Scanning.tentatively_scan(scanner): + scanner.next() + # no error - but this block should be unwound by the outer block too + scanner.next() + scanner.error("Oooops") + self.assertEqual((scanner.sy, scanner.systring), (sy1, systring1)) + + + + +if __name__ == "__main__": + unittest.main() diff --git a/Cython/Compiler/Tests/TestSignatureMatching.py b/Cython/Compiler/Tests/TestSignatureMatching.py index 166bb225b..57647c873 100644 --- a/Cython/Compiler/Tests/TestSignatureMatching.py +++ b/Cython/Compiler/Tests/TestSignatureMatching.py @@ -47,7 +47,7 @@ class SignatureMatcherTest(unittest.TestCase): self.assertMatches(function_types[1], [pt.c_long_type, pt.c_int_type], functions) def test_cpp_reference_cpp_class(self): - classes = [ cppclasstype("Test%d"%i, []) for i in range(2) ] + classes = [ cppclasstype("Test%d" % i, []) for i in range(2) ] function_types = [ cfunctype(pt.CReferenceType(classes[0])), cfunctype(pt.CReferenceType(classes[1])), @@ -58,7 +58,7 @@ class SignatureMatcherTest(unittest.TestCase): self.assertMatches(function_types[1], [classes[1]], functions) def test_cpp_reference_cpp_class_and_int(self): - classes = [ cppclasstype("Test%d"%i, []) for i in range(2) ] + classes = [ cppclasstype("Test%d" % i, []) for i in range(2) ] function_types = [ cfunctype(pt.CReferenceType(classes[0]), pt.c_int_type), cfunctype(pt.CReferenceType(classes[0]), pt.c_long_type), diff --git a/Cython/Compiler/Tests/TestTreeFragment.py b/Cython/Compiler/Tests/TestTreeFragment.py index 9ee8da547..d2006b402 100644 --- a/Cython/Compiler/Tests/TestTreeFragment.py +++ b/Cython/Compiler/Tests/TestTreeFragment.py @@ -2,7 +2,6 @@ from Cython.TestUtils import CythonTest from Cython.Compiler.TreeFragment import * from Cython.Compiler.Nodes import * from Cython.Compiler.UtilNodes import * -import Cython.Compiler.Naming as Naming class TestTreeFragments(CythonTest): diff --git a/Cython/Compiler/Tests/TestTreePath.py b/Cython/Compiler/Tests/TestTreePath.py index bee53b3d2..b2013086b 100644 --- a/Cython/Compiler/Tests/TestTreePath.py +++ b/Cython/Compiler/Tests/TestTreePath.py @@ -1,5 +1,4 @@ import unittest -from Cython.Compiler.Visitor import PrintTree from Cython.TestUtils import TransformTest from Cython.Compiler.TreePath import find_first, find_all from Cython.Compiler import Nodes, ExprNodes diff --git a/Cython/Compiler/Tests/TestTypes.py b/Cython/Compiler/Tests/TestTypes.py index f2f6f3773..4693dd806 100644 --- a/Cython/Compiler/Tests/TestTypes.py +++ b/Cython/Compiler/Tests/TestTypes.py @@ -17,3 +17,59 @@ class TestMethodDispatcherTransform(unittest.TestCase): cenum = PT.CEnumType("E", "cenum", typedef_flag=False) assert_widest(PT.c_int_type, cenum, PT.c_int_type) + + +class TestTypeIdentifiers(unittest.TestCase): + + TEST_DATA = [ + ("char*", "char__ptr"), + ("char *", "char__ptr"), + ("char **", "char__ptr__ptr"), + ("_typedef", "_typedef"), + ("__typedef", "__dundertypedef"), + ("___typedef", "__dunder_typedef"), + ("____typedef", "__dunder__dundertypedef"), + ("_____typedef", "__dunder__dunder_typedef"), + ("const __typedef", "__const___dundertypedef"), + ("int[42]", "int__lArr42__rArr"), + ("int[:]", "int__lArr__D__rArr"), + ("int[:,:]", "int__lArr__D__comma___D__rArr"), + ("int[:,:,:]", "int__lArr__D__comma___D__comma___D__rArr"), + ("int[:,:,...]", "int__lArr__D__comma___D__comma___EL__rArr"), + ("std::vector", "std__in_vector"), + ("std::vector&&", "std__in_vector__fwref"), + ("const std::vector", "__const_std__in_vector"), + ("const std::vector&", "__const_std__in_vector__ref"), + ("const_std", "const_std"), + ] + + def test_escape_special_type_characters(self): + test_func = PT._escape_special_type_characters # keep test usage visible for IDEs + function_name = "_escape_special_type_characters" + self._test_escape(function_name) + + def test_type_identifier_for_declaration(self): + test_func = PT.type_identifier_from_declaration # keep test usage visible for IDEs + function_name = test_func.__name__ + self._test_escape(function_name) + + # differences due to whitespace removal + test_data = [ + ("const &std::vector", "const__refstd__in_vector"), + ("const &std::vector<int>", "const__refstd__in_vector__lAngint__rAng"), + ("const &&std::vector", "const__fwrefstd__in_vector"), + ("const &&&std::vector", "const__fwref__refstd__in_vector"), + ("const &&std::vector", "const__fwrefstd__in_vector"), + ("void (*func)(int x, float y)", + "975d51__void__lParen__ptrfunc__rParen__lParenint__space_x__comma_float__space_y__rParen__etc"), + ("float ** (*func)(int x, int[:] y)", + "31883a__float__ptr__ptr__lParen__ptrfunc__rParen__lParenint__space_x__comma_int__lArr__D__rArry__rParen__etc"), + ] + self._test_escape(function_name, test_data) + + def _test_escape(self, func_name, test_data=TEST_DATA): + escape = getattr(PT, func_name) + for declaration, expected in test_data: + escaped_value = escape(declaration) + self.assertEqual(escaped_value, expected, "%s('%s') == '%s' != '%s'" % ( + func_name, declaration, escaped_value, expected)) diff --git a/Cython/Compiler/Tests/TestUtilityLoad.py b/Cython/Compiler/Tests/TestUtilityLoad.py index 3d1906ca0..44ae461ab 100644 --- a/Cython/Compiler/Tests/TestUtilityLoad.py +++ b/Cython/Compiler/Tests/TestUtilityLoad.py @@ -22,14 +22,11 @@ class TestUtilityLoader(unittest.TestCase): cls = Code.UtilityCode def test_load_as_string(self): - got = strip_2tup(self.cls.load_as_string(self.name)) - self.assertEqual(got, self.expected) - got = strip_2tup(self.cls.load_as_string(self.name, self.filename)) self.assertEqual(got, self.expected) def test_load(self): - utility = self.cls.load(self.name) + utility = self.cls.load(self.name, from_file=self.filename) got = strip_2tup((utility.proto, utility.impl)) self.assertEqual(got, self.expected) @@ -37,10 +34,6 @@ class TestUtilityLoader(unittest.TestCase): got = strip_2tup((required.proto, required.impl)) self.assertEqual(got, self.required) - utility = self.cls.load(self.name, from_file=self.filename) - got = strip_2tup((utility.proto, utility.impl)) - self.assertEqual(got, self.expected) - utility = self.cls.load_cached(self.name, from_file=self.filename) got = strip_2tup((utility.proto, utility.impl)) self.assertEqual(got, self.expected) @@ -59,11 +52,11 @@ class TestTempitaUtilityLoader(TestUtilityLoader): cls = Code.TempitaUtilityCode def test_load_as_string(self): - got = strip_2tup(self.cls.load_as_string(self.name, context=self.context)) + got = strip_2tup(self.cls.load_as_string(self.name, self.filename, context=self.context)) self.assertEqual(got, self.expected_tempita) def test_load(self): - utility = self.cls.load(self.name, context=self.context) + utility = self.cls.load(self.name, self.filename, context=self.context) got = strip_2tup((utility.proto, utility.impl)) self.assertEqual(got, self.expected_tempita) diff --git a/Cython/Compiler/Tests/Utils.py b/Cython/Compiler/Tests/Utils.py new file mode 100644 index 000000000..a158ecc50 --- /dev/null +++ b/Cython/Compiler/Tests/Utils.py @@ -0,0 +1,36 @@ +import copy + +from .. import Options + + +def backup_Options(): + backup = {} + for name, value in vars(Options).items(): + # we need a deep copy of _directive_defaults, because they can be changed + if name == '_directive_defaults': + value = copy.deepcopy(value) + backup[name] = value + return backup + + +def restore_Options(backup): + no_value = object() + for name, orig_value in backup.items(): + if getattr(Options, name, no_value) != orig_value: + setattr(Options, name, orig_value) + # strip Options from new keys that might have been added: + for name in vars(Options).keys(): + if name not in backup: + delattr(Options, name) + + +def check_global_options(expected_options, white_list=[]): + """ + returns error message of "" if check Ok + """ + no_value = object() + for name, orig_value in expected_options.items(): + if name not in white_list: + if getattr(Options, name, no_value) != orig_value: + return "error in option " + name + return "" diff --git a/Cython/Compiler/TreeFragment.py b/Cython/Compiler/TreeFragment.py index b85da8191..cef4469b5 100644 --- a/Cython/Compiler/TreeFragment.py +++ b/Cython/Compiler/TreeFragment.py @@ -29,8 +29,7 @@ class StringParseContext(Main.Context): include_directories = [] if compiler_directives is None: compiler_directives = {} - # TODO: see if "language_level=3" also works for our internal code here. - Main.Context.__init__(self, include_directories, compiler_directives, cpp=cpp, language_level=2) + Main.Context.__init__(self, include_directories, compiler_directives, cpp=cpp, language_level='3str') self.module_name = name def find_module(self, module_name, relative_to=None, pos=None, need_pxd=1, absolute_fallback=True): @@ -179,7 +178,7 @@ class TemplateTransform(VisitorTransform): if pos is None: pos = node.pos return ApplyPositionAndCopy(pos)(sub) else: - return self.visit_Node(node) # make copy as usual + return self.visit_Node(node) # make copy as usual def visit_NameNode(self, node): temphandle = self.tempmap.get(node.name) @@ -235,7 +234,7 @@ class TreeFragment(object): fmt_pxds[key] = fmt(value) mod = t = parse_from_strings(name, fmt_code, fmt_pxds, level=level, initial_pos=initial_pos) if level is None: - t = t.body # Make sure a StatListNode is at the top + t = t.body # Make sure a StatListNode is at the top if not isinstance(t, StatListNode): t = StatListNode(pos=mod.pos, stats=[t]) for transform in pipeline: diff --git a/Cython/Compiler/TypeInference.py b/Cython/Compiler/TypeInference.py index c7ffee7d2..447d352be 100644 --- a/Cython/Compiler/TypeInference.py +++ b/Cython/Compiler/TypeInference.py @@ -104,10 +104,11 @@ class MarkParallelAssignments(EnvTransform): is_special = False sequence = node.iterator.sequence target = node.target + iterator_scope = node.iterator.expr_scope or self.current_env() if isinstance(sequence, ExprNodes.SimpleCallNode): function = sequence.function if sequence.self is None and function.is_name: - entry = self.current_env().lookup(function.name) + entry = iterator_scope.lookup(function.name) if not entry or entry.is_builtin: if function.name == 'reversed' and len(sequence.args) == 1: sequence = sequence.args[0] @@ -115,7 +116,7 @@ class MarkParallelAssignments(EnvTransform): if target.is_sequence_constructor and len(target.args) == 2: iterator = sequence.args[0] if iterator.is_name: - iterator_type = iterator.infer_type(self.current_env()) + iterator_type = iterator.infer_type(iterator_scope) if iterator_type.is_builtin_type: # assume that builtin types have a length within Py_ssize_t self.mark_assignment( @@ -127,7 +128,7 @@ class MarkParallelAssignments(EnvTransform): if isinstance(sequence, ExprNodes.SimpleCallNode): function = sequence.function if sequence.self is None and function.is_name: - entry = self.current_env().lookup(function.name) + entry = iterator_scope.lookup(function.name) if not entry or entry.is_builtin: if function.name in ('range', 'xrange'): is_special = True @@ -140,7 +141,6 @@ class MarkParallelAssignments(EnvTransform): '+', sequence.args[0], sequence.args[2])) - if not is_special: # A for-loop basically translates to subsequent calls to # __getitem__(), so using an IndexNode here allows us to @@ -178,7 +178,7 @@ class MarkParallelAssignments(EnvTransform): return node def visit_FromCImportStatNode(self, node): - pass # Can't be assigned to... + return node # Can't be assigned to... def visit_FromImportStatNode(self, node): for name, target in node.items: @@ -308,10 +308,10 @@ class MarkOverflowingArithmetic(CythonTransform): def visit_SimpleCallNode(self, node): if node.function.is_name and node.function.name == 'abs': - # Overflows for minimum value of fixed size ints. - return self.visit_dangerous_node(node) + # Overflows for minimum value of fixed size ints. + return self.visit_dangerous_node(node) else: - return self.visit_neutral_node(node) + return self.visit_neutral_node(node) visit_UnopNode = visit_neutral_node @@ -359,10 +359,17 @@ class SimpleAssignmentTypeInferer(object): Note: in order to support cross-closure type inference, this must be applies to nested scopes in top-down order. """ - def set_entry_type(self, entry, entry_type): - entry.type = entry_type + def set_entry_type(self, entry, entry_type, scope): for e in entry.all_entries(): e.type = entry_type + if e.type.is_memoryviewslice: + # memoryview slices crash if they don't get initialized + e.init = e.type.default_value + if e.type.is_cpp_class: + if scope.directives['cpp_locals']: + e.make_cpp_optional() + else: + e.type.check_nullary_constructor(entry.pos) def infer_types(self, scope): enabled = scope.directives['infer_types'] @@ -370,12 +377,12 @@ class SimpleAssignmentTypeInferer(object): if enabled == True: spanning_type = aggressive_spanning_type - elif enabled is None: # safe mode + elif enabled is None: # safe mode spanning_type = safe_spanning_type else: for entry in scope.entries.values(): if entry.type is unspecified_type: - self.set_entry_type(entry, py_object_type) + self.set_entry_type(entry, py_object_type, scope) return # Set of assignments @@ -404,7 +411,7 @@ class SimpleAssignmentTypeInferer(object): else: entry = node.entry node_type = spanning_type( - types, entry.might_overflow, entry.pos, scope) + types, entry.might_overflow, scope) node.inferred_type = node_type def infer_name_node_type_partial(node): @@ -413,7 +420,7 @@ class SimpleAssignmentTypeInferer(object): if not types: return entry = node.entry - return spanning_type(types, entry.might_overflow, entry.pos, scope) + return spanning_type(types, entry.might_overflow, scope) def inferred_types(entry): has_none = False @@ -488,9 +495,9 @@ class SimpleAssignmentTypeInferer(object): types = inferred_types(entry) if types and all(types): entry_type = spanning_type( - types, entry.might_overflow, entry.pos, scope) + types, entry.might_overflow, scope) inferred.add(entry) - self.set_entry_type(entry, entry_type) + self.set_entry_type(entry, entry_type, scope) def reinfer(): dirty = False @@ -498,9 +505,9 @@ class SimpleAssignmentTypeInferer(object): for assmt in entry.cf_assignments: assmt.infer_type() types = inferred_types(entry) - new_type = spanning_type(types, entry.might_overflow, entry.pos, scope) + new_type = spanning_type(types, entry.might_overflow, scope) if new_type != entry.type: - self.set_entry_type(entry, new_type) + self.set_entry_type(entry, new_type, scope) dirty = True return dirty @@ -530,22 +537,20 @@ def find_spanning_type(type1, type2): return PyrexTypes.c_double_type return result_type -def simply_type(result_type, pos): +def simply_type(result_type): if result_type.is_reference: result_type = result_type.ref_base_type - if result_type.is_const: - result_type = result_type.const_base_type - if result_type.is_cpp_class: - result_type.check_nullary_constructor(pos) + if result_type.is_cv_qualified: + result_type = result_type.cv_base_type if result_type.is_array: result_type = PyrexTypes.c_ptr_type(result_type.base_type) return result_type -def aggressive_spanning_type(types, might_overflow, pos, scope): - return simply_type(reduce(find_spanning_type, types), pos) +def aggressive_spanning_type(types, might_overflow, scope): + return simply_type(reduce(find_spanning_type, types)) -def safe_spanning_type(types, might_overflow, pos, scope): - result_type = simply_type(reduce(find_spanning_type, types), pos) +def safe_spanning_type(types, might_overflow, scope): + result_type = simply_type(reduce(find_spanning_type, types)) if result_type.is_pyobject: # In theory, any specific Python type is always safe to # infer. However, inferring str can cause some existing code @@ -555,9 +560,11 @@ def safe_spanning_type(types, might_overflow, pos, scope): return py_object_type else: return result_type - elif result_type is PyrexTypes.c_double_type: + elif (result_type is PyrexTypes.c_double_type or + result_type is PyrexTypes.c_float_type): # Python's float type is just a C double, so it's safe to use - # the C type instead + # the C type instead. Similarly if given a C float, it leads to + # a small loss of precision vs Python but is otherwise the same return result_type elif result_type is PyrexTypes.c_bint_type: # find_spanning_type() only returns 'bint' for clean boolean @@ -577,8 +584,12 @@ def safe_spanning_type(types, might_overflow, pos, scope): # used, won't arise in pure Python, and there shouldn't be side # effects, so I'm declaring this safe. return result_type - # TODO: double complex should be OK as well, but we need - # to make sure everything is supported. + elif result_type.is_memoryviewslice: + return result_type + elif result_type is PyrexTypes.soft_complex_type: + return result_type + elif result_type == PyrexTypes.c_double_complex_type: + return result_type elif (result_type.is_int or result_type.is_enum) and not might_overflow: return result_type elif (not result_type.can_coerce_to_pyobject(scope) diff --git a/Cython/Compiler/TypeSlots.py b/Cython/Compiler/TypeSlots.py index 0ef17ede6..2c891f648 100644 --- a/Cython/Compiler/TypeSlots.py +++ b/Cython/Compiler/TypeSlots.py @@ -9,6 +9,8 @@ from . import Naming from . import PyrexTypes from .Errors import error +import copy + invisible = ['__cinit__', '__dealloc__', '__richcmp__', '__nonzero__', '__bool__'] @@ -23,6 +25,7 @@ class Signature(object): # fixed_arg_format string # ret_format string # error_value string + # use_fastcall boolean # # The formats are strings made up of the following # characters: @@ -49,6 +52,7 @@ class Signature(object): # '*' rest of args passed as generic Python # arg tuple and kw dict (must be last # char in format string) + # '?' optional object arg (currently for pow only) format_map = { 'O': PyrexTypes.py_object_type, @@ -68,6 +72,7 @@ class Signature(object): 'S': PyrexTypes.c_char_ptr_ptr_type, 'r': PyrexTypes.c_returncode_type, 'B': PyrexTypes.c_py_buffer_ptr_type, + '?': PyrexTypes.py_object_type # 'T', '-' and '*' are handled otherwise # and are not looked up in here } @@ -86,20 +91,27 @@ class Signature(object): 'z': "-1", } - def __init__(self, arg_format, ret_format): - self.has_dummy_arg = 0 - self.has_generic_args = 0 + # Use METH_FASTCALL instead of METH_VARARGS + use_fastcall = False + + def __init__(self, arg_format, ret_format, nogil=False): + self.has_dummy_arg = False + self.has_generic_args = False + self.optional_object_arg_count = 0 if arg_format[:1] == '-': - self.has_dummy_arg = 1 + self.has_dummy_arg = True arg_format = arg_format[1:] if arg_format[-1:] == '*': - self.has_generic_args = 1 + self.has_generic_args = True arg_format = arg_format[:-1] + if arg_format[-1:] == '?': + self.optional_object_arg_count += 1 self.fixed_arg_format = arg_format self.ret_format = ret_format self.error_value = self.error_value_map.get(ret_format, None) self.exception_check = ret_format != 'r' and self.error_value is not None self.is_staticmethod = False + self.nogil = nogil def __repr__(self): return '<Signature[%s(%s%s)]>' % ( @@ -107,7 +119,10 @@ class Signature(object): ', '.join(self.fixed_arg_format), '*' if self.has_generic_args else '') - def num_fixed_args(self): + def min_num_fixed_args(self): + return self.max_num_fixed_args() - self.optional_object_arg_count + + def max_num_fixed_args(self): return len(self.fixed_arg_format) def is_self_arg(self, i): @@ -135,7 +150,7 @@ class Signature(object): def function_type(self, self_arg_override=None): # Construct a C function type descriptor for this signature args = [] - for i in range(self.num_fixed_args()): + for i in range(self.max_num_fixed_args()): if self_arg_override is not None and self.is_self_arg(i): assert isinstance(self_arg_override, PyrexTypes.CFuncTypeArg) args.append(self_arg_override) @@ -149,7 +164,8 @@ class Signature(object): exc_value = self.exception_value() return PyrexTypes.CFuncType( ret_type, args, exception_value=exc_value, - exception_check=self.exception_check) + exception_check=self.exception_check, + nogil=self.nogil) def method_flags(self): if self.ret_format == "O": @@ -157,17 +173,50 @@ class Signature(object): if self.has_dummy_arg: full_args = "O" + full_args if full_args in ["O", "T"]: - if self.has_generic_args: - return [method_varargs, method_keywords] - else: + if not self.has_generic_args: return [method_noargs] + elif self.use_fastcall: + return [method_fastcall, method_keywords] + else: + return [method_varargs, method_keywords] elif full_args in ["OO", "TO"] and not self.has_generic_args: return [method_onearg] if self.is_staticmethod: - return [method_varargs, method_keywords] + if self.use_fastcall: + return [method_fastcall, method_keywords] + else: + return [method_varargs, method_keywords] return None + def method_function_type(self): + # Return the C function type + mflags = self.method_flags() + kw = "WithKeywords" if (method_keywords in mflags) else "" + for m in mflags: + if m == method_noargs or m == method_onearg: + return "PyCFunction" + if m == method_varargs: + return "PyCFunction" + kw + if m == method_fastcall: + return "__Pyx_PyCFunction_FastCall" + kw + return None + + def with_fastcall(self): + # Return a copy of this Signature with use_fastcall=True + sig = copy.copy(self) + sig.use_fastcall = True + return sig + + @property + def fastvar(self): + # Used to select variants of functions, one dealing with METH_VARARGS + # and one dealing with __Pyx_METH_FASTCALL + if self.use_fastcall: + return "FASTCALL" + else: + return "VARARGS" + class SlotDescriptor(object): # Abstract base class for type slot descriptors. @@ -178,15 +227,26 @@ class SlotDescriptor(object): # py3 Indicates presence of slot in Python 3 # py2 Indicates presence of slot in Python 2 # ifdef Full #ifdef string that slot is wrapped in. Using this causes py3, py2 and flags to be ignored.) + # used_ifdef Full #ifdef string that the slot value is wrapped in (otherwise it is assigned NULL) + # Unlike "ifdef" the slot is defined and this just controls if it receives a value def __init__(self, slot_name, dynamic=False, inherited=False, - py3=True, py2=True, ifdef=None): + py3=True, py2=True, ifdef=None, is_binop=False, + used_ifdef=None): self.slot_name = slot_name self.is_initialised_dynamically = dynamic self.is_inherited = inherited self.ifdef = ifdef + self.used_ifdef = used_ifdef self.py3 = py3 self.py2 = py2 + self.is_binop = is_binop + + def slot_code(self, scope): + raise NotImplementedError() + + def spec_value(self, scope): + return self.slot_code(scope) def preprocessor_guard_code(self): ifdef = self.ifdef @@ -194,13 +254,30 @@ class SlotDescriptor(object): py3 = self.py3 guard = None if ifdef: - guard = ("#if %s" % ifdef) + guard = "#if %s" % ifdef elif not py3 or py3 == '<RESERVED>': - guard = ("#if PY_MAJOR_VERSION < 3") + guard = "#if PY_MAJOR_VERSION < 3" elif not py2: - guard = ("#if PY_MAJOR_VERSION >= 3") + guard = "#if PY_MAJOR_VERSION >= 3" return guard + def generate_spec(self, scope, code): + if self.is_initialised_dynamically: + return + value = self.spec_value(scope) + if value == "0": + return + preprocessor_guard = self.preprocessor_guard_code() + if not preprocessor_guard: + if self.py3 and self.slot_name.startswith('bf_'): + # The buffer protocol requires Limited API 3.11, so check if the spec slots are available. + preprocessor_guard = "#if defined(Py_%s)" % self.slot_name + if preprocessor_guard: + code.putln(preprocessor_guard) + code.putln("{Py_%s, (void *)%s}," % (self.slot_name, value)) + if preprocessor_guard: + code.putln("#endif") + def generate(self, scope, code): preprocessor_guard = self.preprocessor_guard_code() if preprocessor_guard: @@ -215,7 +292,7 @@ class SlotDescriptor(object): # PyPy currently has a broken PyType_Ready() that fails to # inherit some slots. To work around this, we explicitly # set inherited slots here, but only in PyPy since CPython - # handles this better than we do. + # handles this better than we do (except for buffer slots in type specs). inherited_value = value current_scope = scope while (inherited_value == "0" @@ -225,12 +302,20 @@ class SlotDescriptor(object): current_scope = current_scope.parent_type.base_type.scope inherited_value = self.slot_code(current_scope) if inherited_value != "0": - code.putln("#if CYTHON_COMPILING_IN_PYPY") + # we always need inherited buffer slots for the type spec + is_buffer_slot = int(self.slot_name in ("bf_getbuffer", "bf_releasebuffer")) + code.putln("#if CYTHON_COMPILING_IN_PYPY || %d" % is_buffer_slot) code.putln("%s, /*%s*/" % (inherited_value, self.slot_name)) code.putln("#else") end_pypy_guard = True + if self.used_ifdef: + code.putln("#if %s" % self.used_ifdef) code.putln("%s, /*%s*/" % (value, self.slot_name)) + if self.used_ifdef: + code.putln("#else") + code.putln("NULL, /*%s*/" % self.slot_name) + code.putln("#endif") if end_pypy_guard: code.putln("#endif") @@ -248,14 +333,20 @@ class SlotDescriptor(object): def generate_dynamic_init_code(self, scope, code): if self.is_initialised_dynamically: - value = self.slot_code(scope) - if value != "0": - code.putln("%s.%s = %s;" % ( - scope.parent_type.typeobj_cname, - self.slot_name, - value - ) - ) + self.generate_set_slot_code( + self.slot_code(scope), scope, code) + + def generate_set_slot_code(self, value, scope, code): + if value == "0": + return + + if scope.parent_type.typeptr_cname: + target = "%s->%s" % (scope.parent_type.typeptr_cname, self.slot_name) + else: + assert scope.parent_type.typeobj_cname + target = "%s.%s" % (scope.parent_type.typeobj_cname, self.slot_name) + + code.putln("%s = %s;" % (target, value)) class FixedSlot(SlotDescriptor): @@ -285,8 +376,8 @@ class MethodSlot(SlotDescriptor): # method_name string The __xxx__ name of the method # alternatives [string] Alternative list of __xxx__ names for the method - def __init__(self, signature, slot_name, method_name, fallback=None, - py3=True, py2=True, ifdef=None, inherited=True): + def __init__(self, signature, slot_name, method_name, method_name_to_slot, + fallback=None, py3=True, py2=True, ifdef=None, inherited=True): SlotDescriptor.__init__(self, slot_name, py3=py3, py2=py2, ifdef=ifdef, inherited=inherited) self.signature = signature @@ -360,30 +451,61 @@ class GCClearReferencesSlot(GCDependentSlot): class ConstructorSlot(InternalMethodSlot): # Descriptor for tp_new and tp_dealloc. - def __init__(self, slot_name, method, **kargs): + def __init__(self, slot_name, method=None, **kargs): InternalMethodSlot.__init__(self, slot_name, **kargs) self.method = method - def slot_code(self, scope): - entry = scope.lookup_here(self.method) - if (self.slot_name != 'tp_new' - and scope.parent_type.base_type + def _needs_own(self, scope): + if (scope.parent_type.base_type and not scope.has_pyobject_attrs and not scope.has_memoryview_attrs - and not scope.has_cpp_class_attrs - and not (entry and entry.is_special)): + and not scope.has_cpp_constructable_attrs + and not (self.slot_name == 'tp_new' and scope.parent_type.vtabslot_cname)): + entry = scope.lookup_here(self.method) if self.method else None + if not (entry and entry.is_special): + return False + # Unless we can safely delegate to the parent, all types need a tp_new(). + return True + + def _parent_slot_function(self, scope): + parent_type_scope = scope.parent_type.base_type.scope + if scope.parent_scope is parent_type_scope.parent_scope: + entry = scope.parent_scope.lookup_here(scope.parent_type.base_type.name) + if entry.visibility != 'extern': + return self.slot_code(parent_type_scope) + return None + + def slot_code(self, scope): + if not self._needs_own(scope): # if the type does not have object attributes, it can # delegate GC methods to its parent - iff the parent # functions are defined in the same module - parent_type_scope = scope.parent_type.base_type.scope - if scope.parent_scope is parent_type_scope.parent_scope: - entry = scope.parent_scope.lookup_here(scope.parent_type.base_type.name) - if entry.visibility != 'extern': - return self.slot_code(parent_type_scope) - if entry and not entry.is_special: - return "0" + slot_code = self._parent_slot_function(scope) + return slot_code or '0' return InternalMethodSlot.slot_code(self, scope) + def spec_value(self, scope): + slot_function = self.slot_code(scope) + if self.slot_name == "tp_dealloc" and slot_function != scope.mangle_internal("tp_dealloc"): + # Not used => inherit from base type. + return "0" + return slot_function + + def generate_dynamic_init_code(self, scope, code): + if self.slot_code(scope) != '0': + return + # If we don't have our own slot function and don't know the + # parent function statically, copy it dynamically. + base_type = scope.parent_type.base_type + if base_type.typeptr_cname: + src = '%s->%s' % (base_type.typeptr_cname, self.slot_name) + elif base_type.is_extension_type and base_type.typeobj_cname: + src = '%s.%s' % (base_type.typeobj_cname, self.slot_name) + else: + return + + self.generate_set_slot_code(src, scope, code) + class SyntheticSlot(InternalMethodSlot): # Type slot descriptor for a synthesized method which @@ -404,6 +526,20 @@ class SyntheticSlot(InternalMethodSlot): else: return self.default_value + def spec_value(self, scope): + return self.slot_code(scope) + + +class BinopSlot(SyntheticSlot): + def __init__(self, signature, slot_name, left_method, method_name_to_slot, **kargs): + assert left_method.startswith('__') + right_method = '__r' + left_method[2:] + SyntheticSlot.__init__( + self, slot_name, [left_method, right_method], "0", is_binop=True, **kargs) + # MethodSlot causes special method registration. + self.left_slot = MethodSlot(signature, "", left_method, method_name_to_slot, **kargs) + self.right_slot = MethodSlot(signature, "", right_method, method_name_to_slot, **kargs) + class RichcmpSlot(MethodSlot): def slot_code(self, scope): @@ -432,8 +568,16 @@ class TypeFlagsSlot(SlotDescriptor): value += "|Py_TPFLAGS_BASETYPE" if scope.needs_gc(): value += "|Py_TPFLAGS_HAVE_GC" + if scope.may_have_finalize(): + value += "|Py_TPFLAGS_HAVE_FINALIZE" + if scope.parent_type.has_sequence_flag: + value += "|Py_TPFLAGS_SEQUENCE" return value + def generate_spec(self, scope, code): + # Flags are stored in the PyType_Spec, not in a PyType_Slot. + return + class DocStringSlot(SlotDescriptor): # Descriptor for the docstring slot. @@ -444,7 +588,7 @@ class DocStringSlot(SlotDescriptor): return "0" if doc.is_unicode: doc = doc.as_utf8_string() - return doc.as_c_string_literal() + return "PyDoc_STR(%s)" % doc.as_c_string_literal() class SuiteSlot(SlotDescriptor): @@ -452,7 +596,7 @@ class SuiteSlot(SlotDescriptor): # # sub_slots [SlotDescriptor] - def __init__(self, sub_slots, slot_type, slot_name, ifdef=None): + def __init__(self, sub_slots, slot_type, slot_name, substructures, ifdef=None): SlotDescriptor.__init__(self, slot_name, ifdef=ifdef) self.sub_slots = sub_slots self.slot_type = slot_type @@ -487,7 +631,9 @@ class SuiteSlot(SlotDescriptor): if self.ifdef: code.putln("#endif") -substructures = [] # List of all SuiteSlot instances + def generate_spec(self, scope, code): + for slot in self.sub_slots: + slot.generate_spec(scope, code) class MethodTableSlot(SlotDescriptor): # Slot descriptor for the method table. @@ -503,8 +649,42 @@ class MemberTableSlot(SlotDescriptor): # Slot descriptor for the table of Python-accessible attributes. def slot_code(self, scope): + # Only used in specs. return "0" + def get_member_specs(self, scope): + return [ + get_slot_by_name("tp_dictoffset", scope.directives).members_slot_value(scope), + #get_slot_by_name("tp_weaklistoffset").spec_value(scope), + ] + + def is_empty(self, scope): + for member_entry in self.get_member_specs(scope): + if member_entry: + return False + return True + + def substructure_cname(self, scope): + return "%s%s_%s" % (Naming.pyrex_prefix, self.slot_name, scope.class_name) + + def generate_substructure_spec(self, scope, code): + if self.is_empty(scope): + return + from .Code import UtilityCode + code.globalstate.use_utility_code(UtilityCode.load_cached("IncludeStructmemberH", "ModuleSetupCode.c")) + + code.putln("static struct PyMemberDef %s[] = {" % self.substructure_cname(scope)) + for member_entry in self.get_member_specs(scope): + if member_entry: + code.putln(member_entry) + code.putln("{NULL, 0, 0, 0, NULL}") + code.putln("};") + + def spec_value(self, scope): + if self.is_empty(scope): + return "0" + return self.substructure_cname(scope) + class GetSetSlot(SlotDescriptor): # Slot descriptor for the table of attribute get & set methods. @@ -520,13 +700,13 @@ class BaseClassSlot(SlotDescriptor): # Slot descriptor for the base class slot. def __init__(self, name): - SlotDescriptor.__init__(self, name, dynamic = 1) + SlotDescriptor.__init__(self, name, dynamic=True) def generate_dynamic_init_code(self, scope, code): base_type = scope.parent_type.base_type if base_type: - code.putln("%s.%s = %s;" % ( - scope.parent_type.typeobj_cname, + code.putln("%s->%s = %s;" % ( + scope.parent_type.typeptr_cname, self.slot_name, base_type.typeptr_cname)) @@ -551,10 +731,11 @@ class DictOffsetSlot(SlotDescriptor): else: return "0" - -# The following dictionary maps __xxx__ method names to slot descriptors. - -method_name_to_slot = {} + def members_slot_value(self, scope): + dict_offset = self.slot_code(scope) + if dict_offset == "0": + return None + return '{"__dictoffset__", T_PYSSIZET, %s, READONLY, NULL},' % dict_offset ## The following slots are (or could be) initialised with an ## extern function pointer. @@ -569,17 +750,6 @@ method_name_to_slot = {} # #------------------------------------------------------------------------------------------ -def get_special_method_signature(name): - # Given a method name, if it is a special method, - # return its signature, else return None. - slot = method_name_to_slot.get(name) - if slot: - return slot.signature - elif name in richcmp_special_methods: - return ibinaryfunc - else: - return None - def get_property_accessor_signature(name): # Return signature of accessor for an extension type @@ -592,7 +762,7 @@ def get_base_slot_function(scope, slot): # This is useful for enabling the compiler to optimize calls # that recursively climb the class hierarchy. base_type = scope.parent_type.base_type - if scope.parent_scope is base_type.scope.parent_scope: + if base_type and scope.parent_scope is base_type.scope.parent_scope: parent_slot = slot.slot_code(base_type.scope) if parent_slot != '0': entry = scope.parent_scope.lookup_here(scope.parent_type.base_type.name) @@ -613,16 +783,16 @@ def get_slot_function(scope, slot): return None -def get_slot_by_name(slot_name): +def get_slot_by_name(slot_name, compiler_directives): # For now, only search the type struct, no referenced sub-structs. - for slot in slot_table: + for slot in get_slot_table(compiler_directives).slot_table: if slot.slot_name == slot_name: return slot assert False, "Slot not found: %s" % slot_name def get_slot_code_by_name(scope, slot_name): - slot = get_slot_by_name(slot_name) + slot = get_slot_by_name(slot_name, scope.directives) return slot.slot_code(scope) def is_reverse_number_slot(name): @@ -634,8 +804,8 @@ def is_reverse_number_slot(name): """ if name.startswith("__r") and name.endswith("__"): forward_name = name.replace("r", "", 1) - for meth in PyNumberMethods: - if getattr(meth, "method_name", None) == forward_name: + for meth in get_slot_table(None).PyNumberMethods: + if hasattr(meth, "right_slot"): return True return False @@ -668,8 +838,8 @@ pyfunction_onearg = Signature("-O", "O") unaryfunc = Signature("T", "O") # typedef PyObject * (*unaryfunc)(PyObject *); binaryfunc = Signature("OO", "O") # typedef PyObject * (*binaryfunc)(PyObject *, PyObject *); ibinaryfunc = Signature("TO", "O") # typedef PyObject * (*binaryfunc)(PyObject *, PyObject *); -ternaryfunc = Signature("OOO", "O") # typedef PyObject * (*ternaryfunc)(PyObject *, PyObject *, PyObject *); -iternaryfunc = Signature("TOO", "O") # typedef PyObject * (*ternaryfunc)(PyObject *, PyObject *, PyObject *); +powternaryfunc = Signature("OO?", "O") # typedef PyObject * (*ternaryfunc)(PyObject *, PyObject *, PyObject *); +ipowternaryfunc = Signature("TO?", "O") # typedef PyObject * (*ternaryfunc)(PyObject *, PyObject *, PyObject *); callfunc = Signature("T*", "O") # typedef PyObject * (*ternaryfunc)(PyObject *, PyObject *, PyObject *); inquiry = Signature("T", "i") # typedef int (*inquiry)(PyObject *); lenfunc = Signature("T", "z") # typedef Py_ssize_t (*lenfunc)(PyObject *); @@ -682,7 +852,7 @@ ssizessizeargfunc = Signature("Tzz", "O") # typedef PyObject *(*ssizessizeargfu intobjargproc = Signature("TiO", 'r') # typedef int(*intobjargproc)(PyObject *, int, PyObject *); ssizeobjargproc = Signature("TzO", 'r') # typedef int(*ssizeobjargproc)(PyObject *, Py_ssize_t, PyObject *); intintobjargproc = Signature("TiiO", 'r') # typedef int(*intintobjargproc)(PyObject *, int, int, PyObject *); -ssizessizeobjargproc = Signature("TzzO", 'r') # typedef int(*ssizessizeobjargproc)(PyObject *, Py_ssize_t, Py_ssize_t, PyObject *); +ssizessizeobjargproc = Signature("TzzO", 'r') # typedef int(*ssizessizeobjargproc)(PyObject *, Py_ssize_t, Py_ssize_t, PyObject *); intintargproc = Signature("Tii", 'r') ssizessizeargproc = Signature("Tzz", 'r') @@ -732,103 +902,8 @@ property_accessor_signatures = { '__del__': Signature("T", 'r') } -#------------------------------------------------------------------------------------------ -# -# Descriptor tables for the slots of the various type object -# substructures, in the order they appear in the structure. -# -#------------------------------------------------------------------------------------------ -PyNumberMethods_Py3_GUARD = "PY_MAJOR_VERSION < 3 || (CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x03050000)" - -PyNumberMethods = ( - MethodSlot(binaryfunc, "nb_add", "__add__"), - MethodSlot(binaryfunc, "nb_subtract", "__sub__"), - MethodSlot(binaryfunc, "nb_multiply", "__mul__"), - MethodSlot(binaryfunc, "nb_divide", "__div__", ifdef = PyNumberMethods_Py3_GUARD), - MethodSlot(binaryfunc, "nb_remainder", "__mod__"), - MethodSlot(binaryfunc, "nb_divmod", "__divmod__"), - MethodSlot(ternaryfunc, "nb_power", "__pow__"), - MethodSlot(unaryfunc, "nb_negative", "__neg__"), - MethodSlot(unaryfunc, "nb_positive", "__pos__"), - MethodSlot(unaryfunc, "nb_absolute", "__abs__"), - MethodSlot(inquiry, "nb_nonzero", "__nonzero__", py3 = ("nb_bool", "__bool__")), - MethodSlot(unaryfunc, "nb_invert", "__invert__"), - MethodSlot(binaryfunc, "nb_lshift", "__lshift__"), - MethodSlot(binaryfunc, "nb_rshift", "__rshift__"), - MethodSlot(binaryfunc, "nb_and", "__and__"), - MethodSlot(binaryfunc, "nb_xor", "__xor__"), - MethodSlot(binaryfunc, "nb_or", "__or__"), - EmptySlot("nb_coerce", ifdef = PyNumberMethods_Py3_GUARD), - MethodSlot(unaryfunc, "nb_int", "__int__", fallback="__long__"), - MethodSlot(unaryfunc, "nb_long", "__long__", fallback="__int__", py3 = "<RESERVED>"), - MethodSlot(unaryfunc, "nb_float", "__float__"), - MethodSlot(unaryfunc, "nb_oct", "__oct__", ifdef = PyNumberMethods_Py3_GUARD), - MethodSlot(unaryfunc, "nb_hex", "__hex__", ifdef = PyNumberMethods_Py3_GUARD), - - # Added in release 2.0 - MethodSlot(ibinaryfunc, "nb_inplace_add", "__iadd__"), - MethodSlot(ibinaryfunc, "nb_inplace_subtract", "__isub__"), - MethodSlot(ibinaryfunc, "nb_inplace_multiply", "__imul__"), - MethodSlot(ibinaryfunc, "nb_inplace_divide", "__idiv__", ifdef = PyNumberMethods_Py3_GUARD), - MethodSlot(ibinaryfunc, "nb_inplace_remainder", "__imod__"), - MethodSlot(ibinaryfunc, "nb_inplace_power", "__ipow__"), # actually ternaryfunc!!! - MethodSlot(ibinaryfunc, "nb_inplace_lshift", "__ilshift__"), - MethodSlot(ibinaryfunc, "nb_inplace_rshift", "__irshift__"), - MethodSlot(ibinaryfunc, "nb_inplace_and", "__iand__"), - MethodSlot(ibinaryfunc, "nb_inplace_xor", "__ixor__"), - MethodSlot(ibinaryfunc, "nb_inplace_or", "__ior__"), - - # Added in release 2.2 - # The following require the Py_TPFLAGS_HAVE_CLASS flag - MethodSlot(binaryfunc, "nb_floor_divide", "__floordiv__"), - MethodSlot(binaryfunc, "nb_true_divide", "__truediv__"), - MethodSlot(ibinaryfunc, "nb_inplace_floor_divide", "__ifloordiv__"), - MethodSlot(ibinaryfunc, "nb_inplace_true_divide", "__itruediv__"), - - # Added in release 2.5 - MethodSlot(unaryfunc, "nb_index", "__index__"), - - # Added in release 3.5 - MethodSlot(binaryfunc, "nb_matrix_multiply", "__matmul__", ifdef="PY_VERSION_HEX >= 0x03050000"), - MethodSlot(ibinaryfunc, "nb_inplace_matrix_multiply", "__imatmul__", ifdef="PY_VERSION_HEX >= 0x03050000"), -) - -PySequenceMethods = ( - MethodSlot(lenfunc, "sq_length", "__len__"), - EmptySlot("sq_concat"), # nb_add used instead - EmptySlot("sq_repeat"), # nb_multiply used instead - SyntheticSlot("sq_item", ["__getitem__"], "0"), #EmptySlot("sq_item"), # mp_subscript used instead - MethodSlot(ssizessizeargfunc, "sq_slice", "__getslice__"), - EmptySlot("sq_ass_item"), # mp_ass_subscript used instead - SyntheticSlot("sq_ass_slice", ["__setslice__", "__delslice__"], "0"), - MethodSlot(cmpfunc, "sq_contains", "__contains__"), - EmptySlot("sq_inplace_concat"), # nb_inplace_add used instead - EmptySlot("sq_inplace_repeat"), # nb_inplace_multiply used instead -) - -PyMappingMethods = ( - MethodSlot(lenfunc, "mp_length", "__len__"), - MethodSlot(objargfunc, "mp_subscript", "__getitem__"), - SyntheticSlot("mp_ass_subscript", ["__setitem__", "__delitem__"], "0"), -) - -PyBufferProcs = ( - MethodSlot(readbufferproc, "bf_getreadbuffer", "__getreadbuffer__", py3 = False), - MethodSlot(writebufferproc, "bf_getwritebuffer", "__getwritebuffer__", py3 = False), - MethodSlot(segcountproc, "bf_getsegcount", "__getsegcount__", py3 = False), - MethodSlot(charbufferproc, "bf_getcharbuffer", "__getcharbuffer__", py3 = False), - - MethodSlot(getbufferproc, "bf_getbuffer", "__getbuffer__"), - MethodSlot(releasebufferproc, "bf_releasebuffer", "__releasebuffer__") -) - -PyAsyncMethods = ( - MethodSlot(unaryfunc, "am_await", "__await__"), - MethodSlot(unaryfunc, "am_aiter", "__aiter__"), - MethodSlot(unaryfunc, "am_anext", "__anext__"), - EmptySlot("am_send", ifdef="PY_VERSION_HEX >= 0x030A00A3"), -) +PyNumberMethods_Py2only_GUARD = "PY_MAJOR_VERSION < 3 || (CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x03050000)" #------------------------------------------------------------------------------------------ # @@ -836,100 +911,262 @@ PyAsyncMethods = ( # top-level type slots, beginning with tp_dealloc, in the order they # appear in the type object. # +# It depends on some compiler directives (currently c_api_binop_methods), so the +# slot tables for each set of compiler directives are generated lazily and put in +# the _slot_table_dict +# #------------------------------------------------------------------------------------------ -slot_table = ( - ConstructorSlot("tp_dealloc", '__dealloc__'), - EmptySlot("tp_print", ifdef="PY_VERSION_HEX < 0x030800b4"), - EmptySlot("tp_vectorcall_offset", ifdef="PY_VERSION_HEX >= 0x030800b4"), - EmptySlot("tp_getattr"), - EmptySlot("tp_setattr"), - - # tp_compare (Py2) / tp_reserved (Py3<3.5) / tp_as_async (Py3.5+) is always used as tp_as_async in Py3 - MethodSlot(cmpfunc, "tp_compare", "__cmp__", ifdef="PY_MAJOR_VERSION < 3"), - SuiteSlot(PyAsyncMethods, "__Pyx_PyAsyncMethodsStruct", "tp_as_async", ifdef="PY_MAJOR_VERSION >= 3"), - - MethodSlot(reprfunc, "tp_repr", "__repr__"), - - SuiteSlot(PyNumberMethods, "PyNumberMethods", "tp_as_number"), - SuiteSlot(PySequenceMethods, "PySequenceMethods", "tp_as_sequence"), - SuiteSlot(PyMappingMethods, "PyMappingMethods", "tp_as_mapping"), - - MethodSlot(hashfunc, "tp_hash", "__hash__", inherited=False), # Py3 checks for __richcmp__ - MethodSlot(callfunc, "tp_call", "__call__"), - MethodSlot(reprfunc, "tp_str", "__str__"), - - SyntheticSlot("tp_getattro", ["__getattr__","__getattribute__"], "0"), #"PyObject_GenericGetAttr"), - SyntheticSlot("tp_setattro", ["__setattr__", "__delattr__"], "0"), #"PyObject_GenericSetAttr"), - - SuiteSlot(PyBufferProcs, "PyBufferProcs", "tp_as_buffer"), - - TypeFlagsSlot("tp_flags"), - DocStringSlot("tp_doc"), - - GCDependentSlot("tp_traverse"), - GCClearReferencesSlot("tp_clear"), - - RichcmpSlot(richcmpfunc, "tp_richcompare", "__richcmp__", inherited=False), # Py3 checks for __hash__ - - EmptySlot("tp_weaklistoffset"), - - MethodSlot(getiterfunc, "tp_iter", "__iter__"), - MethodSlot(iternextfunc, "tp_iternext", "__next__"), +class SlotTable(object): + def __init__(self, old_binops): + # The following dictionary maps __xxx__ method names to slot descriptors. + method_name_to_slot = {} + self._get_slot_by_method_name = method_name_to_slot.get + self.substructures = [] # List of all SuiteSlot instances + + bf = binaryfunc if old_binops else ibinaryfunc + ptf = powternaryfunc if old_binops else ipowternaryfunc + + # Descriptor tables for the slots of the various type object + # substructures, in the order they appear in the structure. + self.PyNumberMethods = ( + BinopSlot(bf, "nb_add", "__add__", method_name_to_slot), + BinopSlot(bf, "nb_subtract", "__sub__", method_name_to_slot), + BinopSlot(bf, "nb_multiply", "__mul__", method_name_to_slot), + BinopSlot(bf, "nb_divide", "__div__", method_name_to_slot, + ifdef = PyNumberMethods_Py2only_GUARD), + BinopSlot(bf, "nb_remainder", "__mod__", method_name_to_slot), + BinopSlot(bf, "nb_divmod", "__divmod__", method_name_to_slot), + BinopSlot(ptf, "nb_power", "__pow__", method_name_to_slot), + MethodSlot(unaryfunc, "nb_negative", "__neg__", method_name_to_slot), + MethodSlot(unaryfunc, "nb_positive", "__pos__", method_name_to_slot), + MethodSlot(unaryfunc, "nb_absolute", "__abs__", method_name_to_slot), + MethodSlot(inquiry, "nb_bool", "__bool__", method_name_to_slot, + py2 = ("nb_nonzero", "__nonzero__")), + MethodSlot(unaryfunc, "nb_invert", "__invert__", method_name_to_slot), + BinopSlot(bf, "nb_lshift", "__lshift__", method_name_to_slot), + BinopSlot(bf, "nb_rshift", "__rshift__", method_name_to_slot), + BinopSlot(bf, "nb_and", "__and__", method_name_to_slot), + BinopSlot(bf, "nb_xor", "__xor__", method_name_to_slot), + BinopSlot(bf, "nb_or", "__or__", method_name_to_slot), + EmptySlot("nb_coerce", ifdef = PyNumberMethods_Py2only_GUARD), + MethodSlot(unaryfunc, "nb_int", "__int__", method_name_to_slot, fallback="__long__"), + MethodSlot(unaryfunc, "nb_long", "__long__", method_name_to_slot, + fallback="__int__", py3 = "<RESERVED>"), + MethodSlot(unaryfunc, "nb_float", "__float__", method_name_to_slot), + MethodSlot(unaryfunc, "nb_oct", "__oct__", method_name_to_slot, + ifdef = PyNumberMethods_Py2only_GUARD), + MethodSlot(unaryfunc, "nb_hex", "__hex__", method_name_to_slot, + ifdef = PyNumberMethods_Py2only_GUARD), + + # Added in release 2.0 + MethodSlot(ibinaryfunc, "nb_inplace_add", "__iadd__", method_name_to_slot), + MethodSlot(ibinaryfunc, "nb_inplace_subtract", "__isub__", method_name_to_slot), + MethodSlot(ibinaryfunc, "nb_inplace_multiply", "__imul__", method_name_to_slot), + MethodSlot(ibinaryfunc, "nb_inplace_divide", "__idiv__", method_name_to_slot, + ifdef = PyNumberMethods_Py2only_GUARD), + MethodSlot(ibinaryfunc, "nb_inplace_remainder", "__imod__", method_name_to_slot), + MethodSlot(ptf, "nb_inplace_power", "__ipow__", method_name_to_slot), + MethodSlot(ibinaryfunc, "nb_inplace_lshift", "__ilshift__", method_name_to_slot), + MethodSlot(ibinaryfunc, "nb_inplace_rshift", "__irshift__", method_name_to_slot), + MethodSlot(ibinaryfunc, "nb_inplace_and", "__iand__", method_name_to_slot), + MethodSlot(ibinaryfunc, "nb_inplace_xor", "__ixor__", method_name_to_slot), + MethodSlot(ibinaryfunc, "nb_inplace_or", "__ior__", method_name_to_slot), + + # Added in release 2.2 + # The following require the Py_TPFLAGS_HAVE_CLASS flag + BinopSlot(bf, "nb_floor_divide", "__floordiv__", method_name_to_slot), + BinopSlot(bf, "nb_true_divide", "__truediv__", method_name_to_slot), + MethodSlot(ibinaryfunc, "nb_inplace_floor_divide", "__ifloordiv__", method_name_to_slot), + MethodSlot(ibinaryfunc, "nb_inplace_true_divide", "__itruediv__", method_name_to_slot), + + # Added in release 2.5 + MethodSlot(unaryfunc, "nb_index", "__index__", method_name_to_slot), + + # Added in release 3.5 + BinopSlot(bf, "nb_matrix_multiply", "__matmul__", method_name_to_slot, + ifdef="PY_VERSION_HEX >= 0x03050000"), + MethodSlot(ibinaryfunc, "nb_inplace_matrix_multiply", "__imatmul__", method_name_to_slot, + ifdef="PY_VERSION_HEX >= 0x03050000"), + ) + + self.PySequenceMethods = ( + MethodSlot(lenfunc, "sq_length", "__len__", method_name_to_slot), + EmptySlot("sq_concat"), # nb_add used instead + EmptySlot("sq_repeat"), # nb_multiply used instead + SyntheticSlot("sq_item", ["__getitem__"], "0"), #EmptySlot("sq_item"), # mp_subscript used instead + MethodSlot(ssizessizeargfunc, "sq_slice", "__getslice__", method_name_to_slot), + EmptySlot("sq_ass_item"), # mp_ass_subscript used instead + SyntheticSlot("sq_ass_slice", ["__setslice__", "__delslice__"], "0"), + MethodSlot(cmpfunc, "sq_contains", "__contains__", method_name_to_slot), + EmptySlot("sq_inplace_concat"), # nb_inplace_add used instead + EmptySlot("sq_inplace_repeat"), # nb_inplace_multiply used instead + ) + + self.PyMappingMethods = ( + MethodSlot(lenfunc, "mp_length", "__len__", method_name_to_slot), + MethodSlot(objargfunc, "mp_subscript", "__getitem__", method_name_to_slot), + SyntheticSlot("mp_ass_subscript", ["__setitem__", "__delitem__"], "0"), + ) + + self.PyBufferProcs = ( + MethodSlot(readbufferproc, "bf_getreadbuffer", "__getreadbuffer__", method_name_to_slot, + py3 = False), + MethodSlot(writebufferproc, "bf_getwritebuffer", "__getwritebuffer__", method_name_to_slot, + py3 = False), + MethodSlot(segcountproc, "bf_getsegcount", "__getsegcount__", method_name_to_slot, + py3 = False), + MethodSlot(charbufferproc, "bf_getcharbuffer", "__getcharbuffer__", method_name_to_slot, + py3 = False), + + MethodSlot(getbufferproc, "bf_getbuffer", "__getbuffer__", method_name_to_slot), + MethodSlot(releasebufferproc, "bf_releasebuffer", "__releasebuffer__", method_name_to_slot) + ) + + self.PyAsyncMethods = ( + MethodSlot(unaryfunc, "am_await", "__await__", method_name_to_slot), + MethodSlot(unaryfunc, "am_aiter", "__aiter__", method_name_to_slot), + MethodSlot(unaryfunc, "am_anext", "__anext__", method_name_to_slot), + EmptySlot("am_send", ifdef="PY_VERSION_HEX >= 0x030A00A3"), + ) + + self.slot_table = ( + ConstructorSlot("tp_dealloc", '__dealloc__'), + EmptySlot("tp_print", ifdef="PY_VERSION_HEX < 0x030800b4"), + EmptySlot("tp_vectorcall_offset", ifdef="PY_VERSION_HEX >= 0x030800b4"), + EmptySlot("tp_getattr"), + EmptySlot("tp_setattr"), + + # tp_compare (Py2) / tp_reserved (Py3<3.5) / tp_as_async (Py3.5+) is always used as tp_as_async in Py3 + MethodSlot(cmpfunc, "tp_compare", "__cmp__", method_name_to_slot, ifdef="PY_MAJOR_VERSION < 3"), + SuiteSlot(self. PyAsyncMethods, "__Pyx_PyAsyncMethodsStruct", "tp_as_async", + self.substructures, ifdef="PY_MAJOR_VERSION >= 3"), + + MethodSlot(reprfunc, "tp_repr", "__repr__", method_name_to_slot), + + SuiteSlot(self.PyNumberMethods, "PyNumberMethods", "tp_as_number", self.substructures), + SuiteSlot(self.PySequenceMethods, "PySequenceMethods", "tp_as_sequence", self.substructures), + SuiteSlot(self.PyMappingMethods, "PyMappingMethods", "tp_as_mapping", self.substructures), + + MethodSlot(hashfunc, "tp_hash", "__hash__", method_name_to_slot, + inherited=False), # Py3 checks for __richcmp__ + MethodSlot(callfunc, "tp_call", "__call__", method_name_to_slot), + MethodSlot(reprfunc, "tp_str", "__str__", method_name_to_slot), + + SyntheticSlot("tp_getattro", ["__getattr__","__getattribute__"], "0"), #"PyObject_GenericGetAttr"), + SyntheticSlot("tp_setattro", ["__setattr__", "__delattr__"], "0"), #"PyObject_GenericSetAttr"), + + SuiteSlot(self.PyBufferProcs, "PyBufferProcs", "tp_as_buffer", self.substructures), + + TypeFlagsSlot("tp_flags"), + DocStringSlot("tp_doc"), + + GCDependentSlot("tp_traverse"), + GCClearReferencesSlot("tp_clear"), + + RichcmpSlot(richcmpfunc, "tp_richcompare", "__richcmp__", method_name_to_slot, + inherited=False), # Py3 checks for __hash__ + + EmptySlot("tp_weaklistoffset"), + + MethodSlot(getiterfunc, "tp_iter", "__iter__", method_name_to_slot), + MethodSlot(iternextfunc, "tp_iternext", "__next__", method_name_to_slot), + + MethodTableSlot("tp_methods"), + MemberTableSlot("tp_members"), + GetSetSlot("tp_getset"), + + BaseClassSlot("tp_base"), #EmptySlot("tp_base"), + EmptySlot("tp_dict"), + + SyntheticSlot("tp_descr_get", ["__get__"], "0"), + SyntheticSlot("tp_descr_set", ["__set__", "__delete__"], "0"), + + DictOffsetSlot("tp_dictoffset", ifdef="!CYTHON_USE_TYPE_SPECS"), # otherwise set via "__dictoffset__" member + + MethodSlot(initproc, "tp_init", "__init__", method_name_to_slot), + EmptySlot("tp_alloc"), #FixedSlot("tp_alloc", "PyType_GenericAlloc"), + ConstructorSlot("tp_new", "__cinit__"), + EmptySlot("tp_free"), + + EmptySlot("tp_is_gc"), + EmptySlot("tp_bases"), + EmptySlot("tp_mro"), + EmptySlot("tp_cache"), + EmptySlot("tp_subclasses"), + EmptySlot("tp_weaklist"), + EmptySlot("tp_del"), + EmptySlot("tp_version_tag"), + SyntheticSlot("tp_finalize", ["__del__"], "0", ifdef="PY_VERSION_HEX >= 0x030400a1", + used_ifdef="CYTHON_USE_TP_FINALIZE"), + EmptySlot("tp_vectorcall", ifdef="PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800)"), + EmptySlot("tp_print", ifdef="__PYX_NEED_TP_PRINT_SLOT == 1"), + EmptySlot("tp_watched", ifdef="PY_VERSION_HEX >= 0x030C0000"), + # PyPy specific extension - only here to avoid C compiler warnings. + EmptySlot("tp_pypy_flags", ifdef="CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000"), + ) + + #------------------------------------------------------------------------------------------ + # + # Descriptors for special methods which don't appear directly + # in the type object or its substructures. These methods are + # called from slot functions synthesized by Cython. + # + #------------------------------------------------------------------------------------------ + + MethodSlot(initproc, "", "__cinit__", method_name_to_slot) + MethodSlot(destructor, "", "__dealloc__", method_name_to_slot) + MethodSlot(destructor, "", "__del__", method_name_to_slot) + MethodSlot(objobjargproc, "", "__setitem__", method_name_to_slot) + MethodSlot(objargproc, "", "__delitem__", method_name_to_slot) + MethodSlot(ssizessizeobjargproc, "", "__setslice__", method_name_to_slot) + MethodSlot(ssizessizeargproc, "", "__delslice__", method_name_to_slot) + MethodSlot(getattrofunc, "", "__getattr__", method_name_to_slot) + MethodSlot(getattrofunc, "", "__getattribute__", method_name_to_slot) + MethodSlot(setattrofunc, "", "__setattr__", method_name_to_slot) + MethodSlot(delattrofunc, "", "__delattr__", method_name_to_slot) + MethodSlot(descrgetfunc, "", "__get__", method_name_to_slot) + MethodSlot(descrsetfunc, "", "__set__", method_name_to_slot) + MethodSlot(descrdelfunc, "", "__delete__", method_name_to_slot) + + def get_special_method_signature(self, name): + # Given a method name, if it is a special method, + # return its signature, else return None. + slot = self._get_slot_by_method_name(name) + if slot: + return slot.signature + elif name in richcmp_special_methods: + return ibinaryfunc + else: + return None - MethodTableSlot("tp_methods"), - MemberTableSlot("tp_members"), - GetSetSlot("tp_getset"), + def get_slot_by_method_name(self, method_name): + # For now, only search the type struct, no referenced sub-structs. + return self._get_slot_by_method_name(method_name) - BaseClassSlot("tp_base"), #EmptySlot("tp_base"), - EmptySlot("tp_dict"), + def __iter__(self): + # make it easier to iterate over all the slots + return iter(self.slot_table) - SyntheticSlot("tp_descr_get", ["__get__"], "0"), - SyntheticSlot("tp_descr_set", ["__set__", "__delete__"], "0"), - DictOffsetSlot("tp_dictoffset"), +_slot_table_dict = {} - MethodSlot(initproc, "tp_init", "__init__"), - EmptySlot("tp_alloc"), #FixedSlot("tp_alloc", "PyType_GenericAlloc"), - InternalMethodSlot("tp_new"), - EmptySlot("tp_free"), +def get_slot_table(compiler_directives): + if not compiler_directives: + # fetch default directives here since the builtin type classes don't have + # directives set + from .Options import get_directive_defaults + compiler_directives = get_directive_defaults() - EmptySlot("tp_is_gc"), - EmptySlot("tp_bases"), - EmptySlot("tp_mro"), - EmptySlot("tp_cache"), - EmptySlot("tp_subclasses"), - EmptySlot("tp_weaklist"), - EmptySlot("tp_del"), - EmptySlot("tp_version_tag"), - EmptySlot("tp_finalize", ifdef="PY_VERSION_HEX >= 0x030400a1"), - EmptySlot("tp_vectorcall", ifdef="PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800)"), - EmptySlot("tp_print", ifdef="PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000"), - # PyPy specific extension - only here to avoid C compiler warnings. - EmptySlot("tp_pypy_flags", ifdef="CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX >= 0x03090000 && PY_VERSION_HEX < 0x030a0000"), -) + old_binops = compiler_directives['c_api_binop_methods'] + key = (old_binops,) + if key not in _slot_table_dict: + _slot_table_dict[key] = SlotTable(old_binops=old_binops) + return _slot_table_dict[key] -#------------------------------------------------------------------------------------------ -# -# Descriptors for special methods which don't appear directly -# in the type object or its substructures. These methods are -# called from slot functions synthesized by Cython. -# -#------------------------------------------------------------------------------------------ -MethodSlot(initproc, "", "__cinit__") -MethodSlot(destructor, "", "__dealloc__") -MethodSlot(objobjargproc, "", "__setitem__") -MethodSlot(objargproc, "", "__delitem__") -MethodSlot(ssizessizeobjargproc, "", "__setslice__") -MethodSlot(ssizessizeargproc, "", "__delslice__") -MethodSlot(getattrofunc, "", "__getattr__") -MethodSlot(getattrofunc, "", "__getattribute__") -MethodSlot(setattrofunc, "", "__setattr__") -MethodSlot(delattrofunc, "", "__delattr__") -MethodSlot(descrgetfunc, "", "__get__") -MethodSlot(descrsetfunc, "", "__set__") -MethodSlot(descrdelfunc, "", "__delete__") +# Populate "special_method_names" based on the default directives (so it can always be accessed quickly). +special_method_names = set(get_slot_table(compiler_directives=None)) # Method flags for python-exposed methods. @@ -937,5 +1174,6 @@ MethodSlot(descrdelfunc, "", "__delete__") method_noargs = "METH_NOARGS" method_onearg = "METH_O" method_varargs = "METH_VARARGS" +method_fastcall = "__Pyx_METH_FASTCALL" # Actually VARARGS on versions < 3.7 method_keywords = "METH_KEYWORDS" method_coexist = "METH_COEXIST" diff --git a/Cython/Compiler/UFuncs.py b/Cython/Compiler/UFuncs.py new file mode 100644 index 000000000..5e641d785 --- /dev/null +++ b/Cython/Compiler/UFuncs.py @@ -0,0 +1,286 @@ +from . import ( + Nodes, + ExprNodes, + FusedNode, + TreeFragment, + Pipeline, + ParseTreeTransforms, + Naming, + UtilNodes, +) +from .Errors import error +from . import PyrexTypes +from .UtilityCode import CythonUtilityCode +from .Code import TempitaUtilityCode, UtilityCode +from .Visitor import PrintTree, TreeVisitor, VisitorTransform + +numpy_int_types = [ + "NPY_BYTE", + "NPY_INT8", + "NPY_SHORT", + "NPY_INT16", + "NPY_INT", + "NPY_INT32", + "NPY_LONG", + "NPY_LONGLONG", + "NPY_INT64", +] +numpy_uint_types = [tp.replace("NPY_", "NPY_U") for tp in numpy_int_types] +# note: half float type is deliberately omitted +numpy_numeric_types = ( + numpy_int_types + + numpy_uint_types + + [ + "NPY_FLOAT", + "NPY_FLOAT32", + "NPY_DOUBLE", + "NPY_FLOAT64", + "NPY_LONGDOUBLE", + ] +) + + +def _get_type_constant(pos, type_): + if type_.is_complex: + # 'is' checks don't seem to work for complex types + if type_ == PyrexTypes.c_float_complex_type: + return "NPY_CFLOAT" + elif type_ == PyrexTypes.c_double_complex_type: + return "NPY_CDOUBLE" + elif type_ == PyrexTypes.c_longdouble_complex_type: + return "NPY_CLONGDOUBLE" + elif type_.is_numeric: + postfix = type_.empty_declaration_code().upper().replace(" ", "") + typename = "NPY_%s" % postfix + if typename in numpy_numeric_types: + return typename + elif type_.is_pyobject: + return "NPY_OBJECT" + # TODO possible NPY_BOOL to bint but it needs a cast? + # TODO NPY_DATETIME, NPY_TIMEDELTA, NPY_STRING, NPY_UNICODE and maybe NPY_VOID might be handleable + error(pos, "Type '%s' cannot be used as a ufunc argument" % type_) + + +class _FindCFuncDefNode(TreeVisitor): + """ + Finds the CFuncDefNode in the tree + + The assumption is that there's only one CFuncDefNode + """ + + found_node = None + + def visit_Node(self, node): + if self.found_node: + return + else: + self.visitchildren(node) + + def visit_CFuncDefNode(self, node): + self.found_node = node + + def __call__(self, tree): + self.visit(tree) + return self.found_node + + +def get_cfunc_from_tree(tree): + return _FindCFuncDefNode()(tree) + + +class _ArgumentInfo(object): + """ + Everything related to defining an input/output argument for a ufunc + + type - PyrexType + type_constant - str such as "NPY_INT8" representing numpy dtype constants + """ + + def __init__(self, type, type_constant): + self.type = type + self.type_constant = type_constant + + +class UFuncConversion(object): + def __init__(self, node): + self.node = node + self.global_scope = node.local_scope.global_scope() + + self.in_definitions = self.get_in_type_info() + self.out_definitions = self.get_out_type_info() + + def get_in_type_info(self): + definitions = [] + for n, arg in enumerate(self.node.args): + type_const = _get_type_constant(self.node.pos, arg.type) + definitions.append(_ArgumentInfo(arg.type, type_const)) + return definitions + + def get_out_type_info(self): + if self.node.return_type.is_ctuple: + components = self.node.return_type.components + else: + components = [self.node.return_type] + definitions = [] + for n, type in enumerate(components): + definitions.append( + _ArgumentInfo(type, _get_type_constant(self.node.pos, type)) + ) + return definitions + + def generate_cy_utility_code(self): + arg_types = [a.type for a in self.in_definitions] + out_types = [a.type for a in self.out_definitions] + inline_func_decl = self.node.entry.type.declaration_code( + self.node.entry.cname, pyrex=True + ) + self.node.entry.used = True + + ufunc_cname = self.global_scope.next_id(self.node.entry.name + "_ufunc_def") + + will_be_called_without_gil = not (any(t.is_pyobject for t in arg_types) or + any(t.is_pyobject for t in out_types)) + + context = dict( + func_cname=ufunc_cname, + in_types=arg_types, + out_types=out_types, + inline_func_call=self.node.entry.cname, + inline_func_declaration=inline_func_decl, + nogil=self.node.entry.type.nogil, + will_be_called_without_gil=will_be_called_without_gil, + ) + + code = CythonUtilityCode.load( + "UFuncDefinition", + "UFuncs.pyx", + context=context, + outer_module_scope=self.global_scope, + ) + + tree = code.get_tree(entries_only=True) + return tree + + def use_generic_utility_code(self): + # use the invariant C utility code + self.global_scope.use_utility_code( + UtilityCode.load_cached("UFuncsInit", "UFuncs_C.c") + ) + self.global_scope.use_utility_code( + UtilityCode.load_cached("NumpyImportUFunc", "NumpyImportArray.c") + ) + + +def convert_to_ufunc(node): + if isinstance(node, Nodes.CFuncDefNode): + if node.local_scope.parent_scope.is_c_class_scope: + error(node.pos, "Methods cannot currently be converted to a ufunc") + return node + converters = [UFuncConversion(node)] + original_node = node + elif isinstance(node, FusedNode.FusedCFuncDefNode) and isinstance( + node.node, Nodes.CFuncDefNode + ): + if node.node.local_scope.parent_scope.is_c_class_scope: + error(node.pos, "Methods cannot currently be converted to a ufunc") + return node + converters = [UFuncConversion(n) for n in node.nodes] + original_node = node.node + else: + error(node.pos, "Only C functions can be converted to a ufunc") + return node + + if not converters: + return # this path probably shouldn't happen + + del converters[0].global_scope.entries[original_node.entry.name] + # the generic utility code is generic, so there's no reason to do it multiple times + converters[0].use_generic_utility_code() + return [node] + _generate_stats_from_converters(converters, original_node) + + +def generate_ufunc_initialization(converters, cfunc_nodes, original_node): + global_scope = converters[0].global_scope + ufunc_funcs_name = global_scope.next_id(Naming.pyrex_prefix + "funcs") + ufunc_types_name = global_scope.next_id(Naming.pyrex_prefix + "types") + ufunc_data_name = global_scope.next_id(Naming.pyrex_prefix + "data") + type_constants = [] + narg_in = None + narg_out = None + for c in converters: + in_const = [d.type_constant for d in c.in_definitions] + if narg_in is not None: + assert narg_in == len(in_const) + else: + narg_in = len(in_const) + type_constants.extend(in_const) + out_const = [d.type_constant for d in c.out_definitions] + if narg_out is not None: + assert narg_out == len(out_const) + else: + narg_out = len(out_const) + type_constants.extend(out_const) + + func_cnames = [cfnode.entry.cname for cfnode in cfunc_nodes] + + context = dict( + ufunc_funcs_name=ufunc_funcs_name, + func_cnames=func_cnames, + ufunc_types_name=ufunc_types_name, + type_constants=type_constants, + ufunc_data_name=ufunc_data_name, + ) + global_scope.use_utility_code( + TempitaUtilityCode.load("UFuncConsts", "UFuncs_C.c", context=context) + ) + + pos = original_node.pos + func_name = original_node.entry.name + docstr = original_node.doc + args_to_func = '%s(), %s, %s(), %s, %s, %s, PyUFunc_None, "%s", %s, 0' % ( + ufunc_funcs_name, + ufunc_data_name, + ufunc_types_name, + len(func_cnames), + narg_in, + narg_out, + func_name, + docstr.as_c_string_literal() if docstr else "NULL", + ) + + call_node = ExprNodes.PythonCapiCallNode( + pos, + function_name="PyUFunc_FromFuncAndData", + # use a dummy type because it's honestly too fiddly + func_type=PyrexTypes.CFuncType( + PyrexTypes.py_object_type, + [PyrexTypes.CFuncTypeArg("dummy", PyrexTypes.c_void_ptr_type, None)], + ), + args=[ + ExprNodes.ConstNode( + pos, type=PyrexTypes.c_void_ptr_type, value=args_to_func + ) + ], + ) + lhs_entry = global_scope.declare_var(func_name, PyrexTypes.py_object_type, pos) + assgn_node = Nodes.SingleAssignmentNode( + pos, + lhs=ExprNodes.NameNode( + pos, name=func_name, type=PyrexTypes.py_object_type, entry=lhs_entry + ), + rhs=call_node, + ) + return assgn_node + + +def _generate_stats_from_converters(converters, node): + stats = [] + for converter in converters: + tree = converter.generate_cy_utility_code() + ufunc_node = get_cfunc_from_tree(tree) + # merge in any utility code + converter.global_scope.utility_code_list.extend(tree.scope.utility_code_list) + stats.append(ufunc_node) + + stats.append(generate_ufunc_initialization(converters, stats, node)) + return stats diff --git a/Cython/Compiler/UtilNodes.py b/Cython/Compiler/UtilNodes.py index c41748ace..81d3038ea 100644 --- a/Cython/Compiler/UtilNodes.py +++ b/Cython/Compiler/UtilNodes.py @@ -10,7 +10,7 @@ from . import Nodes from . import ExprNodes from .Nodes import Node from .ExprNodes import AtomicExprNode -from .PyrexTypes import c_ptr_type +from .PyrexTypes import c_ptr_type, c_bint_type class TempHandle(object): @@ -45,7 +45,7 @@ class TempRefNode(AtomicExprNode): def calculate_result_code(self): result = self.handle.temp - if result is None: result = "<error>" # might be called and overwritten + if result is None: result = "<error>" # might be called and overwritten return result def generate_result_code(self, code): @@ -122,8 +122,7 @@ class ResultRefNode(AtomicExprNode): self.may_hold_none = may_hold_none if expression is not None: self.pos = expression.pos - if hasattr(expression, "type"): - self.type = expression.type + self.type = getattr(expression, "type", None) if pos is not None: self.pos = pos if type is not None: @@ -144,13 +143,17 @@ class ResultRefNode(AtomicExprNode): def update_expression(self, expression): self.expression = expression - if hasattr(expression, "type"): - self.type = expression.type + type = getattr(expression, "type", None) + if type: + self.type = type + + def analyse_target_declaration(self, env): + pass # OK - we can assign to this def analyse_types(self, env): if self.expression is not None: if not self.expression.type: - self.expression = self.expression.analyse_types(env) + self.expression = self.expression.analyse_types(env) self.type = self.expression.type return self @@ -175,7 +178,7 @@ class ResultRefNode(AtomicExprNode): return self.expression.may_be_none() if self.type is not None: return self.type.is_pyobject - return True # play safe + return True # play it safe def is_simple(self): return True @@ -233,7 +236,10 @@ class LetNodeMixin: if self._result_in_temp: self.temp = self.temp_expression.result() else: - self.temp_expression.make_owned_reference(code) + if self.temp_type.is_memoryviewslice: + self.temp_expression.make_owned_memoryviewslice(code) + else: + self.temp_expression.make_owned_reference(code) self.temp = code.funcstate.allocate_temp( self.temp_type, manage_ref=True) code.putln("%s = %s;" % (self.temp, self.temp_expression.result())) @@ -246,7 +252,7 @@ class LetNodeMixin: self.temp_expression.generate_disposal_code(code) self.temp_expression.free_temps(code) else: - if self.temp_type.is_pyobject: + if self.temp_type.needs_refcounting: code.put_decref_clear(self.temp, self.temp_type) code.funcstate.release_temp(self.temp) @@ -354,6 +360,29 @@ class TempResultFromStatNode(ExprNodes.ExprNode): self.body = self.body.analyse_expressions(env) return self + def may_be_none(self): + return self.result_ref.may_be_none() + def generate_result_code(self, code): self.result_ref.result_code = self.result() self.body.generate_execution_code(code) + + def generate_function_definitions(self, env, code): + self.body.generate_function_definitions(env, code) + + +class HasGilNode(AtomicExprNode): + """ + Simple node that evaluates to 0 or 1 depending on whether we're + in a nogil context + """ + type = c_bint_type + + def analyse_types(self, env): + return self + + def generate_result_code(self, code): + self.has_gil = code.funcstate.gil_owned + + def calculate_result_code(self): + return "1" if self.has_gil else "0" diff --git a/Cython/Compiler/UtilityCode.py b/Cython/Compiler/UtilityCode.py index 98e9ab5bf..e2df2586b 100644 --- a/Cython/Compiler/UtilityCode.py +++ b/Cython/Compiler/UtilityCode.py @@ -131,7 +131,7 @@ class CythonUtilityCode(Code.UtilityCodeBase): p = [] for t in pipeline: p.append(t) - if isinstance(p, ParseTreeTransforms.AnalyseDeclarationsTransform): + if isinstance(t, ParseTreeTransforms.AnalyseDeclarationsTransform): break pipeline = p @@ -173,8 +173,15 @@ class CythonUtilityCode(Code.UtilityCodeBase): if self.context_types: # inject types into module scope def scope_transform(module_node): + dummy_entry = object() for name, type in self.context_types.items(): + # Restore the old type entry after declaring the type. + # We need to access types in the scope, but this shouldn't alter the entry + # that is visible from everywhere else + old_type_entry = getattr(type, "entry", dummy_entry) entry = module_node.scope.declare_type(name, type, None, visibility='extern') + if old_type_entry is not dummy_entry: + type.entry = old_type_entry entry.in_cinclude = True return module_node @@ -196,10 +203,10 @@ class CythonUtilityCode(Code.UtilityCodeBase): Load a utility code as a string. Returns (proto, implementation) """ util = cls.load(util_code_name, from_file, **kwargs) - return util.proto, util.impl # keep line numbers => no lstrip() + return util.proto, util.impl # keep line numbers => no lstrip() def declare_in_scope(self, dest_scope, used=False, cython_scope=None, - whitelist=None): + allowlist=None): """ Declare all entries from the utility code in dest_scope. Code will only be included for used entries. If module_name is given, declare the @@ -218,7 +225,7 @@ class CythonUtilityCode(Code.UtilityCodeBase): entry.used = used original_scope = tree.scope - dest_scope.merge_in(original_scope, merge_unused=True, whitelist=whitelist) + dest_scope.merge_in(original_scope, merge_unused=True, allowlist=allowlist) tree.scope = dest_scope for dep in self.requires: @@ -227,6 +234,27 @@ class CythonUtilityCode(Code.UtilityCodeBase): return original_scope + @staticmethod + def filter_inherited_directives(current_directives): + """ + Cython utility code should usually only pick up a few directives from the + environment (those that intentionally control its function) and ignore most + other compiler directives. This function provides a sensible default list + of directives to copy. + """ + from .Options import _directive_defaults + utility_code_directives = dict(_directive_defaults) + inherited_directive_names = ( + 'binding', 'always_allow_keywords', 'allow_none_for_extension_args', + 'auto_pickle', 'ccomplex', + 'c_string_type', 'c_string_encoding', + 'optimize.inline_defnode_calls', 'optimize.unpack_method_calls', + 'optimize.unpack_method_calls_in_pyinit', 'optimize.use_switch') + for name in inherited_directive_names: + if name in current_directives: + utility_code_directives[name] = current_directives[name] + return utility_code_directives + def declare_declarations_in_scope(declaration_string, env, private_type=True, *args, **kwargs): diff --git a/Cython/Compiler/Visitor.pxd b/Cython/Compiler/Visitor.pxd index d5d5692aa..9c5aac6dc 100644 --- a/Cython/Compiler/Visitor.pxd +++ b/Cython/Compiler/Visitor.pxd @@ -1,4 +1,4 @@ -from __future__ import absolute_import +# cython: language_level=3str cimport cython @@ -10,15 +10,15 @@ cdef class TreeVisitor: cdef _visit(self, obj) cdef find_handler(self, obj) cdef _visitchild(self, child, parent, attrname, idx) - cdef dict _visitchildren(self, parent, attrs) - cpdef visitchildren(self, parent, attrs=*) + cdef dict _visitchildren(self, parent, attrs, exclude) + cpdef visitchildren(self, parent, attrs=*, exclude=*) cdef _raise_compiler_error(self, child, e) cdef class VisitorTransform(TreeVisitor): - cdef dict _process_children(self, parent, attrs=*) + cdef dict _process_children(self, parent, attrs=*, exclude=*) cpdef visitchildren(self, parent, attrs=*, exclude=*) cdef list _flatten_list(self, list orig_list) - cdef list _select_attrs(self, attrs, exclude) + cpdef visitchild(self, parent, str attr, idx=*) cdef class CythonTransform(VisitorTransform): cdef public context @@ -47,8 +47,8 @@ cdef class MethodDispatcherTransform(EnvTransform): node, function, arg_list, kwargs) cdef class RecursiveNodeReplacer(VisitorTransform): - cdef public orig_node - cdef public new_node + cdef public orig_node + cdef public new_node cdef class NodeFinder(TreeVisitor): cdef node diff --git a/Cython/Compiler/Visitor.py b/Cython/Compiler/Visitor.py index a35d13e1d..92e2eb9c0 100644 --- a/Cython/Compiler/Visitor.py +++ b/Cython/Compiler/Visitor.py @@ -1,5 +1,5 @@ # cython: infer_types=True -# cython: language_level=3 +# cython: language_level=3str # cython: auto_pickle=False # @@ -80,7 +80,7 @@ class TreeVisitor(object): def dump_node(self, node): ignored = list(node.child_attrs or []) + [ - u'child_attrs', u'pos', u'gil_message', u'cpp_message', u'subexprs'] + 'child_attrs', 'pos', 'gil_message', 'cpp_message', 'subexprs'] values = [] pos = getattr(node, 'pos', None) if pos: @@ -116,7 +116,7 @@ class TreeVisitor(object): nodes = [] while hasattr(stacktrace, 'tb_frame'): frame = stacktrace.tb_frame - node = frame.f_locals.get(u'self') + node = frame.f_locals.get('self') if isinstance(node, Nodes.Node): code = frame.f_code method_name = code.co_name @@ -153,12 +153,12 @@ class TreeVisitor(object): def find_handler(self, obj): # to resolve, try entire hierarchy cls = type(obj) - pattern = "visit_%s" mro = inspect.getmro(cls) for mro_cls in mro: - handler_method = getattr(self, pattern % mro_cls.__name__, None) + handler_method = getattr(self, "visit_" + mro_cls.__name__, None) if handler_method is not None: return handler_method + print(type(self), cls) if self.access_path: print(self.access_path) @@ -167,10 +167,12 @@ class TreeVisitor(object): raise RuntimeError("Visitor %r does not accept object: %s" % (self, obj)) def visit(self, obj): + # generic def entry point for calls from Python subclasses return self._visit(obj) @cython.final def _visit(self, obj): + # fast cdef entry point for calls from Cython subclasses try: try: handler_method = self.dispatch_table[type(obj)] @@ -189,17 +191,20 @@ class TreeVisitor(object): @cython.final def _visitchild(self, child, parent, attrname, idx): + # fast cdef entry point for calls from Cython subclasses self.access_path.append((parent, attrname, idx)) result = self._visit(child) self.access_path.pop() return result - def visitchildren(self, parent, attrs=None): - return self._visitchildren(parent, attrs) + def visitchildren(self, parent, attrs=None, exclude=None): + # generic def entry point for calls from Python subclasses + return self._visitchildren(parent, attrs, exclude) @cython.final @cython.locals(idx=cython.Py_ssize_t) - def _visitchildren(self, parent, attrs): + def _visitchildren(self, parent, attrs, exclude): + # fast cdef entry point for calls from Cython subclasses """ Visits the children of the given parent. If parent is None, returns immediately (returning None). @@ -213,6 +218,7 @@ class TreeVisitor(object): result = {} for attr in parent.child_attrs: if attrs is not None and attr not in attrs: continue + if exclude is not None and attr in exclude: continue child = getattr(parent, attr) if child is not None: if type(child) is list: @@ -246,18 +252,12 @@ class VisitorTransform(TreeVisitor): """ def visitchildren(self, parent, attrs=None, exclude=None): # generic def entry point for calls from Python subclasses - if exclude is not None: - attrs = self._select_attrs(parent.child_attrs if attrs is None else attrs, exclude) - return self._process_children(parent, attrs) + return self._process_children(parent, attrs, exclude) @cython.final - def _select_attrs(self, attrs, exclude): - return [name for name in attrs if name not in exclude] - - @cython.final - def _process_children(self, parent, attrs=None): + def _process_children(self, parent, attrs=None, exclude=None): # fast cdef entry point for calls from Cython subclasses - result = self._visitchildren(parent, attrs) + result = self._visitchildren(parent, attrs, exclude) for attr, newnode in result.items(): if type(newnode) is list: newnode = self._flatten_list(newnode) @@ -276,6 +276,16 @@ class VisitorTransform(TreeVisitor): newlist.append(x) return newlist + def visitchild(self, parent, attr, idx=0): + # Helper to visit specific children from Python subclasses + child = getattr(parent, attr) + if child is not None: + node = self._visitchild(child, parent, attr, idx) + if node is not child: + setattr(parent, attr, node) + child = node + return child + def recurse_to_children(self, node): self._process_children(node) return node @@ -296,8 +306,8 @@ class CythonTransform(VisitorTransform): self.context = context def __call__(self, node): - from . import ModuleNode - if isinstance(node, ModuleNode.ModuleNode): + from .ModuleNode import ModuleNode + if isinstance(node, ModuleNode): self.current_directives = node.directives return super(CythonTransform, self).__call__(node) @@ -370,11 +380,15 @@ class EnvTransform(CythonTransform): self.env_stack.pop() def visit_FuncDefNode(self, node): + self.visit_func_outer_attrs(node) self.enter_scope(node, node.local_scope) - self._process_children(node) + self.visitchildren(node, attrs=None, exclude=node.outer_attrs) self.exit_scope() return node + def visit_func_outer_attrs(self, node): + self.visitchildren(node, attrs=node.outer_attrs) + def visit_GeneratorBodyDefNode(self, node): self._process_children(node) return node @@ -569,7 +583,18 @@ class MethodDispatcherTransform(EnvTransform): ### dispatch to specific handlers def _find_handler(self, match_name, has_kwargs): - call_type = has_kwargs and 'general' or 'simple' + try: + match_name.encode('ascii') + except UnicodeEncodeError: + # specifically when running the Cython compiler under Python 2 + # getattr can't take a unicode string. + # Classes with unicode names won't have specific handlers and thus it + # should be OK to return None. + # Doing the test here ensures that the same code gets run on + # Python 2 and 3 + return None + + call_type = 'general' if has_kwargs else 'simple' handler = getattr(self, '_handle_%s_%s' % (call_type, match_name), None) if handler is None: handler = getattr(self, '_handle_any_%s' % match_name, None) @@ -662,8 +687,8 @@ class MethodDispatcherTransform(EnvTransform): method_handler = self._find_handler( "method_%s_%s" % (type_name, attr_name), kwargs) if method_handler is None: - if (attr_name in TypeSlots.method_name_to_slot - or attr_name == '__new__'): + if (attr_name in TypeSlots.special_method_names + or attr_name in ['__new__', '__class__']): method_handler = self._find_handler( "slot%s" % attr_name, kwargs) if method_handler is None: @@ -733,7 +758,7 @@ class NodeFinder(TreeVisitor): elif node is self.node: self.found = True else: - self._visitchildren(node, None) + self._visitchildren(node, None, None) def tree_contains(tree, node): finder = NodeFinder(node) @@ -821,6 +846,12 @@ class PrintTree(TreeVisitor): result += "(type=%s, name=\"%s\")" % (repr(node.type), node.name) elif isinstance(node, Nodes.DefNode): result += "(name=\"%s\")" % node.name + elif isinstance(node, Nodes.CFuncDefNode): + result += "(name=\"%s\")" % node.declared_name() + elif isinstance(node, ExprNodes.AttributeNode): + result += "(type=%s, attribute=\"%s\")" % (repr(node.type), node.attribute) + elif isinstance(node, (ExprNodes.ConstNode, ExprNodes.PyConstNode)): + result += "(type=%s, value=%r)" % (repr(node.type), node.value) elif isinstance(node, ExprNodes.ExprNode): t = node.type result += "(type=%s)" % repr(t) diff --git a/Cython/Coverage.py b/Cython/Coverage.py index 269896e13..147df8050 100644 --- a/Cython/Coverage.py +++ b/Cython/Coverage.py @@ -2,6 +2,48 @@ A Cython plugin for coverage.py Requires the coverage package at least in version 4.0 (which added the plugin API). + +This plugin requires the generated C sources to be available, next to the extension module. +It parses the C file and reads the original source files from it, which are stored in C comments. +It then reports a source file to coverage.py when it hits one of its lines during line tracing. + +Basically, Cython can (on request) emit explicit trace calls into the C code that it generates, +and as a general human debugging helper, it always copies the current source code line +(and its surrounding context) into the C files before it generates code for that line, e.g. + +:: + + /* "line_trace.pyx":147 + * def cy_add_with_nogil(a,b): + * cdef int z, x=a, y=b # 1 + * with nogil: # 2 # <<<<<<<<<<<<<< + * z = 0 # 3 + * z += cy_add_nogil(x, y) # 4 + */ + __Pyx_TraceLine(147,1,__PYX_ERR(0, 147, __pyx_L4_error)) + [C code generated for file line_trace.pyx, line 147, follows here] + +The crux is that multiple source files can contribute code to a single C (or C++) file +(and thus, to a single extension module) besides the main module source file (.py/.pyx), +usually shared declaration files (.pxd) but also literally included files (.pxi). + +Therefore, the coverage plugin doesn't actually try to look at the file that happened +to contribute the current source line for the trace call, but simply looks up the single +.c file from which the extension was compiled (which usually lies right next to it after +the build, having the same name), and parses the code copy comments from that .c file +to recover the original source files and their code as a line-to-file mapping. + +That mapping is then used to report the ``__Pyx_TraceLine()`` calls to the coverage tool. +The plugin also reports the line of source code that it found in the C file to the coverage +tool to support annotated source representations. For this, again, it does not look at the +actual source files but only reports the source code that it found in the C code comments. + +Apart from simplicity (read one file instead of finding and parsing many), part of the +reasoning here is that any line in the original sources for which there is no comment line +(and trace call) in the generated C code cannot count as executed, really, so the C code +comments are a very good source for coverage reporting. They already filter out purely +declarative code lines that do not contribute executable code, and such (missing) lines +can then be marked as excluded from coverage analysis. """ from __future__ import absolute_import @@ -14,7 +56,7 @@ from collections import defaultdict from coverage.plugin import CoveragePlugin, FileTracer, FileReporter # requires coverage.py 4.0+ from coverage.files import canonical_filename -from .Utils import find_root_package_dir, is_package_dir, open_source_file +from .Utils import find_root_package_dir, is_package_dir, is_cython_generated_file, open_source_file from . import __version__ @@ -41,6 +83,23 @@ def _find_dep_file_path(main_file, file_path, relative_path_search=False): rel_file_path = os.path.join(os.path.dirname(main_file), file_path) if os.path.exists(rel_file_path): abs_path = os.path.abspath(rel_file_path) + + abs_no_ext = os.path.splitext(abs_path)[0] + file_no_ext, extension = os.path.splitext(file_path) + # We check if the paths match by matching the directories in reverse order. + # pkg/module.pyx /long/absolute_path/bla/bla/site-packages/pkg/module.c should match. + # this will match the pairs: module-module and pkg-pkg. After which there is nothing left to zip. + abs_no_ext = os.path.normpath(abs_no_ext) + file_no_ext = os.path.normpath(file_no_ext) + matching_paths = zip(reversed(abs_no_ext.split(os.sep)), reversed(file_no_ext.split(os.sep))) + for one, other in matching_paths: + if one != other: + break + else: # No mismatches detected + matching_abs_path = os.path.splitext(main_file)[0] + extension + if os.path.exists(matching_abs_path): + return canonical_filename(matching_abs_path) + # search sys.path for external locations if a valid file hasn't been found if not os.path.exists(abs_path): for sys_path in sys.path: @@ -57,10 +116,19 @@ class Plugin(CoveragePlugin): _c_files_map = None # map from parsed C files to their content _parsed_c_files = None + # map from traced files to lines that are excluded from coverage + _excluded_lines_map = None + # list of regex patterns for lines to exclude + _excluded_line_patterns = () def sys_info(self): return [('Cython version', __version__)] + def configure(self, config): + # Entry point for coverage "configurer". + # Read the regular expressions from the coverage config that match lines to be excluded from coverage. + self._excluded_line_patterns = config.get_option("report:exclude_lines") + def file_tracer(self, filename): """ Try to find a C source file for a file path found by the tracer. @@ -108,7 +176,13 @@ class Plugin(CoveragePlugin): rel_file_path, code = self._read_source_lines(c_file, filename) if code is None: return None # no source found - return CythonModuleReporter(c_file, filename, rel_file_path, code) + return CythonModuleReporter( + c_file, + filename, + rel_file_path, + code, + self._excluded_lines_map.get(rel_file_path, frozenset()) + ) def _find_source_files(self, filename): basename, ext = os.path.splitext(filename) @@ -150,12 +224,11 @@ class Plugin(CoveragePlugin): py_source_file = os.path.splitext(c_file)[0] + '.py' if not os.path.exists(py_source_file): py_source_file = None - - try: - with open(c_file, 'rb') as f: - if b'/* Generated by Cython ' not in f.read(30): - return None, None # not a Cython file - except (IOError, OSError): + if not is_cython_generated_file(c_file, if_not_found=False): + if py_source_file and os.path.exists(c_file): + # if we did not generate the C file, + # then we probably also shouldn't care about the .py file. + py_source_file = None c_file = None return c_file, py_source_file @@ -218,10 +291,16 @@ class Plugin(CoveragePlugin): r'(?:struct|union|enum|class)' r'(\s+[^:]+|)\s*:' ).match + if self._excluded_line_patterns: + line_is_excluded = re.compile("|".join(["(?:%s)" % regex for regex in self._excluded_line_patterns])).search + else: + line_is_excluded = lambda line: False code_lines = defaultdict(dict) executable_lines = defaultdict(set) current_filename = None + if self._excluded_lines_map is None: + self._excluded_lines_map = defaultdict(set) with open(c_file) as lines: lines = iter(lines) @@ -242,6 +321,9 @@ class Plugin(CoveragePlugin): code_line = match.group(1).rstrip() if not_executable(code_line): break + if line_is_excluded(code_line): + self._excluded_lines_map[filename].add(lineno) + break code_lines[filename][lineno] = code_line break elif match_comment_end(comment_line): @@ -298,11 +380,12 @@ class CythonModuleReporter(FileReporter): """ Provide detailed trace information for one source file to coverage.py. """ - def __init__(self, c_file, source_file, rel_file_path, code): + def __init__(self, c_file, source_file, rel_file_path, code, excluded_lines): super(CythonModuleReporter, self).__init__(source_file) self.name = rel_file_path self.c_file = c_file self._code = code + self._excluded_lines = excluded_lines def lines(self): """ @@ -310,6 +393,12 @@ class CythonModuleReporter(FileReporter): """ return set(self._code) + def excluded_lines(self): + """ + Return set of line numbers that are excluded from coverage. + """ + return self._excluded_lines + def _iter_source_tokens(self): current_line = 1 for line_no, code_line in sorted(self._code.items()): @@ -345,4 +434,6 @@ class CythonModuleReporter(FileReporter): def coverage_init(reg, options): - reg.add_file_tracer(Plugin()) + plugin = Plugin() + reg.add_configurer(plugin) + reg.add_file_tracer(plugin) diff --git a/Cython/Debugger/Cygdb.py b/Cython/Debugger/Cygdb.py index 45f31ce6f..596e5e11b 100644 --- a/Cython/Debugger/Cygdb.py +++ b/Cython/Debugger/Cygdb.py @@ -22,7 +22,9 @@ import logging logger = logging.getLogger(__name__) -def make_command_file(path_to_debug_info, prefix_code='', no_import=False): + +def make_command_file(path_to_debug_info, prefix_code='', + no_import=False, skip_interpreter=False): if not no_import: pattern = os.path.join(path_to_debug_info, 'cython_debug', @@ -45,17 +47,23 @@ def make_command_file(path_to_debug_info, prefix_code='', no_import=False): set print pretty on python - # Activate virtualenv, if we were launched from one - import os - virtualenv = os.getenv('VIRTUAL_ENV') - if virtualenv: - path_to_activate_this_py = os.path.join(virtualenv, 'bin', 'activate_this.py') - print("gdb command file: Activating virtualenv: %s; path_to_activate_this_py: %s" % ( - virtualenv, path_to_activate_this_py)) - with open(path_to_activate_this_py) as f: - exec(f.read(), dict(__file__=path_to_activate_this_py)) - - from Cython.Debugger import libcython, libpython + try: + # Activate virtualenv, if we were launched from one + import os + virtualenv = os.getenv('VIRTUAL_ENV') + if virtualenv: + path_to_activate_this_py = os.path.join(virtualenv, 'bin', 'activate_this.py') + print("gdb command file: Activating virtualenv: %s; path_to_activate_this_py: %s" % ( + virtualenv, path_to_activate_this_py)) + with open(path_to_activate_this_py) as f: + exec(f.read(), dict(__file__=path_to_activate_this_py)) + from Cython.Debugger import libcython, libpython + except Exception as ex: + from traceback import print_exc + print("There was an error in Python code originating from the file ''' + str(__file__) + '''") + print("It used the Python interpreter " + str(sys.executable)) + print_exc() + exit(1) end ''')) @@ -64,27 +72,33 @@ def make_command_file(path_to_debug_info, prefix_code='', no_import=False): # f.write("file %s\n" % sys.executable) pass else: - path = os.path.join(path_to_debug_info, "cython_debug", "interpreter") - interpreter_file = open(path) - try: - interpreter = interpreter_file.read() - finally: - interpreter_file.close() - f.write("file %s\n" % interpreter) - f.write('\n'.join('cy import %s\n' % fn for fn in debug_files)) - f.write(textwrap.dedent('''\ - python - import sys + if not skip_interpreter: + # Point Cygdb to the interpreter that was used to generate + # the debugging information. + path = os.path.join(path_to_debug_info, "cython_debug", "interpreter") + interpreter_file = open(path) try: - gdb.lookup_type('PyModuleObject') - except RuntimeError: - sys.stderr.write( - 'Python was not compiled with debug symbols (or it was ' - 'stripped). Some functionality may not work (properly).\\n') - end - - source .cygdbinit - ''')) + interpreter = interpreter_file.read() + finally: + interpreter_file.close() + f.write("file %s\n" % interpreter) + + f.write('\n'.join('cy import %s\n' % fn for fn in debug_files)) + + if not skip_interpreter: + f.write(textwrap.dedent('''\ + python + import sys + try: + gdb.lookup_type('PyModuleObject') + except RuntimeError: + sys.stderr.write( + "''' + interpreter + ''' was not compiled with debug symbols (or it was " + "stripped). Some functionality may not work (properly).\\n") + end + ''')) + + f.write("source .cygdbinit") finally: f.close() @@ -109,6 +123,10 @@ def main(path_to_debug_info=None, gdb_argv=None, no_import=False): parser.add_option("--verbose", "-v", dest="verbosity", action="count", default=0, help="Verbose mode. Multiple -v options increase the verbosity") + parser.add_option("--skip-interpreter", + dest="skip_interpreter", default=False, action="store_true", + help="Do not automatically point GDB to the same interpreter " + "used to generate debugging information") (options, args) = parser.parse_args() if path_to_debug_info is None: @@ -130,12 +148,16 @@ def main(path_to_debug_info=None, gdb_argv=None, no_import=False): logging_level = logging.DEBUG logging.basicConfig(level=logging_level) + skip_interpreter = options.skip_interpreter + logger.info("verbosity = %r", options.verbosity) logger.debug("options = %r; args = %r", options, args) logger.debug("Done parsing command-line options. path_to_debug_info = %r, gdb_argv = %r", path_to_debug_info, gdb_argv) - tempfilename = make_command_file(path_to_debug_info, no_import=no_import) + tempfilename = make_command_file(path_to_debug_info, + no_import=no_import, + skip_interpreter=skip_interpreter) logger.info("Launching %s with command file: %s and gdb_argv: %s", options.gdb, tempfilename, gdb_argv) with open(tempfilename) as tempfile: diff --git a/Cython/Debugger/DebugWriter.py b/Cython/Debugger/DebugWriter.py index 876a3a216..8b1fb75b0 100644 --- a/Cython/Debugger/DebugWriter.py +++ b/Cython/Debugger/DebugWriter.py @@ -5,8 +5,8 @@ import sys import errno try: - from lxml import etree - have_lxml = True + from lxml import etree + have_lxml = True except ImportError: have_lxml = False try: diff --git a/Cython/Debugger/Tests/TestLibCython.py b/Cython/Debugger/Tests/TestLibCython.py index 13560646f..0d8a3e613 100644 --- a/Cython/Debugger/Tests/TestLibCython.py +++ b/Cython/Debugger/Tests/TestLibCython.py @@ -56,13 +56,13 @@ def test_gdb(): stdout, _ = p.communicate() try: internal_python_version = list(map(int, stdout.decode('ascii', 'ignore').split())) - if internal_python_version < [2, 6]: + if internal_python_version < [2, 7]: have_gdb = False except ValueError: have_gdb = False if not have_gdb: - warnings.warn('Skipping gdb tests, need gdb >= 7.2 with Python >= 2.6') + warnings.warn('Skipping gdb tests, need gdb >= 7.2 with Python >= 2.7') return have_gdb @@ -99,6 +99,7 @@ class DebuggerTestCase(unittest.TestCase): opts = dict( test_directory=self.tempdir, module='codefile', + module_path=self.destfile, ) optimization_disabler = build_ext.Optimization() @@ -131,10 +132,11 @@ class DebuggerTestCase(unittest.TestCase): ) cython_compile_testcase.run_distutils( + test_directory=opts['test_directory'], + module=opts['module'], + workdir=opts['test_directory'], incdir=None, - workdir=self.tempdir, extra_extension_args={'extra_objects':['cfuncs.o']}, - **opts ) finally: optimization_disabler.restore_state() diff --git a/Cython/Debugger/Tests/codefile b/Cython/Debugger/Tests/codefile index 6b4c6b6ad..ee587cbb1 100644 --- a/Cython/Debugger/Tests/codefile +++ b/Cython/Debugger/Tests/codefile @@ -37,14 +37,13 @@ def outer(): def inner(): b = 2 # access closed over variables - print a, b + print(a, b) return inner - outer()() spam() -print "bye!" +print("bye!") def use_ham(): ham() diff --git a/Cython/Debugger/Tests/test_libcython_in_gdb.py b/Cython/Debugger/Tests/test_libcython_in_gdb.py index bd7608d60..bb06c2905 100644 --- a/Cython/Debugger/Tests/test_libcython_in_gdb.py +++ b/Cython/Debugger/Tests/test_libcython_in_gdb.py @@ -134,7 +134,7 @@ class TestDebugInformationClasses(DebugTestCase): self.assertEqual(self.spam_func.arguments, ['a']) self.assertEqual(self.spam_func.step_into_functions, - set(['puts', 'some_c_function'])) + {'puts', 'some_c_function'}) expected_lineno = test_libcython.source_to_lineno['def spam(a=0):'] self.assertEqual(self.spam_func.lineno, expected_lineno) @@ -177,12 +177,13 @@ class TestBreak(DebugTestCase): assert step_result.rstrip().endswith(nextline) -class TestKilled(DebugTestCase): - - def test_abort(self): - gdb.execute("set args -c 'import os; os.abort()'") - output = gdb.execute('cy run', to_string=True) - assert 'abort' in output.lower() +# I removed this testcase, because it will never work, because +# gdb.execute(..., to_string=True) does not capture stdout and stderr of python. +# class TestKilled(DebugTestCase): +# def test_abort(self): +# gdb.execute("set args -c 'import os;print(123456789);os.abort()'") +# output = gdb.execute('cy run', to_string=True) +# assert 'abort' in output.lower() class DebugStepperTestCase(DebugTestCase): @@ -322,6 +323,61 @@ class TestPrint(DebugTestCase): self.break_and_run('c = 2') result = gdb.execute('cy print b', to_string=True) self.assertEqual('b = (int) 1\n', result) + result = gdb.execute('cy print python_var', to_string=True) + self.assertEqual('python_var = 13\n', result) + result = gdb.execute('cy print c_var', to_string=True) + self.assertEqual('c_var = (int) 12\n', result) + +correct_result_test_list_inside_func = '''\ + 14 int b, c + 15 + 16 b = c = d = 0 + 17 + 18 b = 1 +> 19 c = 2 + 20 int(10) + 21 puts("spam") + 22 os.path.join("foo", "bar") + 23 some_c_function() +''' +correct_result_test_list_outside_func = '''\ + 5 void some_c_function() + 6 + 7 import os + 8 + 9 cdef int c_var = 12 +> 10 python_var = 13 + 11 + 12 def spam(a=0): + 13 cdef: + 14 int b, c +''' + + +class TestList(DebugTestCase): + def workaround_for_coding_style_checker(self, correct_result_wrong_whitespace): + correct_result = "" + for line in correct_result_test_list_inside_func.split("\n"): + if len(line) < 10 and len(line) > 0: + line += " "*4 + correct_result += line + "\n" + correct_result = correct_result[:-1] + + def test_list_inside_func(self): + self.break_and_run('c = 2') + result = gdb.execute('cy list', to_string=True) + # We don't want to fail because of some trailing whitespace, + # so we remove trailing whitespaces with the following line + result = "\n".join([line.rstrip() for line in result.split("\n")]) + self.assertEqual(correct_result_test_list_inside_func, result) + + def test_list_outside_func(self): + self.break_and_run('python_var = 13') + result = gdb.execute('cy list', to_string=True) + # We don't want to fail because of some trailing whitespace, + # so we remove trailing whitespaces with the following line + result = "\n".join([line.rstrip() for line in result.split("\n")]) + self.assertEqual(correct_result_test_list_outside_func, result) class TestUpDown(DebugTestCase): @@ -362,6 +418,7 @@ class TestExec(DebugTestCase): # test normal behaviour self.assertEqual("[0]", self.eval_command('[a]')) + return #The test after this return freezes gdb, so I temporarily removed it. # test multiline code result = gdb.execute(textwrap.dedent('''\ cy exec diff --git a/Cython/Debugger/Tests/test_libpython_in_gdb.py b/Cython/Debugger/Tests/test_libpython_in_gdb.py index 6f34cee47..4640dbac1 100644 --- a/Cython/Debugger/Tests/test_libpython_in_gdb.py +++ b/Cython/Debugger/Tests/test_libpython_in_gdb.py @@ -6,16 +6,13 @@ Lib/test/test_gdb.py in the Python source. These tests are run in gdb and called from test_libcython_in_gdb.main() """ -import os -import sys - import gdb from Cython.Debugger import libcython from Cython.Debugger import libpython from . import test_libcython_in_gdb -from .test_libcython_in_gdb import _debug, inferior_python_version +from .test_libcython_in_gdb import inferior_python_version class TestPrettyPrinters(test_libcython_in_gdb.DebugTestCase): diff --git a/Cython/Debugger/libcython.py b/Cython/Debugger/libcython.py index 23153789b..dd634cc35 100644 --- a/Cython/Debugger/libcython.py +++ b/Cython/Debugger/libcython.py @@ -11,7 +11,6 @@ except NameError: import sys import textwrap -import traceback import functools import itertools import collections @@ -69,19 +68,6 @@ _filesystemencoding = sys.getfilesystemencoding() or 'UTF-8' # decorators -def dont_suppress_errors(function): - "*sigh*, readline" - @functools.wraps(function) - def wrapper(*args, **kwargs): - try: - return function(*args, **kwargs) - except Exception: - traceback.print_exc() - raise - - return wrapper - - def default_selected_gdb_frame(err=True): def decorator(function): @functools.wraps(function) @@ -252,7 +238,9 @@ class CythonBase(object): filename = lineno = lexer = None if self.is_cython_function(frame): filename = self.get_cython_function(frame).module.filename - lineno = self.get_cython_lineno(frame) + filename_and_lineno = self.get_cython_lineno(frame) + assert filename == filename_and_lineno[0] + lineno = filename_and_lineno[1] if pygments: lexer = pygments.lexers.CythonLexer(stripall=False) elif self.is_python_function(frame): @@ -334,7 +322,7 @@ class CythonBase(object): func_name = cyfunc.name func_cname = cyfunc.cname - func_args = [] # [(arg, f(arg)) for arg in cyfunc.arguments] + func_args = [] # [(arg, f(arg)) for arg in cyfunc.arguments] else: source_desc, lineno = self.get_source_desc(frame) func_name = frame.name() @@ -392,7 +380,7 @@ class CythonBase(object): result = {} seen = set() - for k, v in pyobject_dict.items(): + for k, v in pyobject_dict.iteritems(): result[k.proxyval(seen)] = v return result @@ -410,7 +398,7 @@ class CythonBase(object): def is_initialized(self, cython_func, local_name): cyvar = cython_func.locals[local_name] - cur_lineno = self.get_cython_lineno() + cur_lineno = self.get_cython_lineno()[1] if '->' in cyvar.cname: # Closed over free variable @@ -695,6 +683,7 @@ class CyImport(CythonCommand): command_class = gdb.COMMAND_STATUS completer_class = gdb.COMPLETE_FILENAME + @libpython.dont_suppress_errors def invoke(self, args, from_tty): if isinstance(args, BYTES): args = args.decode(_filesystemencoding) @@ -742,11 +731,12 @@ class CyImport(CythonCommand): funcarg.tag for funcarg in function.find('Arguments')) for marker in module.find('LineNumberMapping'): - cython_lineno = int(marker.attrib['cython_lineno']) + src_lineno = int(marker.attrib['src_lineno']) + src_path = marker.attrib['src_path'] c_linenos = list(map(int, marker.attrib['c_linenos'].split())) - cython_module.lineno_cy2c[cython_lineno] = min(c_linenos) + cython_module.lineno_cy2c[src_path, src_lineno] = min(c_linenos) for c_lineno in c_linenos: - cython_module.lineno_c2cy[c_lineno] = cython_lineno + cython_module.lineno_c2cy[c_lineno] = (src_path, src_lineno) class CyBreak(CythonCommand): @@ -784,8 +774,8 @@ class CyBreak(CythonCommand): else: cython_module = self.get_cython_function().module - if lineno in cython_module.lineno_cy2c: - c_lineno = cython_module.lineno_cy2c[lineno] + if (cython_module.filename, lineno) in cython_module.lineno_cy2c: + c_lineno = cython_module.lineno_cy2c[cython_module.filename, lineno] breakpoint = '%s:%s' % (cython_module.c_filename, c_lineno) gdb.execute('break ' + breakpoint) else: @@ -841,6 +831,7 @@ class CyBreak(CythonCommand): if func.pf_cname: gdb.execute('break %s' % func.pf_cname) + @libpython.dont_suppress_errors def invoke(self, function_names, from_tty): if isinstance(function_names, BYTES): function_names = function_names.decode(_filesystemencoding) @@ -859,7 +850,7 @@ class CyBreak(CythonCommand): else: self._break_funcname(funcname) - @dont_suppress_errors + @libpython.dont_suppress_errors def complete(self, text, word): # Filter init-module functions (breakpoints can be set using # modulename:linenumber). @@ -904,7 +895,7 @@ class CythonInfo(CythonBase, libpython.PythonInfo): # stepping through Python code, but it would not step back into Cython- # related code. The C level should be dispatched to the 'step' command. if self.is_cython_function(frame): - return self.get_cython_lineno(frame) + return self.get_cython_lineno(frame)[1] return super(CythonInfo, self).lineno(frame) def get_source_line(self, frame): @@ -944,6 +935,7 @@ class CyStep(CythonExecutionControlCommand, libpython.PythonStepperMixin): name = 'cy -step' stepinto = True + @libpython.dont_suppress_errors def invoke(self, args, from_tty): if self.is_python_function(): self.python_step(self.stepinto) @@ -973,7 +965,7 @@ class CyRun(CythonExecutionControlCommand): name = 'cy run' - invoke = CythonExecutionControlCommand.run + invoke = libpython.dont_suppress_errors(CythonExecutionControlCommand.run) class CyCont(CythonExecutionControlCommand): @@ -983,7 +975,7 @@ class CyCont(CythonExecutionControlCommand): """ name = 'cy cont' - invoke = CythonExecutionControlCommand.cont + invoke = libpython.dont_suppress_errors(CythonExecutionControlCommand.cont) class CyFinish(CythonExecutionControlCommand): @@ -992,7 +984,7 @@ class CyFinish(CythonExecutionControlCommand): """ name = 'cy finish' - invoke = CythonExecutionControlCommand.finish + invoke = libpython.dont_suppress_errors(CythonExecutionControlCommand.finish) class CyUp(CythonCommand): @@ -1002,6 +994,7 @@ class CyUp(CythonCommand): name = 'cy up' _command = 'up' + @libpython.dont_suppress_errors def invoke(self, *args): try: gdb.execute(self._command, to_string=True) @@ -1036,6 +1029,7 @@ class CySelect(CythonCommand): name = 'cy select' + @libpython.dont_suppress_errors def invoke(self, stackno, from_tty): try: stackno = int(stackno) @@ -1062,6 +1056,7 @@ class CyBacktrace(CythonCommand): command_class = gdb.COMMAND_STACK completer_class = gdb.COMPLETE_NONE + @libpython.dont_suppress_errors @require_running_program def invoke(self, args, from_tty): # get the first frame @@ -1095,6 +1090,7 @@ class CyList(CythonCommand): command_class = gdb.COMMAND_FILES completer_class = gdb.COMPLETE_NONE + @libpython.dont_suppress_errors # @dispatch_on_frame(c_command='list') def invoke(self, _, from_tty): sd, lineno = self.get_source_desc() @@ -1111,8 +1107,28 @@ class CyPrint(CythonCommand): name = 'cy print' command_class = gdb.COMMAND_DATA - def invoke(self, name, from_tty, max_name_length=None): - if self.is_python_function(): + @libpython.dont_suppress_errors + def invoke(self, name, from_tty): + global_python_dict = self.get_cython_globals_dict() + module_globals = self.get_cython_function().module.globals + + if name in global_python_dict: + value = global_python_dict[name].get_truncated_repr(libpython.MAX_OUTPUT_LEN) + print('%s = %s' % (name, value)) + #This also would work, but beacause the output of cy exec is not captured in gdb.execute, TestPrint would fail + #self.cy.exec_.invoke("print('"+name+"','=', type(" + name + "), "+name+", flush=True )", from_tty) + elif name in module_globals: + cname = module_globals[name].cname + try: + value = gdb.parse_and_eval(cname) + except RuntimeError: + print("unable to get value of %s" % name) + else: + if not value.is_optimized_out: + self.print_gdb_value(name, value) + else: + print("%s is optimized out" % name) + elif self.is_python_function(): return gdb.execute('py-print ' + name) elif self.is_cython_function(): value = self.cy.cy_cvalue.invoke(name.lstrip('*')) @@ -1122,7 +1138,7 @@ class CyPrint(CythonCommand): else: break - self.print_gdb_value(name, value, max_name_length) + self.print_gdb_value(name, value) else: gdb.execute('print ' + name) @@ -1146,6 +1162,7 @@ class CyLocals(CythonCommand): command_class = gdb.COMMAND_STACK completer_class = gdb.COMPLETE_NONE + @libpython.dont_suppress_errors @dispatch_on_frame(c_command='info locals', python_command='py-locals') def invoke(self, args, from_tty): cython_function = self.get_cython_function() @@ -1173,6 +1190,7 @@ class CyGlobals(CyLocals): command_class = gdb.COMMAND_STACK completer_class = gdb.COMPLETE_NONE + @libpython.dont_suppress_errors @dispatch_on_frame(c_command='info variables', python_command='py-globals') def invoke(self, args, from_tty): global_python_dict = self.get_cython_globals_dict() @@ -1189,6 +1207,7 @@ class CyGlobals(CyLocals): seen = set() print('Python globals:') + for k, v in sorted(global_python_dict.items(), key=sortkey): v = v.get_truncated_repr(libpython.MAX_OUTPUT_LEN) seen.add(k) @@ -1219,7 +1238,9 @@ class EvaluateOrExecuteCodeMixin(object): cython_func = self.get_cython_function() for name, cyvar in cython_func.locals.items(): - if cyvar.type == PythonObject and self.is_initialized(cython_func, name): + if (cyvar.type == PythonObject + and self.is_initialized(cython_func, name)): + try: val = gdb.parse_and_eval(cyvar.cname) except RuntimeError: @@ -1247,8 +1268,8 @@ class EvaluateOrExecuteCodeMixin(object): def _find_first_cython_or_python_frame(self): frame = gdb.selected_frame() while frame: - if (self.is_cython_function(frame) or - self.is_python_function(frame)): + if (self.is_cython_function(frame) + or self.is_python_function(frame)): frame.select() return frame @@ -1295,10 +1316,11 @@ class CyExec(CythonCommand, libpython.PyExec, EvaluateOrExecuteCodeMixin): command_class = gdb.COMMAND_STACK completer_class = gdb.COMPLETE_NONE + @libpython.dont_suppress_errors def invoke(self, expr, from_tty): expr, input_type = self.readcode(expr) executor = libpython.PythonCodeExecutor() - executor.xdecref(self.evalcode(expr, executor.Py_single_input)) + executor.xdecref(self.evalcode(expr, executor.Py_file_input)) class CySet(CythonCommand): @@ -1317,6 +1339,7 @@ class CySet(CythonCommand): command_class = gdb.COMMAND_DATA completer_class = gdb.COMPLETE_NONE + @libpython.dont_suppress_errors @require_cython_frame def invoke(self, expr, from_tty): name_and_expr = expr.split('=', 1) @@ -1340,6 +1363,7 @@ class CyCName(gdb.Function, CythonBase): print $cy_cname("module.function") """ + @libpython.dont_suppress_errors @require_cython_frame @gdb_function_value_to_unicode def invoke(self, cyname, frame=None): @@ -1371,6 +1395,7 @@ class CyCValue(CyCName): Get the value of a Cython variable. """ + @libpython.dont_suppress_errors @require_cython_frame @gdb_function_value_to_unicode def invoke(self, cyname, frame=None): @@ -1391,9 +1416,10 @@ class CyLine(gdb.Function, CythonBase): Get the current Cython line. """ + @libpython.dont_suppress_errors @require_cython_frame def invoke(self): - return self.get_cython_lineno() + return self.get_cython_lineno()[1] class CyEval(gdb.Function, CythonBase, EvaluateOrExecuteCodeMixin): @@ -1401,6 +1427,7 @@ class CyEval(gdb.Function, CythonBase, EvaluateOrExecuteCodeMixin): Evaluate Python code in the nearest Python or Cython frame and return """ + @libpython.dont_suppress_errors @gdb_function_value_to_unicode def invoke(self, python_expression): input_type = libpython.PythonCodeExecutor.Py_eval_input diff --git a/Cython/Debugger/libpython.py b/Cython/Debugger/libpython.py index fea626dd7..30713e0d2 100644 --- a/Cython/Debugger/libpython.py +++ b/Cython/Debugger/libpython.py @@ -1,9 +1,10 @@ #!/usr/bin/python -# NOTE: this file is taken from the Python source distribution +# NOTE: Most of this file is taken from the Python source distribution # It can be found under Tools/gdb/libpython.py. It is shipped with Cython # because it's not installed as a python module, and because changes are only # merged into new python versions (v3.2+). +# We added some of our code below the "## added, not in CPython" comment. ''' From gdb 7 onwards, gdb's build can be configured --with-python, allowing gdb @@ -105,6 +106,8 @@ hexdigits = "0123456789abcdef" ENCODING = locale.getpreferredencoding() +FRAME_INFO_OPTIMIZED_OUT = '(frame information optimized out)' +UNABLE_READ_INFO_PYTHON_FRAME = 'Unable to read information on python frame' EVALFRAME = '_PyEval_EvalFrameDefault' class NullPyObjectPtr(RuntimeError): @@ -276,12 +279,13 @@ class PyObjectPtr(object): def safe_tp_name(self): try: - return self.type().field('tp_name').string() - except NullPyObjectPtr: - # NULL tp_name? - return 'unknown' - except RuntimeError: - # Can't even read the object at all? + ob_type = self.type() + tp_name = ob_type.field('tp_name') + return tp_name.string() + # NullPyObjectPtr: NULL tp_name? + # RuntimeError: Can't even read the object at all? + # UnicodeDecodeError: Failed to decode tp_name bytestring + except (NullPyObjectPtr, RuntimeError, UnicodeDecodeError): return 'unknown' def proxyval(self, visited): @@ -355,7 +359,9 @@ class PyObjectPtr(object): try: tp_name = t.field('tp_name').string() tp_flags = int(t.field('tp_flags')) - except RuntimeError: + # RuntimeError: NULL pointers + # UnicodeDecodeError: string() fails to decode the bytestring + except (RuntimeError, UnicodeDecodeError): # Handle any kind of error e.g. NULL ptrs by simply using the base # class return cls @@ -622,8 +628,11 @@ class PyCFunctionObjectPtr(PyObjectPtr): _typename = 'PyCFunctionObject' def proxyval(self, visited): - m_ml = self.field('m_ml') # m_ml is a (PyMethodDef*) - ml_name = m_ml['ml_name'].string() + m_ml = self.field('m_ml') # m_ml is a (PyMethodDef*) + try: + ml_name = m_ml['ml_name'].string() + except UnicodeDecodeError: + ml_name = '<ml_name:UnicodeDecodeError>' pyop_m_self = self.pyop_field('m_self') if pyop_m_self.is_null(): @@ -736,7 +745,7 @@ class PyDictObjectPtr(PyObjectPtr): else: offset = 8 * dk_size - ent_addr = keys['dk_indices']['as_1'].address + ent_addr = keys['dk_indices'].address ent_addr = ent_addr.cast(_type_unsigned_char_ptr()) + offset ent_ptr_t = gdb.lookup_type('PyDictKeyEntry').pointer() ent_addr = ent_addr.cast(ent_ptr_t) @@ -918,7 +927,7 @@ class PyFrameObjectPtr(PyObjectPtr): def filename(self): '''Get the path of the current Python source file, as a string''' if self.is_optimized_out(): - return '(frame information optimized out)' + return FRAME_INFO_OPTIMIZED_OUT return self.co_filename.proxyval(set()) def current_line_num(self): @@ -934,35 +943,50 @@ class PyFrameObjectPtr(PyObjectPtr): if long(f_trace) != 0: # we have a non-NULL f_trace: return self.f_lineno - else: - #try: + + try: return self.co.addr2line(self.f_lasti) - #except ValueError: - # return self.f_lineno + except Exception: + # bpo-34989: addr2line() is a complex function, it can fail in many + # ways. For example, it fails with a TypeError on "FakeRepr" if + # gdb fails to load debug symbols. Use a catch-all "except + # Exception" to make the whole function safe. The caller has to + # handle None anyway for optimized Python. + return None def current_line(self): '''Get the text of the current source line as a string, with a trailing newline character''' if self.is_optimized_out(): - return '(frame information optimized out)' + return FRAME_INFO_OPTIMIZED_OUT + + lineno = self.current_line_num() + if lineno is None: + return '(failed to get frame line number)' + filename = self.filename() try: - f = open(os_fsencode(filename), 'r') + with open(os_fsencode(filename), 'r') as fp: + lines = fp.readlines() except IOError: return None - with f: - all_lines = f.readlines() - # Convert from 1-based current_line_num to 0-based list offset: - return all_lines[self.current_line_num()-1] + + try: + # Convert from 1-based current_line_num to 0-based list offset + return lines[lineno - 1] + except IndexError: + return None def write_repr(self, out, visited): if self.is_optimized_out(): - out.write('(frame information optimized out)') + out.write(FRAME_INFO_OPTIMIZED_OUT) return - out.write('Frame 0x%x, for file %s, line %i, in %s (' + lineno = self.current_line_num() + lineno = str(lineno) if lineno is not None else "?" + out.write('Frame 0x%x, for file %s, line %s, in %s (' % (self.as_address(), self.co_filename.proxyval(visited), - self.current_line_num(), + lineno, self.co_name.proxyval(visited))) first = True for pyop_name, pyop_value in self.iter_locals(): @@ -978,12 +1002,14 @@ class PyFrameObjectPtr(PyObjectPtr): def print_traceback(self): if self.is_optimized_out(): - sys.stdout.write(' (frame information optimized out)\n') + sys.stdout.write(' %s\n' % FRAME_INFO_OPTIMIZED_OUT) return visited = set() - sys.stdout.write(' File "%s", line %i, in %s\n' + lineno = self.current_line_num() + lineno = str(lineno) if lineno is not None else "?" + sys.stdout.write(' File "%s", line %s, in %s\n' % (self.co_filename.proxyval(visited), - self.current_line_num(), + lineno, self.co_name.proxyval(visited))) class PySetObjectPtr(PyObjectPtr): @@ -1091,11 +1117,6 @@ class PyBytesObjectPtr(PyObjectPtr): out.write(byte) out.write(quote) - -class PyStringObjectPtr(PyBytesObjectPtr): - _typename = 'PyStringObject' - - class PyTupleObjectPtr(PyObjectPtr): _typename = 'PyTupleObject' @@ -1166,7 +1187,7 @@ class PyUnicodeObjectPtr(PyObjectPtr): def proxyval(self, visited): global _is_pep393 if _is_pep393 is None: - fields = gdb.lookup_type('PyUnicodeObject').target().fields() + fields = gdb.lookup_type('PyUnicodeObject').fields() _is_pep393 = 'data' in [f.name for f in fields] if _is_pep393: # Python 3.3 and newer @@ -1285,8 +1306,8 @@ class PyUnicodeObjectPtr(PyObjectPtr): # If sizeof(Py_UNICODE) is 2 here (in gdb), join # surrogate pairs before calling _unichr_is_printable. if (i < len(proxy) - and 0xD800 <= ord(ch) < 0xDC00 \ - and 0xDC00 <= ord(proxy[i]) <= 0xDFFF): + and 0xD800 <= ord(ch) < 0xDC00 + and 0xDC00 <= ord(proxy[i]) <= 0xDFFF): ch2 = proxy[i] ucs = ch + ch2 i += 1 @@ -1351,13 +1372,13 @@ class wrapperobject(PyObjectPtr): try: name = self.field('descr')['d_base']['name'].string() return repr(name) - except (NullPyObjectPtr, RuntimeError): + except (NullPyObjectPtr, RuntimeError, UnicodeDecodeError): return '<unknown name>' def safe_tp_name(self): try: return self.field('self')['ob_type']['tp_name'].string() - except (NullPyObjectPtr, RuntimeError): + except (NullPyObjectPtr, RuntimeError, UnicodeDecodeError): return '<unknown tp_name>' def safe_self_addresss(self): @@ -1380,7 +1401,7 @@ class wrapperobject(PyObjectPtr): def int_from_int(gdbval): - return int(str(gdbval)) + return int(gdbval) def stringify(val): @@ -1551,8 +1572,8 @@ class Frame(object): if not caller: return False - if caller in ('_PyCFunction_FastCallDict', - '_PyCFunction_FastCallKeywords'): + if (caller.startswith('cfunction_vectorcall_') or + caller == 'cfunction_call'): arg_name = 'func' # Within that frame: # "func" is the local containing the PyObject* of the @@ -1563,15 +1584,22 @@ class Frame(object): # Use the prettyprinter for the func: func = frame.read_var(arg_name) return str(func) + except ValueError: + return ('PyCFunction invocation (unable to read %s: ' + 'missing debuginfos?)' % arg_name) except RuntimeError: return 'PyCFunction invocation (unable to read %s)' % arg_name if caller == 'wrapper_call': + arg_name = 'wp' try: - func = frame.read_var('wp') + func = frame.read_var(arg_name) return str(func) + except ValueError: + return ('<wrapper_call invocation (unable to read %s: ' + 'missing debuginfos?)>' % arg_name) except RuntimeError: - return '<wrapper_call invocation>' + return '<wrapper_call invocation (unable to read %s)>' % arg_name # This frame isn't worth reporting: return False @@ -1581,7 +1609,7 @@ class Frame(object): # This assumes the _POSIX_THREADS version of Python/ceval_gil.h: name = self._gdbframe.name() if name: - return 'pthread_cond_timedwait' in name + return (name == 'take_gil') def is_gc_collect(self): '''Is this frame "collect" within the garbage-collector?''' @@ -1725,11 +1753,14 @@ class PyList(gdb.Command): pyop = frame.get_pyop() if not pyop or pyop.is_optimized_out(): - print('Unable to read information on python frame') + print(UNABLE_READ_INFO_PYTHON_FRAME) return filename = pyop.filename() lineno = pyop.current_line_num() + if lineno is None: + print('Unable to read python frame line number') + return if start is None: start = lineno - 5 @@ -1882,7 +1913,7 @@ class PyPrint(gdb.Command): pyop_frame = frame.get_pyop() if not pyop_frame: - print('Unable to read information on python frame') + print(UNABLE_READ_INFO_PYTHON_FRAME) return pyop_var, scope = pyop_frame.get_var_by_name(name) @@ -1899,9 +1930,9 @@ PyPrint() class PyLocals(gdb.Command): 'Look up the given python variable name, and print it' - def __init__(self, command="py-locals"): + def __init__(self): gdb.Command.__init__ (self, - command, + "py-locals", gdb.COMMAND_DATA, gdb.COMPLETE_NONE) @@ -1916,22 +1947,14 @@ class PyLocals(gdb.Command): pyop_frame = frame.get_pyop() if not pyop_frame: - print('Unable to read information on python frame') + print(UNABLE_READ_INFO_PYTHON_FRAME) return - namespace = self.get_namespace(pyop_frame) - namespace = [(name.proxyval(set()), val) for name, val in namespace] - - if namespace: - name, val = max(namespace, key=lambda item: len(item[0])) - max_name_length = len(name) - - for name, pyop_value in namespace: - value = pyop_value.get_truncated_repr(MAX_OUTPUT_LEN) - print('%-*s = %s' % (max_name_length, name, value)) - - def get_namespace(self, pyop_frame): - return pyop_frame.iter_locals() + for pyop_name, pyop_value in pyop_frame.iter_locals(): + print('%s = %s' % ( + pyop_name.proxyval(set()), + pyop_value.get_truncated_repr(MAX_OUTPUT_LEN), + )) PyLocals() @@ -1943,24 +1966,80 @@ PyLocals() import re import warnings import tempfile +import functools import textwrap import itertools +import traceback -class PyGlobals(PyLocals): + +def dont_suppress_errors(function): + "*sigh*, readline" + @functools.wraps(function) + def wrapper(*args, **kwargs): + try: + return function(*args, **kwargs) + except Exception: + traceback.print_exc() + raise + + return wrapper + +class PyGlobals(gdb.Command): 'List all the globals in the currently select Python frame' + def __init__(self): + gdb.Command.__init__ (self, + "py-globals", + gdb.COMMAND_DATA, + gdb.COMPLETE_NONE) + + @dont_suppress_errors + def invoke(self, args, from_tty): + name = str(args) + + frame = Frame.get_selected_python_frame() + if not frame: + print('Unable to locate python frame') + return + + pyop_frame = frame.get_pyop() + if not pyop_frame: + print(UNABLE_READ_INFO_PYTHON_FRAME) + return + + for pyop_name, pyop_value in pyop_frame.iter_locals(): + print('%s = %s' + % (pyop_name.proxyval(set()), + pyop_value.get_truncated_repr(MAX_OUTPUT_LEN))) def get_namespace(self, pyop_frame): return pyop_frame.iter_globals() -PyGlobals("py-globals") +PyGlobals() +# This function used to be a part of CPython's libpython.py (as a member function of frame). +# It isn't anymore, so I copied it. +def is_evalframeex(frame): + '''Is this a PyEval_EvalFrameEx frame?''' + if frame._gdbframe.name() == 'PyEval_EvalFrameEx': + ''' + I believe we also need to filter on the inline + struct frame_id.inline_depth, only regarding frames with + an inline depth of 0 as actually being this function + + So we reject those with type gdb.INLINE_FRAME + ''' + if frame._gdbframe.type() == gdb.NORMAL_FRAME: + # We have a PyEval_EvalFrameEx frame: + return True + + return False class PyNameEquals(gdb.Function): def _get_pycurframe_attr(self, attr): frame = Frame(gdb.selected_frame()) - if frame.is_evalframeex(): + if is_evalframeex(frame): pyframe = frame.get_pyop() if pyframe is None: warnings.warn("Use a Python debug build, Python breakpoints " @@ -1971,6 +2050,7 @@ class PyNameEquals(gdb.Function): return None + @dont_suppress_errors def invoke(self, funcname): attr = self._get_pycurframe_attr('co_name') return attr is not None and attr == funcname.string() @@ -1980,6 +2060,7 @@ PyNameEquals("pyname_equals") class PyModEquals(PyNameEquals): + @dont_suppress_errors def invoke(self, modname): attr = self._get_pycurframe_attr('co_filename') if attr is not None: @@ -2003,6 +2084,7 @@ class PyBreak(gdb.Command): py-break func """ + @dont_suppress_errors def invoke(self, funcname, from_tty): if '.' in funcname: modname, dot, funcname = funcname.rpartition('.') @@ -2457,6 +2539,7 @@ class PyStep(ExecutionControlCommandBase, PythonStepperMixin): stepinto = True + @dont_suppress_errors def invoke(self, args, from_tty): self.python_step(stepinto=self.stepinto) @@ -2470,18 +2553,18 @@ class PyNext(PyStep): class PyFinish(ExecutionControlCommandBase): "Execute until function returns to a caller." - invoke = ExecutionControlCommandBase.finish + invoke = dont_suppress_errors(ExecutionControlCommandBase.finish) class PyRun(ExecutionControlCommandBase): "Run the program." - invoke = ExecutionControlCommandBase.run + invoke = dont_suppress_errors(ExecutionControlCommandBase.run) class PyCont(ExecutionControlCommandBase): - invoke = ExecutionControlCommandBase.cont + invoke = dont_suppress_errors(ExecutionControlCommandBase.cont) def _pointervalue(gdbval): @@ -2574,7 +2657,7 @@ class PythonCodeExecutor(object): return pointer def free(self, pointer): - gdb.parse_and_eval("free((void *) %d)" % pointer) + gdb.parse_and_eval("(void) free((void *) %d)" % pointer) def incref(self, pointer): "Increment the reference count of a Python object in the inferior." @@ -2693,6 +2776,7 @@ class FixGdbCommand(gdb.Command): pass # warnings.resetwarnings() + @dont_suppress_errors def invoke(self, args, from_tty): self.fix_gdb() try: @@ -2726,7 +2810,10 @@ class PyExec(gdb.Command): lines = [] while True: try: - line = input('>') + if sys.version_info[0] == 2: + line = raw_input() + else: + line = input('>') except EOFError: break else: @@ -2737,6 +2824,7 @@ class PyExec(gdb.Command): return '\n'.join(lines), PythonCodeExecutor.Py_file_input + @dont_suppress_errors def invoke(self, expr, from_tty): expr, input_type = self.readcode(expr) executor = PythonCodeExecutor() diff --git a/Cython/Distutils/build_ext.py b/Cython/Distutils/build_ext.py index 598bb4a89..0ab979b83 100644 --- a/Cython/Distutils/build_ext.py +++ b/Cython/Distutils/build_ext.py @@ -1,25 +1,130 @@ import sys +import os -if 'setuptools' in sys.modules: - try: - from setuptools.command.build_ext import build_ext as _build_ext - except ImportError: - # We may be in the process of importing setuptools, which tries - # to import this. - from distutils.command.build_ext import build_ext as _build_ext -else: +try: + from __builtin__ import basestring +except ImportError: + basestring = str + +# Always inherit from the "build_ext" in distutils since setuptools already imports +# it from Cython if available, and does the proper distutils fallback otherwise. +# https://github.com/pypa/setuptools/blob/9f1822ee910df3df930a98ab99f66d18bb70659b/setuptools/command/build_ext.py#L16 + +# setuptools imports Cython's "build_ext", so make sure we go first. +_build_ext_module = sys.modules.get('setuptools.command.build_ext') +if _build_ext_module is None: + import distutils.command.build_ext as _build_ext_module + +# setuptools remembers the original distutils "build_ext" as "_du_build_ext" +_build_ext = getattr(_build_ext_module, '_du_build_ext', None) +if _build_ext is None: + _build_ext = getattr(_build_ext_module, 'build_ext', None) +if _build_ext is None: from distutils.command.build_ext import build_ext as _build_ext -class new_build_ext(_build_ext, object): +class build_ext(_build_ext, object): + + user_options = _build_ext.user_options + [ + ('cython-cplus', None, + "generate C++ source files"), + ('cython-create-listing', None, + "write errors to a listing file"), + ('cython-line-directives', None, + "emit source line directives"), + ('cython-include-dirs=', None, + "path to the Cython include files" + _build_ext.sep_by), + ('cython-c-in-temp', None, + "put generated C files in temp directory"), + ('cython-gen-pxi', None, + "generate .pxi file for public declarations"), + ('cython-directives=', None, + "compiler directive overrides"), + ('cython-gdb', None, + "generate debug information for cygdb"), + ('cython-compile-time-env', None, + "cython compile time environment"), + ] + + boolean_options = _build_ext.boolean_options + [ + 'cython-cplus', 'cython-create-listing', 'cython-line-directives', + 'cython-c-in-temp', 'cython-gdb', + ] + + def initialize_options(self): + super(build_ext, self).initialize_options() + self.cython_cplus = 0 + self.cython_create_listing = 0 + self.cython_line_directives = 0 + self.cython_include_dirs = None + self.cython_directives = None + self.cython_c_in_temp = 0 + self.cython_gen_pxi = 0 + self.cython_gdb = False + self.cython_compile_time_env = None + def finalize_options(self): - if self.distribution.ext_modules: - nthreads = getattr(self, 'parallel', None) # -j option in Py3.5+ - nthreads = int(nthreads) if nthreads else None - from Cython.Build.Dependencies import cythonize - self.distribution.ext_modules[:] = cythonize( - self.distribution.ext_modules, nthreads=nthreads, force=self.force) - super(new_build_ext, self).finalize_options() - -# This will become new_build_ext in the future. -from .old_build_ext import old_build_ext as build_ext + super(build_ext, self).finalize_options() + if self.cython_include_dirs is None: + self.cython_include_dirs = [] + elif isinstance(self.cython_include_dirs, basestring): + self.cython_include_dirs = \ + self.cython_include_dirs.split(os.pathsep) + if self.cython_directives is None: + self.cython_directives = {} + + def get_extension_attr(self, extension, option_name, default=False): + return getattr(self, option_name) or getattr(extension, option_name, default) + + def build_extension(self, ext): + from Cython.Build.Dependencies import cythonize + + # Set up the include_path for the Cython compiler: + # 1. Start with the command line option. + # 2. Add in any (unique) paths from the extension + # cython_include_dirs (if Cython.Distutils.extension is used). + # 3. Add in any (unique) paths from the extension include_dirs + includes = list(self.cython_include_dirs) + for include_dir in getattr(ext, 'cython_include_dirs', []): + if include_dir not in includes: + includes.append(include_dir) + + # In case extension.include_dirs is a generator, evaluate it and keep + # result + ext.include_dirs = list(ext.include_dirs) + for include_dir in ext.include_dirs + list(self.include_dirs): + if include_dir not in includes: + includes.append(include_dir) + + # Set up Cython compiler directives: + # 1. Start with the command line option. + # 2. Add in any (unique) entries from the extension + # cython_directives (if Cython.Distutils.extension is used). + directives = dict(self.cython_directives) + if hasattr(ext, "cython_directives"): + directives.update(ext.cython_directives) + + if self.get_extension_attr(ext, 'cython_cplus'): + ext.language = 'c++' + + options = { + 'use_listing_file': self.get_extension_attr(ext, 'cython_create_listing'), + 'emit_linenums': self.get_extension_attr(ext, 'cython_line_directives'), + 'include_path': includes, + 'compiler_directives': directives, + 'build_dir': self.build_temp if self.get_extension_attr(ext, 'cython_c_in_temp') else None, + 'generate_pxi': self.get_extension_attr(ext, 'cython_gen_pxi'), + 'gdb_debug': self.get_extension_attr(ext, 'cython_gdb'), + 'c_line_in_traceback': not getattr(ext, 'no_c_in_traceback', 0), + 'compile_time_env': self.get_extension_attr(ext, 'cython_compile_time_env', default=None), + } + + new_ext = cythonize( + ext,force=self.force, quiet=self.verbose == 0, **options + )[0] + + ext.sources = new_ext.sources + super(build_ext, self).build_extension(ext) + +# backward compatibility +new_build_ext = build_ext diff --git a/Cython/Distutils/extension.py b/Cython/Distutils/extension.py index d8bdbf0f5..93744ec45 100644 --- a/Cython/Distutils/extension.py +++ b/Cython/Distutils/extension.py @@ -8,11 +8,6 @@ __revision__ = "$Id:$" import sys import distutils.extension as _Extension -try: - import warnings -except ImportError: - warnings = None - class Extension(_Extension.Extension): # When adding arguments to this constructor, be sure to update diff --git a/Cython/Distutils/old_build_ext.py b/Cython/Distutils/old_build_ext.py index aa2a1cf22..cec54d93d 100644 --- a/Cython/Distutils/old_build_ext.py +++ b/Cython/Distutils/old_build_ext.py @@ -8,7 +8,6 @@ Note that this module is deprecated. Use cythonize() instead. __revision__ = "$Id:$" -import inspect import sys import os from distutils.errors import DistutilsPlatformError @@ -16,7 +15,6 @@ from distutils.dep_util import newer, newer_group from distutils import log from distutils.command import build_ext as _build_ext from distutils import sysconfig -import warnings try: @@ -25,22 +23,30 @@ except ImportError: basestring = str +# FIXME: the below does not work as intended since importing 'Cython.Distutils' already +# imports this module through 'Cython/Distutils/build_ext.py', so the condition is +# always false and never prints the warning. +""" +import inspect +import warnings + def _check_stack(path): - try: - for frame in inspect.getouterframes(inspect.currentframe(), 0): - if path in frame[1].replace(os.sep, '/'): - return True - except Exception: - pass - return False + try: + for frame in inspect.getouterframes(inspect.currentframe(), 0): + if path in frame[1].replace(os.sep, '/'): + return True + except Exception: + pass + return False + if (not _check_stack('setuptools/extensions.py') - and not _check_stack('pyximport/pyxbuild.py') - and not _check_stack('Cython/Distutils/build_ext.py')): + and not _check_stack('pyximport/pyxbuild.py') + and not _check_stack('Cython/Distutils/build_ext.py')): warnings.warn( "Cython.Distutils.old_build_ext does not properly handle dependencies " "and is deprecated.") - +""" extension_name_re = _build_ext.extension_name_re @@ -163,7 +169,7 @@ class old_build_ext(_build_ext.build_ext): # _build_ext.build_ext.__setattr__(self, name, value) self.__dict__[name] = value - def finalize_options (self): + def finalize_options(self): _build_ext.build_ext.finalize_options(self) if self.cython_include_dirs is None: self.cython_include_dirs = [] @@ -185,14 +191,11 @@ class old_build_ext(_build_ext.build_ext): _build_ext.build_ext.run(self) - def build_extensions(self): - # First, sanity-check the 'extensions' list - self.check_extensions_list(self.extensions) - + def check_extensions_list(self, extensions): + # Note: might get called multiple times. + _build_ext.build_ext.check_extensions_list(self, extensions) for ext in self.extensions: ext.sources = self.cython_sources(ext.sources, ext) - # Call original build_extensions - _build_ext.build_ext.build_extensions(self) def cython_sources(self, sources, extension): """ @@ -201,17 +204,6 @@ class old_build_ext(_build_ext.build_ext): found, and return a modified 'sources' list with Cython source files replaced by the generated C (or C++) files. """ - try: - from Cython.Compiler.Main \ - import CompilationOptions, \ - default_options as cython_default_options, \ - compile as cython_compile - from Cython.Compiler.Errors import PyrexError - except ImportError: - e = sys.exc_info()[1] - print("failed to import Cython: %s" % e) - raise DistutilsPlatformError("Cython does not appear to be installed") - new_sources = [] cython_sources = [] cython_targets = {} @@ -252,10 +244,10 @@ class old_build_ext(_build_ext.build_ext): # 2. Add in any (unique) paths from the extension # cython_include_dirs (if Cython.Distutils.extension is used). # 3. Add in any (unique) paths from the extension include_dirs - includes = self.cython_include_dirs + includes = list(self.cython_include_dirs) try: for i in extension.cython_include_dirs: - if not i in includes: + if i not in includes: includes.append(i) except AttributeError: pass @@ -264,19 +256,18 @@ class old_build_ext(_build_ext.build_ext): # result extension.include_dirs = list(extension.include_dirs) for i in extension.include_dirs: - if not i in includes: + if i not in includes: includes.append(i) # Set up Cython compiler directives: # 1. Start with the command line option. # 2. Add in any (unique) entries from the extension # cython_directives (if Cython.Distutils.extension is used). - directives = self.cython_directives + directives = dict(self.cython_directives) if hasattr(extension, "cython_directives"): directives.update(extension.cython_directives) - # Set the target_ext to '.c'. Cython will change this to '.cpp' if - # needed. + # Set the target file extension for C/C++ mode. if cplus: target_ext = '.cpp' else: @@ -314,13 +305,24 @@ class old_build_ext(_build_ext.build_ext): if not cython_sources: return new_sources + try: + from Cython.Compiler.Main \ + import CompilationOptions, \ + default_options as cython_default_options, \ + compile as cython_compile + from Cython.Compiler.Errors import PyrexError + except ImportError: + e = sys.exc_info()[1] + print("failed to import Cython: %s" % e) + raise DistutilsPlatformError("Cython does not appear to be installed") + module_name = extension.name for source in cython_sources: target = cython_targets[source] depends = [source] + list(extension.depends or ()) - if(source[-4:].lower()==".pyx" and os.path.isfile(source[:-3]+"pxd")): - depends += [source[:-3]+"pxd"] + if source[-4:].lower() == ".pyx" and os.path.isfile(source[:-3] + "pxd"): + depends += [source[:-3] + "pxd"] rebuild = self.force or newer_group(depends, target, 'newer') if not rebuild and newest_dependency is not None: rebuild = newer(newest_dependency, target) diff --git a/Cython/Includes/Deprecated/python.pxd b/Cython/Includes/Deprecated/python.pxd deleted file mode 100644 index 56236e925..000000000 --- a/Cython/Includes/Deprecated/python.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython cimport * diff --git a/Cython/Includes/Deprecated/python_bool.pxd b/Cython/Includes/Deprecated/python_bool.pxd deleted file mode 100644 index 9a6d253f4..000000000 --- a/Cython/Includes/Deprecated/python_bool.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.bool cimport * diff --git a/Cython/Includes/Deprecated/python_buffer.pxd b/Cython/Includes/Deprecated/python_buffer.pxd deleted file mode 100644 index 2baeaae00..000000000 --- a/Cython/Includes/Deprecated/python_buffer.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.buffer cimport * diff --git a/Cython/Includes/Deprecated/python_bytes.pxd b/Cython/Includes/Deprecated/python_bytes.pxd deleted file mode 100644 index 87af662de..000000000 --- a/Cython/Includes/Deprecated/python_bytes.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.bytes cimport * diff --git a/Cython/Includes/Deprecated/python_cobject.pxd b/Cython/Includes/Deprecated/python_cobject.pxd deleted file mode 100644 index ed32c6b87..000000000 --- a/Cython/Includes/Deprecated/python_cobject.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.cobject cimport * diff --git a/Cython/Includes/Deprecated/python_complex.pxd b/Cython/Includes/Deprecated/python_complex.pxd deleted file mode 100644 index 0a780b3b2..000000000 --- a/Cython/Includes/Deprecated/python_complex.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.complex cimport * diff --git a/Cython/Includes/Deprecated/python_dict.pxd b/Cython/Includes/Deprecated/python_dict.pxd deleted file mode 100644 index 05b5f4796..000000000 --- a/Cython/Includes/Deprecated/python_dict.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.dict cimport * diff --git a/Cython/Includes/Deprecated/python_exc.pxd b/Cython/Includes/Deprecated/python_exc.pxd deleted file mode 100644 index 6eb236bcc..000000000 --- a/Cython/Includes/Deprecated/python_exc.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.exc cimport * diff --git a/Cython/Includes/Deprecated/python_float.pxd b/Cython/Includes/Deprecated/python_float.pxd deleted file mode 100644 index 7e133ef9b..000000000 --- a/Cython/Includes/Deprecated/python_float.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.float cimport * diff --git a/Cython/Includes/Deprecated/python_function.pxd b/Cython/Includes/Deprecated/python_function.pxd deleted file mode 100644 index 1461c4e63..000000000 --- a/Cython/Includes/Deprecated/python_function.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.function cimport * diff --git a/Cython/Includes/Deprecated/python_getargs.pxd b/Cython/Includes/Deprecated/python_getargs.pxd deleted file mode 100644 index 3852d6a6a..000000000 --- a/Cython/Includes/Deprecated/python_getargs.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.getargs cimport * diff --git a/Cython/Includes/Deprecated/python_instance.pxd b/Cython/Includes/Deprecated/python_instance.pxd deleted file mode 100644 index 99cb5a909..000000000 --- a/Cython/Includes/Deprecated/python_instance.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.instance cimport * diff --git a/Cython/Includes/Deprecated/python_int.pxd b/Cython/Includes/Deprecated/python_int.pxd deleted file mode 100644 index c1fd5178d..000000000 --- a/Cython/Includes/Deprecated/python_int.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.int cimport * diff --git a/Cython/Includes/Deprecated/python_iterator.pxd b/Cython/Includes/Deprecated/python_iterator.pxd deleted file mode 100644 index e09aad279..000000000 --- a/Cython/Includes/Deprecated/python_iterator.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.iterator cimport * diff --git a/Cython/Includes/Deprecated/python_list.pxd b/Cython/Includes/Deprecated/python_list.pxd deleted file mode 100644 index 64febcf96..000000000 --- a/Cython/Includes/Deprecated/python_list.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.list cimport * diff --git a/Cython/Includes/Deprecated/python_long.pxd b/Cython/Includes/Deprecated/python_long.pxd deleted file mode 100644 index 1a24380c4..000000000 --- a/Cython/Includes/Deprecated/python_long.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.long cimport * diff --git a/Cython/Includes/Deprecated/python_mapping.pxd b/Cython/Includes/Deprecated/python_mapping.pxd deleted file mode 100644 index cd01bee01..000000000 --- a/Cython/Includes/Deprecated/python_mapping.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.mapping cimport * diff --git a/Cython/Includes/Deprecated/python_mem.pxd b/Cython/Includes/Deprecated/python_mem.pxd deleted file mode 100644 index d74429ea3..000000000 --- a/Cython/Includes/Deprecated/python_mem.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.mem cimport * diff --git a/Cython/Includes/Deprecated/python_method.pxd b/Cython/Includes/Deprecated/python_method.pxd deleted file mode 100644 index e7da5154e..000000000 --- a/Cython/Includes/Deprecated/python_method.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.method cimport * diff --git a/Cython/Includes/Deprecated/python_module.pxd b/Cython/Includes/Deprecated/python_module.pxd deleted file mode 100644 index 6310c0247..000000000 --- a/Cython/Includes/Deprecated/python_module.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.module cimport * diff --git a/Cython/Includes/Deprecated/python_number.pxd b/Cython/Includes/Deprecated/python_number.pxd deleted file mode 100644 index ae67da1c3..000000000 --- a/Cython/Includes/Deprecated/python_number.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.number cimport * diff --git a/Cython/Includes/Deprecated/python_object.pxd b/Cython/Includes/Deprecated/python_object.pxd deleted file mode 100644 index 3981bfa44..000000000 --- a/Cython/Includes/Deprecated/python_object.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.object cimport * diff --git a/Cython/Includes/Deprecated/python_oldbuffer.pxd b/Cython/Includes/Deprecated/python_oldbuffer.pxd deleted file mode 100644 index e03e66a2e..000000000 --- a/Cython/Includes/Deprecated/python_oldbuffer.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.oldbuffer cimport * diff --git a/Cython/Includes/Deprecated/python_pycapsule.pxd b/Cython/Includes/Deprecated/python_pycapsule.pxd deleted file mode 100644 index fe9cf8f8d..000000000 --- a/Cython/Includes/Deprecated/python_pycapsule.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.pycapsule cimport * diff --git a/Cython/Includes/Deprecated/python_ref.pxd b/Cython/Includes/Deprecated/python_ref.pxd deleted file mode 100644 index 944741819..000000000 --- a/Cython/Includes/Deprecated/python_ref.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.ref cimport * diff --git a/Cython/Includes/Deprecated/python_sequence.pxd b/Cython/Includes/Deprecated/python_sequence.pxd deleted file mode 100644 index fdef5b63e..000000000 --- a/Cython/Includes/Deprecated/python_sequence.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.sequence cimport * diff --git a/Cython/Includes/Deprecated/python_set.pxd b/Cython/Includes/Deprecated/python_set.pxd deleted file mode 100644 index a2feb9371..000000000 --- a/Cython/Includes/Deprecated/python_set.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.set cimport * diff --git a/Cython/Includes/Deprecated/python_string.pxd b/Cython/Includes/Deprecated/python_string.pxd deleted file mode 100644 index 24c818338..000000000 --- a/Cython/Includes/Deprecated/python_string.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.string cimport * diff --git a/Cython/Includes/Deprecated/python_tuple.pxd b/Cython/Includes/Deprecated/python_tuple.pxd deleted file mode 100644 index 190713b02..000000000 --- a/Cython/Includes/Deprecated/python_tuple.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.tuple cimport * diff --git a/Cython/Includes/Deprecated/python_type.pxd b/Cython/Includes/Deprecated/python_type.pxd deleted file mode 100644 index 3ac47d1b3..000000000 --- a/Cython/Includes/Deprecated/python_type.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.type cimport * diff --git a/Cython/Includes/Deprecated/python_unicode.pxd b/Cython/Includes/Deprecated/python_unicode.pxd deleted file mode 100644 index 2b488b2dc..000000000 --- a/Cython/Includes/Deprecated/python_unicode.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.unicode cimport * diff --git a/Cython/Includes/Deprecated/python_version.pxd b/Cython/Includes/Deprecated/python_version.pxd deleted file mode 100644 index c27ca4df9..000000000 --- a/Cython/Includes/Deprecated/python_version.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.version cimport * diff --git a/Cython/Includes/Deprecated/python_weakref.pxd b/Cython/Includes/Deprecated/python_weakref.pxd deleted file mode 100644 index 1f84f1a17..000000000 --- a/Cython/Includes/Deprecated/python_weakref.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from cpython.weakref cimport * diff --git a/Cython/Includes/Deprecated/stdio.pxd b/Cython/Includes/Deprecated/stdio.pxd deleted file mode 100644 index 41a4aebf1..000000000 --- a/Cython/Includes/Deprecated/stdio.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from libc.stdio cimport * diff --git a/Cython/Includes/Deprecated/stdlib.pxd b/Cython/Includes/Deprecated/stdlib.pxd deleted file mode 100644 index 499511cde..000000000 --- a/Cython/Includes/Deprecated/stdlib.pxd +++ /dev/null @@ -1,2 +0,0 @@ -# Present for backwards compatibility -from libc.stdlib cimport * diff --git a/Cython/Includes/Deprecated/stl.pxd b/Cython/Includes/Deprecated/stl.pxd deleted file mode 100644 index 22248d265..000000000 --- a/Cython/Includes/Deprecated/stl.pxd +++ /dev/null @@ -1,91 +0,0 @@ -cdef extern from "<vector>" namespace std: - - cdef cppclass vector[TYPE]: - #constructors - __init__() - __init__(vector&) - __init__(int) - __init__(int, TYPE&) - __init__(iterator, iterator) - #operators - TYPE& __getitem__(int) - TYPE& __setitem__(int, TYPE&) - vector __new__(vector&) - bool __eq__(vector&, vector&) - bool __ne__(vector&, vector&) - bool __lt__(vector&, vector&) - bool __gt__(vector&, vector&) - bool __le__(vector&, vector&) - bool __ge__(vector&, vector&) - #others - void assign(int, TYPE) - #void assign(iterator, iterator) - TYPE& at(int) - TYPE& back() - iterator begin() - int capacity() - void clear() - bool empty() - iterator end() - iterator erase(iterator) - iterator erase(iterator, iterator) - TYPE& front() - iterator insert(iterator, TYPE&) - void insert(iterator, int, TYPE&) - void insert(iterator, iterator) - int max_size() - void pop_back() - void push_back(TYPE&) - iterator rbegin() - iterator rend() - void reserve(int) - void resize(int) - void resize(int, TYPE&) #void resize(size_type num, const TYPE& = TYPE()) - int size() - void swap(container&) - -cdef extern from "<deque>" namespace std: - - cdef cppclass deque[TYPE]: - #constructors - __init__() - __init__(deque&) - __init__(int) - __init__(int, TYPE&) - __init__(iterator, iterator) - #operators - TYPE& operator[]( size_type index ); - const TYPE& operator[]( size_type index ) const; - deque __new__(deque&); - bool __eq__(deque&, deque&); - bool __ne__(deque&, deque&); - bool __lt__(deque&, deque&); - bool __gt__(deque&, deque&); - bool __le__(deque&, deque&); - bool __ge__(deque&, deque&); - #others - void assign(int, TYPE&) - void assign(iterator, iterator) - TYPE& at(int) - TYPE& back() - iterator begin() - void clear() - bool empty() - iterator end() - iterator erase(iterator) - iterator erase(iterator, iterator) - TYPE& front() - iterator insert(iterator, TYPE&) - void insert(iterator, int, TYPE&) - void insert(iterator, iterator, iterator) - int max_size() - void pop_back() - void pop_front() - void push_back(TYPE&) - void push_front(TYPE&) - iterator rbegin() - iterator rend() - void resize(int) - void resize(int, TYPE&) - int size() - void swap(container&) diff --git a/Cython/Includes/cpython/__init__.pxd b/Cython/Includes/cpython/__init__.pxd index c81f4e665..7ad2684aa 100644 --- a/Cython/Includes/cpython/__init__.pxd +++ b/Cython/Includes/cpython/__init__.pxd @@ -179,6 +179,9 @@ from cpython.bytes cimport * # Python >= 3.0 from cpython.pycapsule cimport * +# Python >= 3.7 +from cpython.contextvars cimport * + ################################################################# # END OF DEPRECATED SECTION ################################################################# diff --git a/Cython/Includes/cpython/array.pxd b/Cython/Includes/cpython/array.pxd index 19230a0a8..8431f7b66 100644 --- a/Cython/Includes/cpython/array.pxd +++ b/Cython/Includes/cpython/array.pxd @@ -46,8 +46,19 @@ : 2012-05-02 andreasvc : (see revision control) """ -from libc.string cimport strcat, strncat, \ - memset, memchr, memcmp, memcpy, memmove + +cdef extern from *: + """ + #if CYTHON_COMPILING_IN_PYPY + #ifdef _MSC_VER + #pragma message ("This module uses CPython specific internals of 'array.array', which are not available in PyPy.") + #else + #warning This module uses CPython specific internals of 'array.array', which are not available in PyPy. + #endif + #endif + """ + +from libc.string cimport memset, memcpy from cpython.object cimport Py_SIZE from cpython.ref cimport PyTypeObject, Py_TYPE diff --git a/Cython/Includes/cpython/bool.pxd b/Cython/Includes/cpython/bool.pxd index c775088ce..335921452 100644 --- a/Cython/Includes/cpython/bool.pxd +++ b/Cython/Includes/cpython/bool.pxd @@ -35,4 +35,3 @@ cdef extern from "Python.h": object PyBool_FromLong(long v) # Return value: New reference. # Return a new reference to Py_True or Py_False depending on the truth value of v. - diff --git a/Cython/Includes/cpython/bytes.pxd b/Cython/Includes/cpython/bytes.pxd index ea72c6aae..8998770d8 100644 --- a/Cython/Includes/cpython/bytes.pxd +++ b/Cython/Includes/cpython/bytes.pxd @@ -68,6 +68,10 @@ cdef extern from "Python.h": # Return value: New reference. # Identical to PyBytes_FromFormat() except that it takes exactly two arguments. + bytes PyBytes_FromObject(object o) + # Return value: New reference. + # Return the bytes representation of object o that implements the buffer protocol. + Py_ssize_t PyBytes_Size(object string) except -1 # Return the length of the string in string object string. @@ -194,5 +198,3 @@ cdef extern from "Python.h": # string encode() method. The codec to be used is looked up using # the Python codec registry. Return NULL if an exception was # raised by the codec. - - diff --git a/Cython/Includes/cpython/complex.pxd b/Cython/Includes/cpython/complex.pxd index f5ba33957..3fa145008 100644 --- a/Cython/Includes/cpython/complex.pxd +++ b/Cython/Includes/cpython/complex.pxd @@ -14,9 +14,14 @@ cdef extern from "Python.h": ctypedef class __builtin__.complex [object PyComplexObject]: cdef Py_complex cval - # not making these available to keep them read-only: - #cdef double imag "cval.imag" - #cdef double real "cval.real" + + @property + cdef inline double real(self): + return self.cval.real + + @property + cdef inline double imag(self): + return self.cval.imag # PyTypeObject PyComplex_Type # This instance of PyTypeObject represents the Python complex diff --git a/Cython/Includes/cpython/contextvars.pxd b/Cython/Includes/cpython/contextvars.pxd new file mode 100644 index 000000000..aa8002664 --- /dev/null +++ b/Cython/Includes/cpython/contextvars.pxd @@ -0,0 +1,140 @@ +from cpython.object cimport PyObject +from cpython.ref cimport Py_XDECREF + +cdef extern from "Python.h": + # Defining PyContextVar_Get() below to always return the default value for Py<3.7 and PyPy<7.3.6 + # to make the inline functions sort-of work. + """ + #if (PY_VERSION_HEX < 0x030700b1 || (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030600)) && !defined(PyContextVar_Get) + #define PyContextVar_Get(var, d, v) \ + ((d) ? \ + ((void)(var), Py_INCREF(d), (v)[0] = (d), 0) : \ + ((v)[0] = NULL, 0) \ + ) + #endif + """ + + ############################################################################ + # Context Variables Objects + ############################################################################ + + # PyContext + # The C structure used to represent a `contextvars.Context` object. + + # PyContextVar + # The C structure used to represent a `contextvars.ContextVar` object. + + # PyContextToken + # The C structure used to represent a `contextvars.Token` object. + + # PyTypeObject PyContext_Type + # Type object representing the `contextvars.Context` type. + + # PyTypeObject PyContextVar_Type + # Type object representing the `contextvars.ContextVar` type. + + # PyTypeObject PyContextToken_Type + # Type object representing the `contextvars.Token` type. + + bint PyContext_CheckExact(object obj) + # Return `true` if `obj` is of type `PyContext_Type`. + # `obj` must not be NULL. This function always succeeds. + + bint PyContextVar_CheckExact(object obj) + # Return `true` if `obj` is of type `PyContextVar_Type`. + # `obj` must not be NULL. This function always succeeds. + + bint PyContextToken_CheckExact(object obj) + # Return `true` if `obj` is of type `PyContextToken_Type`. + # `obj` must not be NULL. This function always succeeds. + + object PyContext_New() + # Return value: New reference. + # Create a new empty context object. + # Returns NULL if an error has occurred. + + object PyContext_Copy(object ctx) + # Return value: New reference. + # Create a shallow copy of the passed `ctx` context object. + # Returns NULL if an error has occurred. + + object PyContext_CopyCurrent() + # Return value: New reference. + # Create a shallow copy of the current thread context. + # Returns NULL if an error has occurred. + + int PyContext_Enter(object ctx) except -1 + # Set `ctx` as the current context for the current thread. + # Returns 0 on success, and -1 on error. + + int PyContext_Exit(object ctx) except -1 + # Deactivate the `ctx` context and restore the previous context + # as the current context for the current thread. + # Returns 0 on success, and -1 on error. + + object PyContextVar_New(const char* name, PyObject* default_value) + # Return value: New reference. + # Create a new ContextVar object. The `name` parameter is used + # for introspection and debug purposes. The `default_value` parameter + # may optionally specify the default value for the context variable. + # If an error has occurred, this function returns NULL. + + object PyContextVar_New_with_default "PyContextVar_New" (const char* name, object default_value) + # A different declaration of PyContextVar_New that requires a default value + # to be passed on call. + + int PyContextVar_Get(object var, PyObject* default_value, PyObject** value) except -1 + # Get the value of a context variable. + # Returns -1 if an error has occurred during lookup, and 0 if no error + # occurred, whether or not a value was found. + # + # If the context variable was found, `value` will be a pointer to it. + # If the context variable was not found, `value` will point to: + # + # • `default_value`, if not NULL; + # • the default value of `var`, if not NULL; + # • NULL + int PyContextVar_Get_with_default "PyContextVar_Get" (object var, object default_value, PyObject** value) except -1 + # A different declaration of PyContextVar_Get that requires a default value + # to be passed on call. + + object PyContextVar_Set(object var, object value) + # Return value: New reference. + # Set the value of `var` to `value` in the current context. + # Returns a token object for this value change, or NULL if an error has occurred. + + int PyContextVar_Reset(object var, object token) except -1 + # Reset the state of the `var` context variable to that it was in + # before `PyContextVar_Set()` that returned `token` was called. + # This function returns 0 on success and -1 on error. + + +cdef inline object get_value(var, default_value=None): + """Return a new reference to the value of the context variable, + or the default value of the context variable, + or None if no such value or default was found. + """ + cdef PyObject *value = NULL + PyContextVar_Get(var, NULL, &value) + if value is NULL: + # context variable does not have a default + pyvalue = default_value + else: + # value or default value of context variable + pyvalue = <object>value + Py_XDECREF(value) # PyContextVar_Get() returned an owned reference as 'PyObject*' + return pyvalue + + +cdef inline object get_value_no_default(var, default_value=None): + """Return a new reference to the value of the context variable, + or the provided default value if no such value was found. + + Ignores the default value of the context variable, if any. + """ + cdef PyObject *value = NULL + PyContextVar_Get(var, <PyObject*>default_value, &value) + # value of context variable or 'default_value' + pyvalue = <object>value + Py_XDECREF(value) # PyContextVar_Get() returned an owned reference as 'PyObject*' + return pyvalue diff --git a/Cython/Includes/cpython/datetime.pxd b/Cython/Includes/cpython/datetime.pxd index cd0f90719..7d6ee29f3 100644 --- a/Cython/Includes/cpython/datetime.pxd +++ b/Cython/Includes/cpython/datetime.pxd @@ -1,22 +1,161 @@ from cpython.object cimport PyObject +from cpython.version cimport PY_VERSION_HEX cdef extern from "Python.h": ctypedef struct PyTypeObject: pass cdef extern from "datetime.h": + """ + /* Backport for Python 2.x */ + #if PY_MAJOR_VERSION < 3 + #ifndef PyDateTime_DELTA_GET_DAYS + #define PyDateTime_DELTA_GET_DAYS(o) (((PyDateTime_Delta*)o)->days) + #endif + #ifndef PyDateTime_DELTA_GET_SECONDS + #define PyDateTime_DELTA_GET_SECONDS(o) (((PyDateTime_Delta*)o)->seconds) + #endif + #ifndef PyDateTime_DELTA_GET_MICROSECONDS + #define PyDateTime_DELTA_GET_MICROSECONDS(o) (((PyDateTime_Delta*)o)->microseconds) + #endif + #endif + + /* Backport for Python < 3.6 */ + #if PY_VERSION_HEX < 0x030600a4 + #ifndef PyDateTime_TIME_GET_FOLD + #define PyDateTime_TIME_GET_FOLD(o) ((void)(o), 0) + #endif + #ifndef PyDateTime_DATE_GET_FOLD + #define PyDateTime_DATE_GET_FOLD(o) ((void)(o), 0) + #endif + #endif + + /* Backport for Python < 3.6 */ + #if PY_VERSION_HEX < 0x030600a4 + #define __Pyx_DateTime_DateTimeWithFold(year, month, day, hour, minute, second, microsecond, tz, fold) \ + ((void)(fold), PyDateTimeAPI->DateTime_FromDateAndTime(year, month, day, hour, minute, second, \ + microsecond, tz, PyDateTimeAPI->DateTimeType)) + #define __Pyx_DateTime_TimeWithFold(hour, minute, second, microsecond, tz, fold) \ + ((void)(fold), PyDateTimeAPI->Time_FromTime(hour, minute, second, microsecond, tz, PyDateTimeAPI->TimeType)) + #else /* For Python 3.6+ so that we can pass tz */ + #define __Pyx_DateTime_DateTimeWithFold(year, month, day, hour, minute, second, microsecond, tz, fold) \ + PyDateTimeAPI->DateTime_FromDateAndTimeAndFold(year, month, day, hour, minute, second, \ + microsecond, tz, fold, PyDateTimeAPI->DateTimeType) + #define __Pyx_DateTime_TimeWithFold(hour, minute, second, microsecond, tz, fold) \ + PyDateTimeAPI->Time_FromTimeAndFold(hour, minute, second, microsecond, tz, fold, PyDateTimeAPI->TimeType) + #endif + + /* Backport for Python < 3.7 */ + #if PY_VERSION_HEX < 0x030700b1 + #define __Pyx_TimeZone_UTC NULL + #define __Pyx_TimeZone_FromOffsetAndName(offset, name) ((void)(offset), (void)(name), (PyObject*)NULL) + #else + #define __Pyx_TimeZone_UTC PyDateTime_TimeZone_UTC + #define __Pyx_TimeZone_FromOffsetAndName(offset, name) PyTimeZone_FromOffsetAndName(offset, name) + #endif + + /* Backport for Python < 3.10 */ + #if PY_VERSION_HEX < 0x030a00a1 + #ifndef PyDateTime_TIME_GET_TZINFO + #define PyDateTime_TIME_GET_TZINFO(o) \ + ((((PyDateTime_Time*)o)->hastzinfo) ? ((PyDateTime_Time*)o)->tzinfo : Py_None) + #endif + #ifndef PyDateTime_DATE_GET_TZINFO + #define PyDateTime_DATE_GET_TZINFO(o) \ + ((((PyDateTime_DateTime*)o)->hastzinfo) ? ((PyDateTime_DateTime*)o)->tzinfo : Py_None) + #endif + #endif + """ ctypedef extern class datetime.date[object PyDateTime_Date]: - pass + @property + cdef inline int year(self): + return PyDateTime_GET_YEAR(self) + + @property + cdef inline int month(self): + return PyDateTime_GET_MONTH(self) + + @property + cdef inline int day(self): + return PyDateTime_GET_DAY(self) ctypedef extern class datetime.time[object PyDateTime_Time]: - pass + @property + cdef inline int hour(self): + return PyDateTime_TIME_GET_HOUR(self) + + @property + cdef inline int minute(self): + return PyDateTime_TIME_GET_MINUTE(self) + + @property + cdef inline int second(self): + return PyDateTime_TIME_GET_SECOND(self) + + @property + cdef inline int microsecond(self): + return PyDateTime_TIME_GET_MICROSECOND(self) + + @property + cdef inline object tzinfo(self): + return <object>PyDateTime_TIME_GET_TZINFO(self) + + @property + cdef inline int fold(self): + # For Python < 3.6 this returns 0 no matter what + return PyDateTime_TIME_GET_FOLD(self) ctypedef extern class datetime.datetime[object PyDateTime_DateTime]: - pass + @property + cdef inline int year(self): + return PyDateTime_GET_YEAR(self) + + @property + cdef inline int month(self): + return PyDateTime_GET_MONTH(self) + + @property + cdef inline int day(self): + return PyDateTime_GET_DAY(self) + + @property + cdef inline int hour(self): + return PyDateTime_DATE_GET_HOUR(self) + + @property + cdef inline int minute(self): + return PyDateTime_DATE_GET_MINUTE(self) + + @property + cdef inline int second(self): + return PyDateTime_DATE_GET_SECOND(self) + + @property + cdef inline int microsecond(self): + return PyDateTime_DATE_GET_MICROSECOND(self) + + @property + cdef inline object tzinfo(self): + return <object>PyDateTime_DATE_GET_TZINFO(self) + + @property + cdef inline int fold(self): + # For Python < 3.6 this returns 0 no matter what + return PyDateTime_DATE_GET_FOLD(self) ctypedef extern class datetime.timedelta[object PyDateTime_Delta]: - pass + @property + cdef inline int day(self): + return PyDateTime_DELTA_GET_DAYS(self) + + @property + cdef inline int second(self): + return PyDateTime_DELTA_GET_SECONDS(self) + + @property + cdef inline int microsecond(self): + return PyDateTime_DELTA_GET_MICROSECONDS(self) ctypedef extern class datetime.tzinfo[object PyDateTime_TZInfo]: pass @@ -25,10 +164,12 @@ cdef extern from "datetime.h": pass ctypedef struct PyDateTime_Time: + unsigned char fold char hastzinfo PyObject *tzinfo ctypedef struct PyDateTime_DateTime: + unsigned char fold char hastzinfo PyObject *tzinfo @@ -47,14 +188,27 @@ cdef extern from "datetime.h": PyTypeObject *TZInfoType # constructors - object (*Date_FromDate)(int, int, int, PyTypeObject*) - object (*DateTime_FromDateAndTime)(int, int, int, int, int, int, int, object, PyTypeObject*) - object (*Time_FromTime)(int, int, int, int, object, PyTypeObject*) - object (*Delta_FromDelta)(int, int, int, int, PyTypeObject*) + date (*Date_FromDate)(int, int, int, PyTypeObject*) + datetime (*DateTime_FromDateAndTime)(int, int, int, int, int, int, int, object, PyTypeObject*) + time (*Time_FromTime)(int, int, int, int, object, PyTypeObject*) + timedelta (*Delta_FromDelta)(int, int, int, int, PyTypeObject*) # constructors for the DB API - object (*DateTime_FromTimestamp)(object, object, object) - object (*Date_FromTimestamp)(object, object) + datetime (*DateTime_FromTimestamp)(PyObject*, object, PyObject*) + date (*Date_FromTimestamp)(PyObject*, object) + + # We cannot use the following because they do not compile in older Python versions. + # Instead, we use datetime.h's macros here that we can backport in C. + + # Python 3.7+ constructors + object (*TimeZone_FromTimeZone)(object offset, PyObject *name) + + # Python 3.7+ singletons + PyObject *TimeZone_UTC + + # Python 3.6+ PEP 495 constructors + datetime (*DateTime_FromDateAndTimeAndFold)(int, int, int, int, int, int, int, object, int, PyTypeObject*) + time (*Time_FromTimeAndFold)(int, int, int ,int, object, int, PyTypeObject*) # Check type of the object. bint PyDate_Check(object op) @@ -82,21 +236,45 @@ cdef extern from "datetime.h": int PyDateTime_DATE_GET_MINUTE(object o) int PyDateTime_DATE_GET_SECOND(object o) int PyDateTime_DATE_GET_MICROSECOND(object o) + int PyDateTime_DATE_GET_FOLD(object o) + PyObject* PyDateTime_DATE_GET_TZINFO(object o) # returns a borrowed reference # Getters for time (C macros). int PyDateTime_TIME_GET_HOUR(object o) int PyDateTime_TIME_GET_MINUTE(object o) int PyDateTime_TIME_GET_SECOND(object o) int PyDateTime_TIME_GET_MICROSECOND(object o) + int PyDateTime_TIME_GET_FOLD(object o) + PyObject* PyDateTime_TIME_GET_TZINFO(object o) # returns a borrowed reference # Getters for timedelta (C macros). int PyDateTime_DELTA_GET_DAYS(object o) int PyDateTime_DELTA_GET_SECONDS(object o) int PyDateTime_DELTA_GET_MICROSECONDS(object o) + # Constructors + object PyTimeZone_FromOffset(object offset) + object PyTimeZone_FromOffsetAndName(object offset, object name) + + # The above macros is Python 3.7+ so we use these instead + object __Pyx_TimeZone_FromOffsetAndName(object offset, PyObject* name) + + # Constructors for the DB API + datetime PyDateTime_FromTimeStamp(object args) + date PyDate_FromTimeStamp(object args) + + # PEP 495 constructors but patched above to allow passing tz + datetime __Pyx_DateTime_DateTimeWithFold(int, int, int, int, int, int, int, object, int) + datetime __Pyx_DateTime_TimeWithFold(int, int, int ,int, object, int) + # PyDateTime CAPI object. PyDateTime_CAPI *PyDateTimeAPI + PyObject* PyDateTime_TimeZone_UTC + + # PyDateTime_TimeZone_UTC is Python 3.7+ so instead we use the following macro + PyObject* __Pyx_TimeZone_UTC + void PyDateTime_IMPORT() # Datetime C API initialization function. @@ -106,42 +284,57 @@ cdef inline void import_datetime(): # Create date object using DateTime CAPI factory function. # Note, there are no range checks for any of the arguments. -cdef inline object date_new(int year, int month, int day): +cdef inline date date_new(int year, int month, int day): return PyDateTimeAPI.Date_FromDate(year, month, day, PyDateTimeAPI.DateType) # Create time object using DateTime CAPI factory function # Note, there are no range checks for any of the arguments. -cdef inline object time_new(int hour, int minute, int second, int microsecond, object tz): - return PyDateTimeAPI.Time_FromTime(hour, minute, second, microsecond, tz, PyDateTimeAPI.TimeType) +cdef inline time time_new(int hour, int minute, int second, int microsecond, object tz, int fold=0): + return __Pyx_DateTime_TimeWithFold(hour, minute, second, microsecond, tz, fold) # Create datetime object using DateTime CAPI factory function. # Note, there are no range checks for any of the arguments. -cdef inline object datetime_new(int year, int month, int day, int hour, int minute, int second, int microsecond, object tz): - return PyDateTimeAPI.DateTime_FromDateAndTime(year, month, day, hour, minute, second, microsecond, tz, PyDateTimeAPI.DateTimeType) +cdef inline datetime datetime_new(int year, int month, int day, int hour, int minute, int second, int microsecond, object tz, int fold=0): + return __Pyx_DateTime_DateTimeWithFold(year, month, day, hour, minute, second, microsecond, tz, fold) # Create timedelta object using DateTime CAPI factory function. # Note, there are no range checks for any of the arguments. -cdef inline object timedelta_new(int days, int seconds, int useconds): +cdef inline timedelta timedelta_new(int days, int seconds, int useconds): return PyDateTimeAPI.Delta_FromDelta(days, seconds, useconds, 1, PyDateTimeAPI.DeltaType) +# Create timedelta object using DateTime CAPI factory function. +cdef inline object timezone_new(object offset, object name=None): + if PY_VERSION_HEX < 0x030700b1: + raise RuntimeError('Time zones are not available from the C-API.') + return __Pyx_TimeZone_FromOffsetAndName(offset, <PyObject*>name if name is not None else NULL) + +# Create datetime object using DB API constructor. +cdef inline datetime datetime_from_timestamp(timestamp, tz=None): + return PyDateTimeAPI.DateTime_FromTimestamp( + <PyObject*>PyDateTimeAPI.DateTimeType, (timestamp, tz) if tz is not None else (timestamp,), NULL) + +# Create date object using DB API constructor. +cdef inline date date_from_timestamp(timestamp): + return PyDateTimeAPI.Date_FromTimestamp(<PyObject*>PyDateTimeAPI.DateType, (timestamp,)) + # More recognizable getters for date/time/datetime/timedelta. # There are no setters because datetime.h hasn't them. # This is because of immutable nature of these objects by design. # If you would change time/date/datetime/timedelta object you need to recreate. +# Get UTC singleton +cdef inline object get_utc(): + if PY_VERSION_HEX < 0x030700b1: + raise RuntimeError('Time zones are not available from the C-API.') + return <object>__Pyx_TimeZone_UTC + # Get tzinfo of time cdef inline object time_tzinfo(object o): - if (<PyDateTime_Time*>o).hastzinfo: - return <object>(<PyDateTime_Time*>o).tzinfo - else: - return None + return <object>PyDateTime_TIME_GET_TZINFO(o) # Get tzinfo of datetime cdef inline object datetime_tzinfo(object o): - if (<PyDateTime_DateTime*>o).hastzinfo: - return <object>(<PyDateTime_DateTime*>o).tzinfo - else: - return None + return <object>PyDateTime_DATE_GET_TZINFO(o) # Get year of date cdef inline int date_year(object o): @@ -183,6 +376,11 @@ cdef inline int time_second(object o): cdef inline int time_microsecond(object o): return PyDateTime_TIME_GET_MICROSECOND(o) +# Get fold of time +cdef inline int time_fold(object o): + # For Python < 3.6 this returns 0 no matter what + return PyDateTime_TIME_GET_FOLD(o) + # Get hour of datetime cdef inline int datetime_hour(object o): return PyDateTime_DATE_GET_HOUR(o) @@ -199,6 +397,11 @@ cdef inline int datetime_second(object o): cdef inline int datetime_microsecond(object o): return PyDateTime_DATE_GET_MICROSECOND(o) +# Get fold of datetime +cdef inline int datetime_fold(object o): + # For Python < 3.6 this returns 0 no matter what + return PyDateTime_DATE_GET_FOLD(o) + # Get days of timedelta cdef inline int timedelta_days(object o): return (<PyDateTime_Delta*>o).days @@ -210,3 +413,14 @@ cdef inline int timedelta_seconds(object o): # Get microseconds of timedelta cdef inline int timedelta_microseconds(object o): return (<PyDateTime_Delta*>o).microseconds + +cdef inline double total_seconds(timedelta obj): + # Mirrors the "timedelta.total_seconds()" method. + # Note that this implementation is not guaranteed to give *exactly* the same + # result as the original method, due to potential differences in floating point rounding. + cdef: + double days, seconds, micros + days = <double>PyDateTime_DELTA_GET_DAYS(obj) + seconds = <double>PyDateTime_DELTA_GET_SECONDS(obj) + micros = <double>PyDateTime_DELTA_GET_MICROSECONDS(obj) + return days * 24 * 3600 + seconds + micros / 1_000_000 diff --git a/Cython/Includes/cpython/descr.pxd b/Cython/Includes/cpython/descr.pxd new file mode 100644 index 000000000..5075f0bbd --- /dev/null +++ b/Cython/Includes/cpython/descr.pxd @@ -0,0 +1,26 @@ +from .object cimport PyObject, PyTypeObject + +cdef extern from "Python.h": + ctypedef object (*wrapperfunc)(self, args, void* wrapped) + ctypedef object (*wrapperfunc_kwds)(self, args, void* wrapped, kwds) + + struct wrapperbase: + char* name + int offset + void* function + wrapperfunc wrapper + char* doc + int flags + PyObject* name_strobj + + int PyWrapperFlag_KEYWORDS + + ctypedef class __builtin__.wrapper_descriptor [object PyWrapperDescrObject]: + cdef type d_type + cdef d_name + cdef wrapperbase* d_base + cdef void* d_wrapped + + object PyDescr_NewWrapper(PyTypeObject* cls, wrapperbase* wrapper, void* wrapped) + + int PyDescr_IsData(descr) diff --git a/Cython/Includes/cpython/dict.pxd b/Cython/Includes/cpython/dict.pxd index 16dd5e145..979dd392a 100644 --- a/Cython/Includes/cpython/dict.pxd +++ b/Cython/Includes/cpython/dict.pxd @@ -1,6 +1,13 @@ from .object cimport PyObject +from .pyport cimport uint64_t cdef extern from "Python.h": + # On Python 2, PyDict_GetItemWithError is called _PyDict_GetItemWithError + """ + #if PY_MAJOR_VERSION <= 2 + #define PyDict_GetItemWithError _PyDict_GetItemWithError + #endif + """ ############################################################################ # 7.4.1 Dictionary Objects @@ -72,11 +79,25 @@ cdef extern from "Python.h": # NULL if the key key is not present, but without setting an # exception. + PyObject* PyDict_GetItemWithError(object p, object key) except? NULL + # Return value: Borrowed reference. + # Variant of PyDict_GetItem() that does not suppress exceptions. Return + # NULL with an exception set if an exception occurred. Return NULL + # without an exception set if the key wasn’t present. + PyObject* PyDict_GetItemString(object p, const char *key) # Return value: Borrowed reference. # This is the same as PyDict_GetItem(), but key is specified as a # char*, rather than a PyObject*. + PyObject* PyDict_SetDefault(object p, object key, object default) except NULL + # Return value: Borrowed reference. + # This is the same as the Python-level dict.setdefault(). If present, it + # returns the value corresponding to key from the dictionary p. If the key + # is not in the dict, it is inserted with value defaultobj and defaultobj + # is returned. This function evaluates the hash function of key only once, + # instead of evaluating it independently for the lookup and the insertion. + list PyDict_Items(object p) # Return value: New reference. # Return a PyListObject containing all the items from the diff --git a/Cython/Includes/cpython/exc.pxd b/Cython/Includes/cpython/exc.pxd index bc57c0e57..756342ad3 100644 --- a/Cython/Includes/cpython/exc.pxd +++ b/Cython/Includes/cpython/exc.pxd @@ -88,6 +88,11 @@ cdef extern from "Python.h": # needs to handle exceptions or by code that needs to save and # restore the error indicator temporarily. + PyObject* PyErr_GetHandledException() + void PyErr_SetHandledException(PyObject* exc) + PyObject* PyErr_GetRaisedException() + void PyErr_SetRaisedException(PyObject* exc) + void PyErr_Restore(PyObject* type, PyObject* value, PyObject* traceback) # Set the error indicator from the three objects. If the error # indicator is already set, it is cleared first. If the objects @@ -236,6 +241,8 @@ cdef extern from "Python.h": # KeyboardInterrupt will be raised. It may be called without # holding the interpreter lock. + int PyErr_SetInterruptEx(int signum) + object PyErr_NewException(char *name, object base, object dict) # Return value: New reference. # This utility function creates and returns a new exception @@ -254,4 +261,3 @@ cdef extern from "Python.h": # identifies the context in which the unraisable exception # occurred. The repr of obj will be printed in the warning # message. - diff --git a/Cython/Includes/cpython/fileobject.pxd b/Cython/Includes/cpython/fileobject.pxd new file mode 100644 index 000000000..e52cd33f5 --- /dev/null +++ b/Cython/Includes/cpython/fileobject.pxd @@ -0,0 +1,57 @@ +""" +From https://docs.python.org/3.9/c-api/file.html + +These APIs are a minimal emulation of the Python 2 C API for built-in file objects, +which used to rely on the buffered I/O (FILE*) support from the C standard library. +In Python 3, files and streams use the new io module, which defines several layers +over the low-level unbuffered I/O of the operating system. The functions described +below are convenience C wrappers over these new APIs, and meant mostly for internal +error reporting in the interpreter; + +third-party code is advised to access the io APIs instead. +""" + +cdef extern from "Python.h": + + ########################################################################### + # File Objects + ########################################################################### + + object PyFile_FromFd(int fd, const char *name, const char *mode, int buffering, + const char *encoding, const char *errors, const char *newline, int closefd) + # Return value: New reference. + # Create a Python file object from the file descriptor of an already + # opened file fd. The arguments name, encoding, errors and newline can be + # NULL to use the defaults; buffering can be -1 to use the default. name + # is ignored and kept for backward compatibility. Return NULL on failure. + # For a more comprehensive description of the arguments, please refer to + # the io.open() function documentation. + + # Warning: Since Python streams have their own buffering layer, mixing + # them with OS-level file descriptors can produce various issues (such as + # unexpected ordering of data). + + # Changed in version 3.2: Ignore name attribute. + + object PyFile_GetLine(object p, int n) + # Return value: New reference. + # Equivalent to p.readline([n]), this function reads one line from the + # object p. p may be a file object or any object with a readline() + # method. If n is 0, exactly one line is read, regardless of the length of + # the line. If n is greater than 0, no more than n bytes will be read from + # the file; a partial line can be returned. In both cases, an empty string + # is returned if the end of the file is reached immediately. If n is less + # than 0, however, one line is read regardless of length, but EOFError is + # raised if the end of the file is reached immediately. + + int PyFile_WriteObject(object obj, object p, int flags) except? -1 + # Write object obj to file object p. The only supported flag for flags + # is Py_PRINT_RAW; if given, the str() of the object is written instead of + # the repr(). Return 0 on success or -1 on failure; the appropriate + # exception will be set. + + int PyFile_WriteString(const char *s, object p) except? -1 + # Write string s to file object p. Return 0 on success or -1 on failure; + # the appropriate exception will be set. + + enum: Py_PRINT_RAW diff --git a/Cython/Includes/cpython/float.pxd b/Cython/Includes/cpython/float.pxd index 65328f31e..7c567a80f 100644 --- a/Cython/Includes/cpython/float.pxd +++ b/Cython/Includes/cpython/float.pxd @@ -1,4 +1,11 @@ cdef extern from "Python.h": + """ + #if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyFloat_FromString(obj) PyFloat_FromString(obj) + #else + #define __Pyx_PyFloat_FromString(obj) PyFloat_FromString(obj, NULL) + #endif + """ ############################################################################ # 7.2.3 @@ -21,7 +28,7 @@ cdef extern from "Python.h": # Return true if its argument is a PyFloatObject, but not a # subtype of PyFloatObject. - object PyFloat_FromString(object str, char **pend) + object PyFloat_FromString "__Pyx_PyFloat_FromString" (object str) # Return value: New reference. # Create a PyFloatObject object based on the string value in str, # or NULL on failure. The pend argument is ignored. It remains diff --git a/Cython/Includes/cpython/list.pxd b/Cython/Includes/cpython/list.pxd index c6a29535c..1d0503c2c 100644 --- a/Cython/Includes/cpython/list.pxd +++ b/Cython/Includes/cpython/list.pxd @@ -42,17 +42,19 @@ cdef extern from "Python.h": int PyList_SetItem(object list, Py_ssize_t index, object item) except -1 # Set the item at index index in list to item. Return 0 on success - # or -1 on failure. Note: This function ``steals'' a reference to - # item and discards a reference to an item already in the list at - # the affected position. + # or -1 on failure. + # + # WARNING: This function _steals_ a reference to item and discards a + # reference to an item already in the list at the affected position. void PyList_SET_ITEM(object list, Py_ssize_t i, object o) # Macro form of PyList_SetItem() without error checking. This is # normally only used to fill in new lists where there is no - # previous content. Note: This function ``steals'' a reference to - # item, and, unlike PyList_SetItem(), does not discard a reference - # to any item that it being replaced; any reference in list at - # position i will be *leaked*. + # previous content. + # + # WARNING: This function _steals_ a reference to item, and, unlike + # PyList_SetItem(), does not discard a reference to any item that + # it being replaced; any reference in list at position i will be *leaked*. int PyList_Insert(object list, Py_ssize_t index, object item) except -1 # Insert the item item into list list in front of index @@ -88,5 +90,3 @@ cdef extern from "Python.h": # Return value: New reference. # Return a new tuple object containing the contents of list; # equivalent to "tuple(list)". - - diff --git a/Cython/Includes/cpython/long.pxd b/Cython/Includes/cpython/long.pxd index eb8140d41..f65cd0073 100644 --- a/Cython/Includes/cpython/long.pxd +++ b/Cython/Includes/cpython/long.pxd @@ -89,7 +89,7 @@ cdef extern from "Python.h": # Return a C long representation of the contents of pylong. If # pylong is greater than LONG_MAX, an OverflowError is raised. - # long PyLong_AsLongAndOverflow(object pylong, int *overflow) except? -1 + long PyLong_AsLongAndOverflow(object pylong, int *overflow) except? -1 # Return a C long representation of the contents of pylong. If pylong is # greater than LONG_MAX or less than LONG_MIN, set *overflow to 1 or -1, # respectively, and return -1; otherwise, set *overflow to 0. If any other @@ -97,7 +97,7 @@ cdef extern from "Python.h": # be returned and *overflow will be 0. # New in version 2.7. - # PY_LONG_LONG PyLong_AsLongLongAndOverflow(object pylong, int *overflow) except? -1 + PY_LONG_LONG PyLong_AsLongLongAndOverflow(object pylong, int *overflow) except? -1 # Return a C long long representation of the contents of pylong. If pylong # is greater than PY_LLONG_MAX or less than PY_LLONG_MIN, set *overflow to # 1 or -1, respectively, and return -1; otherwise, set *overflow to 0. If diff --git a/Cython/Includes/cpython/mapping.pxd b/Cython/Includes/cpython/mapping.pxd index 3d235b65e..5e54af531 100644 --- a/Cython/Includes/cpython/mapping.pxd +++ b/Cython/Includes/cpython/mapping.pxd @@ -61,4 +61,3 @@ cdef extern from "Python.h": # Map the object key to the value v in object o. Returns -1 on # failure. This is the equivalent of the Python statement "o[key] # = v". - diff --git a/Cython/Includes/cpython/marshal.pxd b/Cython/Includes/cpython/marshal.pxd new file mode 100644 index 000000000..1d8a54c86 --- /dev/null +++ b/Cython/Includes/cpython/marshal.pxd @@ -0,0 +1,66 @@ +from libc.stdio cimport FILE + +cdef extern from "Python.h": + + ########################################################################### + # Data marshalling support + ########################################################################### + + const int Py_MARSHAL_VERSION + + void PyMarshal_WriteLongToFile(long value, FILE *file, int version) + # Marshal a long integer, value, to file. This will only write the + # least-significant 32 bits of value, regardless of the size of the native + # long type. version indicates the file format. + + void PyMarshal_WriteObjectToFile(object value, FILE *file, int version) + # Marshal a Python object, value, to file. version indicates the file + # format. + + bytes PyMarshal_WriteObjectToString(object value, int version) + # Return value: New reference. + # Return a bytes object containing the marshalled representation of value. + # version indicates the file format. + + long PyMarshal_ReadLongFromFile(FILE *file) except? -1 + # Return a C long from the data stream in a FILE* opened for reading. Only + # a 32-bit value can be read in using this function, regardless of the + # native size of long. + + # On error, sets the appropriate exception (EOFError) and returns -1. + + int PyMarshal_ReadShortFromFile(FILE *file) except? -1 + # Return a C short from the data stream in a FILE* opened for reading. Only + # a 16-bit value can be read in using this function, regardless of the + # native size of short. + + # On error, sets the appropriate exception (EOFError) and returns -1. + + object PyMarshal_ReadObjectFromFile(FILE *file) + # Return value: New reference. + # Return a Python object from the data stream in a FILE* opened for + # reading. + + # On error, sets the appropriate exception (EOFError, ValueError or + # TypeError) and returns NULL. + + object PyMarshal_ReadLastObjectFromFile(FILE *file) + # Return value: New reference. + # Return a Python object from the data stream in a FILE* opened for + # reading. Unlike PyMarshal_ReadObjectFromFile(), this function assumes + # that no further objects will be read from the file, allowing it to + # aggressively load file data into memory so that the de-serialization can + # operate from data in memory, rather than reading a byte at a time from the + # file. Only use these variant if you are certain that you won’t be reading + # anything else from the file. + + # On error, sets the appropriate exception (EOFError, ValueError or + # TypeError) and returns NULL. + + object PyMarshal_ReadObjectFromString(const char *data, Py_ssize_t len) + # Return value: New reference. + # Return a Python object from the data stream in a byte buffer containing + # len bytes pointed to by data. + + # On error, sets the appropriate exception (EOFError, ValueError or + # TypeError) and returns NULL. diff --git a/Cython/Includes/cpython/mem.pxd b/Cython/Includes/cpython/mem.pxd index af820f2ee..236d111f6 100644 --- a/Cython/Includes/cpython/mem.pxd +++ b/Cython/Includes/cpython/mem.pxd @@ -35,6 +35,15 @@ cdef extern from "Python.h": # PyMem_Malloc(1) had been called instead. The memory will not # have been initialized in any way. + void* PyMem_RawCalloc(size_t nelem, size_t elsize) nogil + void* PyMem_Calloc(size_t nelem, size_t elsize) + # Allocates nelem elements each whose size in bytes is elsize and + # returns a pointer of type void* to the allocated memory, or NULL if + # the request fails. The memory is initialized to zeros. Requesting + # zero elements or elements of size zero bytes returns a distinct + # non-NULL pointer if possible, as if PyMem_Calloc(1, 1) had been + # called instead. + void* PyMem_RawRealloc(void *p, size_t n) nogil void* PyMem_Realloc(void *p, size_t n) # Resizes the memory block pointed to by p to n bytes. The @@ -43,13 +52,13 @@ cdef extern from "Python.h": # else if n is equal to zero, the memory block is resized but is # not freed, and the returned pointer is non-NULL. Unless p is # NULL, it must have been returned by a previous call to - # PyMem_Malloc() or PyMem_Realloc(). + # PyMem_Malloc(), PyMem_Realloc(), or PyMem_Calloc(). void PyMem_RawFree(void *p) nogil void PyMem_Free(void *p) # Frees the memory block pointed to by p, which must have been - # returned by a previous call to PyMem_Malloc() or - # PyMem_Realloc(). Otherwise, or if PyMem_Free(p) has been called + # returned by a previous call to PyMem_Malloc(), PyMem_Realloc(), or + # PyMem_Calloc(). Otherwise, or if PyMem_Free(p) has been called # before, undefined behavior occurs. If p is NULL, no operation is # performed. diff --git a/Cython/Includes/cpython/module.pxd b/Cython/Includes/cpython/module.pxd index 8eb323b01..ea4c3817e 100644 --- a/Cython/Includes/cpython/module.pxd +++ b/Cython/Includes/cpython/module.pxd @@ -145,6 +145,12 @@ cdef extern from "Python.h": bint PyModule_CheckExact(object p) # Return true if p is a module object, but not a subtype of PyModule_Type. + object PyModule_NewObject(object name) + # Return a new module object with the __name__ attribute set to name. + # The module’s __name__, __doc__, __package__, and __loader__ + # attributes are filled in (all but __name__ are set to None); the caller + # is responsible for providing a __file__ attribute. + object PyModule_New(const char *name) # Return value: New reference. # Return a new module object with the __name__ attribute set to @@ -160,21 +166,35 @@ cdef extern from "Python.h": # use other PyModule_*() and PyObject_*() functions rather than # directly manipulate a module's __dict__. + object PyModule_GetNameObject(object module) + # Return module’s __name__ value. If the module does not provide one, or if + # it is not a string, SystemError is raised and NULL is returned. + char* PyModule_GetName(object module) except NULL - # Return module's __name__ value. If the module does not provide - # one, or if it is not a string, SystemError is raised and NULL is - # returned. + # Similar to PyModule_GetNameObject() but return the name encoded + # to 'utf-8'. + + void* PyModule_GetState(object module) + # Return the “state” of the module, that is, a pointer to the block of + # memory allocated at module creation time, or NULL. + # See PyModuleDef.m_size. + + object PyModule_GetFilenameObject(object module) + # Return the name of the file from which module was loaded using module’s + # __file__ attribute. If this is not defined, or if it is not a unicode + # string, raise SystemError and return NULL; otherwise return a reference + # to a Unicode object. char* PyModule_GetFilename(object module) except NULL - # Return the name of the file from which module was loaded using - # module's __file__ attribute. If this is not defined, or if it is - # not a string, raise SystemError and return NULL. + # Similar to PyModule_GetFilenameObject() but return the filename encoded + # to ‘utf-8’. int PyModule_AddObject(object module, const char *name, object value) except -1 # Add an object to module as name. This is a convenience function - # which can be used from the module's initialization - # function. This steals a reference to value. Return -1 on error, - # 0 on success. + # which can be used from the module's initialization function. + # Return -1 on error, 0 on success. + # + # WARNING: This _steals_ a reference to value. int PyModule_AddIntConstant(object module, const char *name, long value) except -1 # Add an integer constant to module as name. This convenience diff --git a/Cython/Includes/cpython/object.pxd b/Cython/Includes/cpython/object.pxd index 5a8116639..41874159c 100644 --- a/Cython/Includes/cpython/object.pxd +++ b/Cython/Includes/cpython/object.pxd @@ -5,7 +5,7 @@ cdef extern from "Python.h": ctypedef struct PyObject # forward declaration - ctypedef object (*newfunc)(cpython.type.type, object, object) # (type, args, kwargs) + ctypedef object (*newfunc)(cpython.type.type, PyObject*, PyObject*) # (type, args|NULL, kwargs|NULL) ctypedef object (*unaryfunc)(object) ctypedef object (*binaryfunc)(object, object) @@ -35,6 +35,14 @@ cdef extern from "Python.h": ctypedef object (*descrgetfunc)(object, object, object) ctypedef int (*descrsetfunc)(object, object, object) except -1 + ctypedef object (*PyCFunction)(object, object) + + ctypedef struct PyMethodDef: + const char* ml_name + PyCFunction ml_meth + int ml_flags + const char* ml_doc + ctypedef struct PyTypeObject: const char* tp_name const char* tp_doc @@ -45,6 +53,8 @@ cdef extern from "Python.h": newfunc tp_new destructor tp_dealloc + destructor tp_del + destructor tp_finalize traverseproc tp_traverse inquiry tp_clear freefunc tp_free @@ -57,12 +67,16 @@ cdef extern from "Python.h": cmpfunc tp_compare richcmpfunc tp_richcompare + PyMethodDef* tp_methods + PyTypeObject* tp_base PyObject* tp_dict descrgetfunc tp_descr_get descrsetfunc tp_descr_set + unsigned int tp_version_tag + ctypedef struct PyObject: Py_ssize_t ob_refcnt PyTypeObject *ob_type @@ -128,6 +142,17 @@ cdef extern from "Python.h": # failure. This is the equivalent of the Python statement "del # o.attr_name". + object PyObject_GenericGetDict(object o, void *context) + # Return value: New reference. + # A generic implementation for the getter of a __dict__ descriptor. It + # creates the dictionary if necessary. + # New in version 3.3. + + int PyObject_GenericSetDict(object o, object value, void *context) except -1 + # A generic implementation for the setter of a __dict__ descriptor. This + # implementation does not allow the dictionary to be deleted. + # New in version 3.3. + int Py_LT, Py_LE, Py_EQ, Py_NE, Py_GT, Py_GE object PyObject_RichCompare(object o1, object o2, int opid) @@ -177,6 +202,14 @@ cdef extern from "Python.h": # equivalent of the Python expression "str(o)". Called by the # str() built-in function and by the print statement. + object PyObject_Bytes(object o) + # Return value: New reference. + # Compute a bytes representation of object o. Return NULL on + # failure and a bytes object on success. This is equivalent to + # the Python expression bytes(o), when o is not an integer. + # Unlike bytes(o), a TypeError is raised when o is an integer + # instead of a zero-initialized bytes object. + object PyObject_Unicode(object o) # Return value: New reference. # Compute a Unicode string representation of object o. Returns the @@ -320,6 +353,13 @@ cdef extern from "Python.h": # returned. On error, -1 is returned. This is the equivalent to # the Python expression "len(o)". + Py_ssize_t PyObject_LengthHint(object o, Py_ssize_t default) except -1 + # Return an estimated length for the object o. First try to return its + # actual length, then an estimate using __length_hint__(), and finally + # return the default value. On error, return -1. This is the equivalent to + # the Python expression "operator.length_hint(o, default)". + # New in version 3.4. + object PyObject_GetItem(object o, object key) # Return value: New reference. # Return element of o corresponding to the object key or NULL on @@ -397,3 +437,4 @@ cdef extern from "Python.h": long Py_TPFLAGS_DEFAULT_EXTERNAL long Py_TPFLAGS_DEFAULT_CORE long Py_TPFLAGS_DEFAULT + long Py_TPFLAGS_HAVE_FINALIZE diff --git a/Cython/Includes/cpython/pycapsule.pxd b/Cython/Includes/cpython/pycapsule.pxd index 08062da85..1c21b370b 100644 --- a/Cython/Includes/cpython/pycapsule.pxd +++ b/Cython/Includes/cpython/pycapsule.pxd @@ -1,5 +1,5 @@ -# available since Python 3.1! +# available since Python 2.7! cdef extern from "Python.h": @@ -141,4 +141,3 @@ cdef extern from "Python.h": # set an exception and return NULL. However, if PyCapsule_Import() # failed to import the module, and no_block was true, no exception # is set. - diff --git a/Cython/Includes/cpython/pyport.pxd b/Cython/Includes/cpython/pyport.pxd new file mode 100644 index 000000000..fec59c9c8 --- /dev/null +++ b/Cython/Includes/cpython/pyport.pxd @@ -0,0 +1,8 @@ +cdef extern from "Python.h": + ctypedef int int32_t + ctypedef int int64_t + ctypedef unsigned int uint32_t + ctypedef unsigned int uint64_t + + const Py_ssize_t PY_SSIZE_T_MIN + const Py_ssize_t PY_SSIZE_T_MAX diff --git a/Cython/Includes/cpython/pystate.pxd b/Cython/Includes/cpython/pystate.pxd index 1af630793..ee8856b20 100644 --- a/Cython/Includes/cpython/pystate.pxd +++ b/Cython/Includes/cpython/pystate.pxd @@ -84,6 +84,9 @@ cdef extern from "Python.h": # PyGILState_Release on the same thread. void PyGILState_Release(PyGILState_STATE) + # Return 1 if the current thread holds the GIL and 0 otherwise. + int PyGILState_Check() + # Routines for advanced debuggers, requested by David Beazley. # Don't use unless you know what you are doing! PyInterpreterState * PyInterpreterState_Head() diff --git a/Cython/Includes/cpython/ref.pxd b/Cython/Includes/cpython/ref.pxd index 4bc9a7d7c..44f870006 100644 --- a/Cython/Includes/cpython/ref.pxd +++ b/Cython/Includes/cpython/ref.pxd @@ -48,4 +48,3 @@ cdef extern from "Python.h": # It is a good idea to use this macro whenever decrementing the # value of a variable that might be traversed during garbage # collection. - diff --git a/Cython/Includes/cpython/sequence.pxd b/Cython/Includes/cpython/sequence.pxd index eb279968d..e50e4c495 100644 --- a/Cython/Includes/cpython/sequence.pxd +++ b/Cython/Includes/cpython/sequence.pxd @@ -132,5 +132,3 @@ cdef extern from "Python.h": # gotten by calling PySequence_Size() on o, but # PySequence_Fast_GET_SIZE() is faster because it can assume o is # a list or tuple. - - diff --git a/Cython/Includes/cpython/string.pxd b/Cython/Includes/cpython/string.pxd index 8af78f3dd..3ba25515f 100644 --- a/Cython/Includes/cpython/string.pxd +++ b/Cython/Includes/cpython/string.pxd @@ -194,5 +194,3 @@ cdef extern from "Python.h": # string encode() method. The codec to be used is looked up using # the Python codec registry. Return NULL if an exception was # raised by the codec. - - diff --git a/Cython/Includes/cpython/time.pxd b/Cython/Includes/cpython/time.pxd new file mode 100644 index 000000000..7f20095a1 --- /dev/null +++ b/Cython/Includes/cpython/time.pxd @@ -0,0 +1,51 @@ +""" +Cython implementation of (parts of) the standard library time module. +""" + +from libc.stdint cimport int64_t +from cpython.exc cimport PyErr_SetFromErrno + +cdef extern from "Python.h": + ctypedef int64_t _PyTime_t + _PyTime_t _PyTime_GetSystemClock() nogil + double _PyTime_AsSecondsDouble(_PyTime_t t) nogil + +from libc.time cimport ( + tm, + time_t, + localtime as libc_localtime, +) + + +cdef inline double time() nogil: + cdef: + _PyTime_t tic + + tic = _PyTime_GetSystemClock() + return _PyTime_AsSecondsDouble(tic) + + +cdef inline int _raise_from_errno() except -1 with gil: + PyErr_SetFromErrno(RuntimeError) + return <int> -1 # Let the C compiler know that this function always raises. + + +cdef inline tm localtime() except * nogil: + """ + Analogue to the stdlib time.localtime. The returned struct + has some entries that the stdlib version does not: tm_gmtoff, tm_zone + """ + cdef: + time_t tic = <time_t>time() + tm* result + + result = libc_localtime(&tic) + if result is NULL: + _raise_from_errno() + # Fix 0-based date values (and the 1900-based year). + # See tmtotuple() in https://github.com/python/cpython/blob/master/Modules/timemodule.c + result.tm_year += 1900 + result.tm_mon += 1 + result.tm_wday = (result.tm_wday + 6) % 7 + result.tm_yday += 1 + return result[0] diff --git a/Cython/Includes/cpython/tuple.pxd b/Cython/Includes/cpython/tuple.pxd index 09c46e0b4..907033fe4 100644 --- a/Cython/Includes/cpython/tuple.pxd +++ b/Cython/Includes/cpython/tuple.pxd @@ -47,13 +47,15 @@ cdef extern from "Python.h": int PyTuple_SetItem(object p, Py_ssize_t pos, object o) except -1 # Insert a reference to object o at position pos of the tuple - # pointed to by p. Return 0 on success. Note: This function - # ``steals'' a reference to o. + # pointed to by p. Return 0 on success. + # + # WARNING: This function _steals_ a reference to o. void PyTuple_SET_ITEM(object p, Py_ssize_t pos, object o) # Like PyTuple_SetItem(), but does no error checking, and should - # only be used to fill in brand new tuples. Note: This function - # ``steals'' a reference to o. + # only be used to fill in brand new tuples. + # + # WARNING: This function _steals_ a reference to o. int _PyTuple_Resize(PyObject **p, Py_ssize_t newsize) except -1 # Can be used to resize a tuple. newsize will be the new length of @@ -68,4 +70,3 @@ cdef extern from "Python.h": # the object referenced by *p is replaced, the original *p is # destroyed. On failure, returns -1 and sets *p to NULL, and # raises MemoryError or SystemError. - diff --git a/Cython/Includes/cpython/type.pxd b/Cython/Includes/cpython/type.pxd index a1d094e37..928a748cd 100644 --- a/Cython/Includes/cpython/type.pxd +++ b/Cython/Includes/cpython/type.pxd @@ -23,6 +23,11 @@ cdef extern from "Python.h": # of the standard type object. Return false in all other # cases. + void PyType_Modified(type type) + # Invalidate the internal lookup cache for the type and all of its + # subtypes. This function must be called after any manual modification + # of the attributes or base classes of the type. + bint PyType_HasFeature(object o, int feature) # Return true if the type object o sets the feature feature. Type # features are denoted by single bit flags. diff --git a/Cython/Includes/cpython/unicode.pxd b/Cython/Includes/cpython/unicode.pxd index 2072c8cb2..6452d892e 100644 --- a/Cython/Includes/cpython/unicode.pxd +++ b/Cython/Includes/cpython/unicode.pxd @@ -1,4 +1,8 @@ + cdef extern from *: + ctypedef unsigned char Py_UCS1 # uint8_t + ctypedef unsigned short Py_UCS2 # uint16_t + # Return true if the object o is a Unicode object or an instance # of a Unicode subtype. Changed in version 2.2: Allowed subtypes # to be accepted. @@ -23,6 +27,21 @@ cdef extern from *: # New in version 3.3. Py_ssize_t PyUnicode_GET_LENGTH(object o) + Py_UCS1 *PyUnicode_1BYTE_DATA(object o) + Py_UCS2 *PyUnicode_2BYTE_DATA(object o) + Py_UCS4 *PyUnicode_4BYTE_DATA(object o) + + int PyUnicode_WCHAR_KIND # Deprecated since Python 3.10, removed in 3.12. + int PyUnicode_1BYTE_KIND + int PyUnicode_2BYTE_KIND + int PyUnicode_4BYTE_KIND + void PyUnicode_WRITE(int kind, void *data, Py_ssize_t index, Py_UCS4 value) + Py_UCS4 PyUnicode_READ(int kind, void *data, Py_ssize_t index) + Py_UCS4 PyUnicode_READ_CHAR(object o, Py_ssize_t index) + + unsigned int PyUnicode_KIND(object o) + void *PyUnicode_DATA(object o) + # Return the size of the object's internal buffer in bytes. o has # to be a PyUnicodeObject (not checked). Py_ssize_t PyUnicode_GET_DATA_SIZE(object o) @@ -35,6 +54,8 @@ cdef extern from *: # be a PyUnicodeObject (not checked). char* PyUnicode_AS_DATA(object o) + bint PyUnicode_IsIdentifier(object o) + # Return 1 or 0 depending on whether ch is a whitespace character. bint Py_UNICODE_ISSPACE(Py_UCS4 ch) @@ -65,6 +86,8 @@ cdef extern from *: # Return 1 or 0 depending on whether ch is an alphanumeric character. bint Py_UNICODE_ISALNUM(Py_UCS4 ch) + bint Py_UNICODE_ISPRINTABLE(Py_UCS4 ch) + # Return the character ch converted to lower case. # Used to return a Py_UNICODE value before Py3.3. Py_UCS4 Py_UNICODE_TOLOWER(Py_UCS4 ch) @@ -103,6 +126,26 @@ cdef extern from *: # when u is NULL. unicode PyUnicode_FromUnicode(Py_UNICODE *u, Py_ssize_t size) + # Similar to PyUnicode_FromUnicode(), but u points to UTF-8 encoded + # bytes + unicode PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) + + # Similar to PyUnicode_FromUnicode(), but u points to null-terminated + # UTF-8 encoded bytes. The size is determined with strlen(). + unicode PyUnicode_FromString(const char *u) + + unicode PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar) + unicode PyUnicode_FromKindAndData(int kind, const void *buffer, Py_ssize_t size) + unicode PyUnicode_FromFormat(const char *format, ...) + Py_ssize_t PyUnicode_GetLength(object unicode) except -1 + Py_ssize_t PyUnicode_CopyCharacters(object to, Py_ssize_t to_start, object from_, Py_ssize_t from_start, Py_ssize_t how_many) except -1 + Py_ssize_t PyUnicode_Fill(object unicode, Py_ssize_t start, Py_ssize_t length, Py_UCS4 fill_char) except -1 + int PyUnicode_WriteChar(object unicode, Py_ssize_t index, Py_UCS4 character) except -1 + Py_UCS4 PyUnicode_ReadChar(object unicode, Py_ssize_t index) except -1 + unicode PyUnicode_Substring(object str, Py_ssize_t start, Py_ssize_t end) + Py_UCS4 *PyUnicode_AsUCS4(object u, Py_UCS4 *buffer, Py_ssize_t buflen, int copy_null) except NULL + Py_UCS4 *PyUnicode_AsUCS4Copy(object u) except NULL + # Create a Unicode Object from the given Unicode code point ordinal. # # The ordinal must be in range(0x10000) on narrow Python builds diff --git a/Cython/Includes/libc/complex.pxd b/Cython/Includes/libc/complex.pxd new file mode 100644 index 000000000..7cd740cb8 --- /dev/null +++ b/Cython/Includes/libc/complex.pxd @@ -0,0 +1,35 @@ +cdef extern from "<complex.h>" nogil: + # Trigonometric functions. + double complex cacos(double complex z) + double complex casin(double complex z) + double complex catan(double complex z) + double complex ccos(double complex z) + double complex csin(double complex z) + double complex ctan(double complex z) + + # Hyperbolic functions. + double complex cacosh(double complex z) + double complex casinh(double complex z) + double complex catanh(double complex z) + double complex ccosh(double complex z) + double complex csinh(double complex z) + double complex ctanh(double complex z) + + # Exponential and logarithmic functions. + double complex cexp(double complex z) + double complex clog(double complex z) + double complex clog10(double complex z) + + # Power functions. + double complex cpow(double complex x, double complex y) + double complex csqrt(double complex z) + + # Absolute value, conjugates, and projection. + double cabs(double complex z) + double carg(double complex z) + double complex conj(double complex z) + double complex cproj(double complex z) + + # Decomposing complex values. + double cimag(double complex z) + double creal(double complex z) diff --git a/Cython/Includes/libc/errno.pxd b/Cython/Includes/libc/errno.pxd index 191d47b3d..9803a25fe 100644 --- a/Cython/Includes/libc/errno.pxd +++ b/Cython/Includes/libc/errno.pxd @@ -125,4 +125,3 @@ cdef extern from "<errno.h>" nogil: EDQUOT int errno - diff --git a/Cython/Includes/libc/math.pxd b/Cython/Includes/libc/math.pxd index b002670b2..4a9858c2a 100644 --- a/Cython/Includes/libc/math.pxd +++ b/Cython/Includes/libc/math.pxd @@ -23,81 +23,178 @@ cdef extern from "<math.h>" nogil: const float HUGE_VALF const long double HUGE_VALL + # All C99 functions in alphabetical order double acos(double x) + float acosf(float) + double acosh(double x) + float acoshf(float) + long double acoshl(long double) + long double acosl(long double) double asin(double x) + float asinf(float) + double asinh(double x) + float asinhf(float) + long double asinhl(long double) + long double asinl(long double) double atan(double x) double atan2(double y, double x) - double cos(double x) - double sin(double x) - double tan(double x) - - double cosh(double x) - double sinh(double x) - double tanh(double x) - double acosh(double x) - double asinh(double x) + float atan2f(float, float) + long double atan2l(long double, long double) + float atanf(float) double atanh(double x) - - double hypot(double x, double y) - - double exp(double x) - double exp2(double x) - double expm1(double x) - double log(double x) - double logb(double x) - double log2(double x) - double log10(double x) - double log1p(double x) - int ilogb(double x) - - double lgamma(double x) - double tgamma(double x) - - double frexp(double x, int* exponent) - double ldexp(double x, int exponent) - - double modf(double x, double* iptr) - double fmod(double x, double y) - double remainder(double x, double y) - double remquo(double x, double y, int *quot) - double pow(double x, double y) - double sqrt(double x) + float atanhf(float) + long double atanhl(long double) + long double atanl(long double) double cbrt(double x) - - double fabs(double x) + float cbrtf(float) + long double cbrtl(long double) double ceil(double x) - double floor(double x) - double trunc(double x) - double rint(double x) - double round(double x) - double nearbyint(double x) - double nextafter(double, double) - double nexttoward(double, long double) - - long long llrint(double) - long lrint(double) - long long llround(double) - long lround(double) - + float ceilf(float) + long double ceill(long double) double copysign(double, double) float copysignf(float, float) long double copysignl(long double, long double) - + double cos(double x) + float cosf(float) + double cosh(double x) + float coshf(float) + long double coshl(long double) + long double cosl(long double) double erf(double) - float erff(float) - long double erfl(long double) double erfc(double) float erfcf(float) long double erfcl(long double) - + float erff(float) + long double erfl(long double) + double exp(double x) + double exp2(double x) + float exp2f(float) + long double exp2l(long double) + float expf(float) + long double expl(long double) + double expm1(double x) + float expm1f(float) + long double expm1l(long double) + double fabs(double x) + float fabsf(float) + long double fabsl(long double) double fdim(double x, double y) + float fdimf(float, float) + long double fdiml(long double, long double) + double floor(double x) + float floorf(float) + long double floorl(long double) double fma(double x, double y, double z) + float fmaf(float, float, float) + long double fmal(long double, long double, long double) double fmax(double x, double y) + float fmaxf(float, float) + long double fmaxl(long double, long double) double fmin(double x, double y) + float fminf(float, float) + long double fminl(long double, long double) + double fmod(double x, double y) + float fmodf(float, float) + long double fmodl(long double, long double) + double frexp(double x, int* exponent) + float frexpf(float, int* exponent) + long double frexpl(long double, int*) + double hypot(double x, double y) + float hypotf(float, float) + long double hypotl(long double, long double) + int ilogb(double x) + int ilogbf(float) + int ilogbl(long double) + double ldexp(double x, int exponent) + float ldexpf(float, int exponent) + long double ldexpl(long double, int exponent) + double lgamma(double x) + float lgammaf(float) + long double lgammal(long double) + long long llrint(double) + long long llrintf(float) + long long llrintl(long double) + long long llround(double) + long long llroundf(float) + long long llroundl(long double) + double log(double x) + double log10(double x) + float log10f(float) + long double log10l(long double) + double log1p(double x) + float log1pf(float) + long double log1pl(long double) + double log2(double x) + float log2f(float) + long double log2l(long double) + double logb(double x) + float logbf(float) + long double logbl(long double) + float logf(float) + long double logl(long double) + long lrint(double) + long lrintf(float) + long lrintl(long double) + long lround(double) + long lroundf(float) + long lroundl(long double) + double modf(double x, double* iptr) + float modff(float, float* iptr) + long double modfl(long double, long double* iptr) + double nan(const char*) + float nanf(const char*) + long double nanl(const char*) + double nearbyint(double x) + float nearbyintf(float) + long double nearbyintl(long double) + double nextafter(double, double) + float nextafterf(float, float) + long double nextafterl(long double, long double) + double nexttoward(double, long double) + float nexttowardf(float, long double) + long double nexttowardl(long double, long double) + double pow(double x, double y) + float powf(float, float) + long double powl(long double, long double) + double remainder(double x, double y) + float remainderf(float, float) + long double remainderl(long double, long double) + double remquo(double x, double y, int* quot) + float remquof(float, float, int* quot) + long double remquol(long double, long double, int* quot) + double rint(double x) + float rintf(float) + long double rintl(long double) + double round(double x) + float roundf(float) + long double roundl(long double) double scalbln(double x, long n) + float scalblnf(float, long) + long double scalblnl(long double, long) double scalbn(double x, int n) - - double nan(const char*) + float scalbnf(float, int) + long double scalbnl(long double, int) + double sin(double x) + float sinf(float) + double sinh(double x) + float sinhf(float) + long double sinhl(long double) + long double sinl(long double) + double sqrt(double x) + float sqrtf(float) + long double sqrtl(long double) + double tan(double x) + float tanf(float) + double tanh(double x) + float tanhf(float) + long double tanhl(long double) + long double tanl(long double) + double tgamma(double x) + float tgammaf(float) + long double tgammal(long double) + double trunc(double x) + float truncf(float) + long double truncl(long double) int isinf(long double) # -1 / 0 / 1 bint isfinite(long double) diff --git a/Cython/Includes/libc/time.pxd b/Cython/Includes/libc/time.pxd index 3aa15a2ee..318212eea 100644 --- a/Cython/Includes/libc/time.pxd +++ b/Cython/Includes/libc/time.pxd @@ -1,4 +1,4 @@ -# http://en.wikipedia.org/wiki/C_date_and_time_functions +# https://en.wikipedia.org/wiki/C_date_and_time_functions from libc.stddef cimport wchar_t @@ -20,8 +20,9 @@ cdef extern from "<time.h>" nogil: int tm_wday int tm_yday int tm_isdst - char *tm_zone - long tm_gmtoff + # GNU specific extensions + #char *tm_zone + #long tm_gmtoff int daylight # global state long timezone diff --git a/Cython/Includes/libcpp/algorithm.pxd b/Cython/Includes/libcpp/algorithm.pxd index ec7c3835b..b961729b6 100644 --- a/Cython/Includes/libcpp/algorithm.pxd +++ b/Cython/Includes/libcpp/algorithm.pxd @@ -1,43 +1,320 @@ from libcpp cimport bool +from libcpp.utility cimport pair +from libc.stddef import ptrdiff_t cdef extern from "<algorithm>" namespace "std" nogil: - # Sorting and searching - bool binary_search[Iter, T](Iter first, Iter last, const T& value) - bool binary_search[Iter, T, Compare](Iter first, Iter last, const T& value, - Compare comp) + # Non-modifying sequence operations + bool all_of[Iter, Pred](Iter first, Iter last, Pred pred) except + + bool all_of[ExecutionPolicy, Iter, Pred](ExecutionPolicy&& policy, Iter first, Iter last, Pred pred) except + + bool any_of[Iter, Pred](Iter first, Iter last, Pred pred) except + + bool any_of[ExecutionPolicy, Iter, Pred](ExecutionPolicy&& policy, Iter first, Iter last, Pred pred) except + + bool none_of[Iter, Pred](Iter first, Iter last, Pred pred) except + + bool none_of[ExecutionPolicy, Iter, Pred](ExecutionPolicy&& policy, Iter first, Iter last, Pred pred) except + - Iter lower_bound[Iter, T](Iter first, Iter last, const T& value) - Iter lower_bound[Iter, T, Compare](Iter first, Iter last, const T& value, - Compare comp) + void for_each[Iter, UnaryFunction](Iter first, Iter last, UnaryFunction f) except + # actually returns f + void for_each[ExecutionPolicy, Iter, UnaryFunction](ExecutionPolicy&& policy, Iter first, Iter last, UnaryFunction f) except + # actually returns f - Iter upper_bound[Iter, T](Iter first, Iter last, const T& value) - Iter upper_bound[Iter, T, Compare](Iter first, Iter last, const T& value, - Compare comp) + ptrdiff_t count[Iter, T](Iter first, Iter last, const T& value) except + + ptrdiff_t count[ExecutionPolicy, Iter, T](ExecutionPolicy&& policy, Iter first, Iter last, const T& value) except + + ptrdiff_t count_if[Iter, Pred](Iter first, Iter last, Pred pred) except + + ptrdiff_t count_if[ExecutionPolicy, Iter, Pred](ExecutionPolicy&& policy, Iter first, Iter last, Pred pred) except + - void partial_sort[Iter](Iter first, Iter middle, Iter last) - void partial_sort[Iter, Compare](Iter first, Iter middle, Iter last, - Compare comp) + pair[Iter1, Iter2] mismatch[Iter1, Iter2]( + Iter1 first1, Iter1 last1, Iter2 first2) except + # other overloads are tricky + pair[Iter1, Iter2] mismatch[ExecutionPolicy, Iter1, Iter2]( + ExecutionPolicy&& policy, Iter1 first1, Iter1 last1, Iter2 first2) except + - void sort[Iter](Iter first, Iter last) - void sort[Iter, Compare](Iter first, Iter last, Compare comp) + Iter find[Iter, T](Iter first, Iter last, const T& value) except + + Iter find[ExecutionPolicy, Iter, T](ExecutionPolicy&& policy, Iter first, Iter last, const T& value) except + - # Removing duplicates - Iter unique[Iter](Iter first, Iter last) - Iter unique[Iter, BinaryPredicate](Iter first, Iter last, BinaryPredicate p) + Iter find_if[Iter, Pred](Iter first, Iter last, Pred pred) except + + Iter find_if[ExecutionPolicy, Iter, Pred](ExecutionPolicy&& policy, Iter first, Iter last, Pred pred) except + + Iter find_if_not[Iter, Pred](Iter first, Iter last, Pred pred) except + + Iter find_if_not[ExecutionPolicy, Iter, Pred](ExecutionPolicy&& policy, Iter first, Iter last, Pred pred) except + - # Binary heaps (priority queues) - void make_heap[Iter](Iter first, Iter last) - void make_heap[Iter, Compare](Iter first, Iter last, Compare comp) + Iter1 find_end[Iter1, Iter2](Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2) except + + Iter1 find_end[ExecutionPolicy, Iter1, Iter2](ExecutionPolicy&& policy, Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2) except + + Iter1 find_end[Iter1, Iter2, BinaryPred]( + Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, BinaryPred pred) except + + Iter1 find_end[ExecutionPolicy, Iter1, Iter2, BinaryPred]( + ExecutionPolicy&& policy, Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, BinaryPred pred) except + - void pop_heap[Iter](Iter first, Iter last) - void pop_heap[Iter, Compare](Iter first, Iter last, Compare comp) - void push_heap[Iter](Iter first, Iter last) - void push_heap[Iter, Compare](Iter first, Iter last, Compare comp) + Iter1 find_first_of[Iter1, Iter2](Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2) except + + Iter1 find_first_of[Iter1, Iter2, BinaryPred]( + Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, BinaryPred pred) except + + Iter1 find_first_of[ExecutionPolicy, Iter1, Iter2](ExecutionPolicy&& policy, Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2) except + + Iter1 find_first_of[ExecutionPolicy, Iter1, Iter2, BinaryPred]( + ExecutionPolicy&& policy, Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, BinaryPred pred) except + - void sort_heap[Iter](Iter first, Iter last) - void sort_heap[Iter, Compare](Iter first, Iter last, Compare comp) + Iter adjacent_find[Iter](Iter first, Iter last) except + + Iter adjacent_find[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter last) except + + Iter adjacent_find[Iter, BinaryPred](Iter first, Iter last, BinaryPred pred) except + + Iter adjacent_find[ExecutionPolicy, Iter, BinaryPred](ExecutionPolicy&& policy, Iter first, Iter last, BinaryPred pred) except + - # Copy - OutputIter copy[InputIter,OutputIter](InputIter,InputIter,OutputIter) + Iter1 search[Iter1, Iter2](Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2) except + + Iter1 search[ExecutionPolicy, Iter1, Iter2](ExecutionPolicy&& policy, Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2) except + + Iter1 search[Iter1, Iter2, BinaryPred]( + Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, BinaryPred pred) except + + Iter1 search[ExecutionPolicy, Iter1, Iter2, BinaryPred]( + ExecutionPolicy&& policy, Iter1 first1, Iter1 last1, Iter2 first2, Iter2 last2, BinaryPred pred) except + + Iter search_n[Iter, Size, T](Iter first1, Iter last1, Size count, const T& value) except + + Iter search_n[ExecutionPolicy, Iter, Size, T](ExecutionPolicy&& policy, Iter first1, Iter last1, Size count, const T& value) except + + Iter search_n[Iter, Size, T, BinaryPred]( + Iter first1, Iter last1, Size count, const T& value, BinaryPred pred) except + + Iter search_n[ExecutionPolicy, Iter, Size, T, BinaryPred]( + ExecutionPolicy&& policy, Iter first1, Iter last1, Size count, const T& value, BinaryPred pred) except + + + # Modifying sequence operations + OutputIt copy[InputIt, OutputIt](InputIt first, InputIt last, OutputIt d_first) except + + OutputIt copy[ExecutionPolicy, InputIt, OutputIt](ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first) except + + OutputIt copy_if[InputIt, OutputIt, Pred](InputIt first, InputIt last, OutputIt d_first, Pred pred) except + + OutputIt copy_if[ExecutionPolicy, InputIt, OutputIt, Pred](ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first, Pred pred) except + + OutputIt copy_n[InputIt, Size, OutputIt](InputIt first, Size count, OutputIt result) except + + OutputIt copy_n[ExecutionPolicy, InputIt, Size, OutputIt](ExecutionPolicy&& policy, InputIt first, Size count, OutputIt result) except + + Iter2 copy_backward[Iter1, Iter2](Iter1 first, Iter1 last, Iter2 d_last) except + + Iter2 copy_backward[ExecutionPolicy, Iter1, Iter2](ExecutionPolicy&& policy, Iter1 first, Iter1 last, Iter2 d_last) except + + + OutputIt move[InputIt, OutputIt](InputIt first, InputIt last, OutputIt d_first) except + + OutputIt move[ExecutionPolicy, InputIt, OutputIt](ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first) except + + Iter2 move_backward[Iter1, Iter2](Iter1 first, Iter1 last, Iter2 d_last) except + + Iter2 move_backward[ExecutionPolicy, Iter1, Iter2](ExecutionPolicy&& policy, Iter1 first, Iter1 last, Iter2 d_last) except + + + void fill[Iter, T](Iter first, Iter last, const T& value) except + + void fill[ExecutionPolicy, Iter, T](ExecutionPolicy&& policy, Iter first, Iter last, const T& value) except + + Iter fill_n[Iter, Size, T](Iter first, Size count, const T& value) except + + Iter fill_n[ExecutionPolicy, Iter, Size, T](ExecutionPolicy&& policy, Iter first, Size count, const T& value) except + + + OutputIt transform[InputIt, OutputIt, UnaryOp]( + InputIt first1, InputIt last1, OutputIt d_first, UnaryOp unary_op) except + + + # This overload is ambiguos with the next one. We just let C++ disambiguate from the arguments + # OutputIt transform[ExecutionPolicy, InputIt, OutputIt, UnaryOp]( + # ExecutionPolicy&& policy, InputIt first1, InputIt last1, OutputIt d_first, UnaryOp unary_op) except + + + OutputIt transform[InputIt1, InputIt2, OutputIt, BinaryOp]( + InputIt1 first1, InputIt1 last1, InputIt2 first2, OutputIt d_first, BinaryOp binary_op) except + + + OutputIt transform[ExecutionPolicy, InputIt1, InputIt2, OutputIt, BinaryOp]( + ExecutionPolicy&& policy, InputIt1 first1, InputIt1 last1, InputIt2 first2, OutputIt d_first, BinaryOp binary_op) except + + + void generate[Iter, Generator](Iter first, Iter last, Generator g) except + + void generate[ExecutionPolicy, Iter, Generator](ExecutionPolicy&& policy, Iter first, Iter last, Generator g) except + + void generate_n[Iter, Size, Generator](Iter first, Size count, Generator g) except + + void generate_n[ExecutionPolicy, Iter, Size, Generator](ExecutionPolicy&& policy, Iter first, Size count, Generator g) except + + + Iter remove[Iter, T](Iter first, Iter last, const T& value) except + + Iter remove[ExecutionPolicy, Iter, T](ExecutionPolicy&& policy, Iter first, Iter last, const T& value) except + + Iter remove_if[Iter, UnaryPred](Iter first, Iter last, UnaryPred pred) except + + Iter remove_if[ExecutionPolicy, Iter, UnaryPred](ExecutionPolicy&& policy, Iter first, Iter last, UnaryPred pred) except + + OutputIt remove_copy[InputIt, OutputIt, T](InputIt first, InputIt last, OutputIt d_first, const T& value) except + + OutputIt remove_copy[ExecutionPolicy, InputIt, OutputIt, T](ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first, const T& value) except + + OutputIt remove_copy_if[InputIt, OutputIt, UnaryPred]( + InputIt first, InputIt last, OutputIt d_first, UnaryPred pred) except + + OutputIt remove_copy_if[ExecutionPolicy, InputIt, OutputIt, UnaryPred]( + ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first, UnaryPred pred) except + + + void replace[Iter, T](Iter first, Iter last, const T& old_value, const T& new_value) except + + void replace[ExecutionPolicy, Iter, T](ExecutionPolicy&& policy, Iter first, Iter last, const T& old_value, const T& new_value) except + + void replace_if[Iter, UnaryPred, T](Iter first, Iter last, UnaryPred pred, const T& new_value) except + + OutputIt replace_copy[InputIt, OutputIt, T]( + InputIt first, InputIt last, OutputIt d_first, const T& old_value, const T& new_value) except + + void replace_if[ExecutionPolicy, Iter, UnaryPred, T](ExecutionPolicy&& policy, Iter first, Iter last, UnaryPred pred, const T& new_value) except + + + OutputIt replace_copy[ExecutionPolicy, InputIt, OutputIt, T]( + ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first, const T& old_value, const T& new_value) except + + OutputIt replace_copy_if[InputIt, OutputIt, UnaryPred, T]( + InputIt first, InputIt last, OutputIt d_first, UnaryPred pred, const T& new_value) except + + OutputIt replace_copy_if[ExecutionPolicy, InputIt, OutputIt, UnaryPred, T]( + ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first, UnaryPred pred, const T& new_value) except + + + void swap[T](T& a, T& b) except + # array overload also works + Iter2 swap_ranges[Iter1, Iter2](Iter1 first1, Iter1 last1, Iter2 first2) except + + void iter_swap[Iter](Iter a, Iter b) except + + + void reverse[Iter](Iter first, Iter last) except + + void reverse[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter last) except + + OutputIt reverse_copy[InputIt, OutputIt](InputIt first, InputIt last, OutputIt d_first) except + + OutputIt reverse_copy[ExecutionPolicy, InputIt, OutputIt](ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first) except + + + Iter rotate[Iter](Iter first, Iter n_first, Iter last) except + + Iter rotate[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter n_first, Iter last) except + + OutputIt rotate_copy[InputIt, OutputIt](InputIt first, InputIt n_first, InputIt last, OutputIt d_first) except + + OutputIt rotate_copy[ExecutionPolicy, InputIt, OutputIt](ExecutionPolicy&& policy, InputIt first, InputIt n_first, InputIt last, OutputIt d_first) except + + + Iter unique[Iter](Iter first, Iter last) except + + Iter unique[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter last) except + + Iter unique[Iter, BinaryPred](Iter first, Iter last, BinaryPred p) except + + Iter unique[ExecutionPolicy, Iter, BinaryPred](ExecutionPolicy&& policy, Iter first, Iter last, BinaryPred p) except + + OutputIt unique_copy[InputIt, OutputIt](InputIt first, InputIt last, OutputIt d_first) except + + OutputIt unique_copy[ExecutionPolicy, InputIt, OutputIt](ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first) except + + OutputIt unique_copy[InputIt, OutputIt, BinaryPred]( + InputIt first, InputIt last, OutputIt d_first, BinaryPred pred) except + + OutputIt unique_copy[ExecutionPolicy, InputIt, OutputIt, BinaryPred]( + ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first, BinaryPred pred) except + + + SampleIt sample[PopulationIt, SampleIt, Distance, URBG](PopulationIt first, PopulationIt last, SampleIt out, Distance n, URBG&& g) except + + + # Partitioning operations + bool is_partitioned[Iter, Pred](Iter first, Iter last, Pred p) except + + bool is_partitioned[ExecutionPolicy, Iter, Pred](ExecutionPolicy&& policy, Iter first, Iter last, Pred p) except + + Iter partition[Iter, Pred](Iter first, Iter last, Pred p) except + + Iter partition[ExecutionPolicy, Iter, Pred](ExecutionPolicy&& policy, Iter first, Iter last, Pred p) except + + pair[OutputIt1, OutputIt2] partition_copy[InputIt, OutputIt1, OutputIt2, Pred]( + InputIt first, InputIt last, OutputIt1 d_first_true, OutputIt2 d_first_false, Pred p) except + + pair[OutputIt1, OutputIt2] partition_copy[ExecutionPolicy, InputIt, OutputIt1, OutputIt2, Pred]( + ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt1 d_first_true, OutputIt2 d_first_false, Pred p) except + + + Iter stable_partition[Iter, Pred](Iter first, Iter last, Pred p) except + + Iter stable_partition[ExecutionPolicy, Iter, Pred](ExecutionPolicy&& policy, Iter first, Iter last, Pred p) except + + Iter partition_point[Iter, Pred](Iter first, Iter last, Pred p) except + + Iter partition_point[ExecutionPolicy, Iter, Pred](ExecutionPolicy&& policy, Iter first, Iter last, Pred p) except + + + # Sorting operations + bool is_sorted[Iter](Iter first, Iter last) except + + bool is_sorted[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter last) except + + bool is_sorted[Iter, Compare](Iter first, Iter last, Compare comp) except + + bool is_sorted[ExecutionPolicy, Iter, Compare](ExecutionPolicy&& policy, Iter first, Iter last, Compare comp) except + + + Iter is_sorted_until[Iter](Iter first, Iter last) except + + Iter is_sorted_until[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter last) except + + Iter is_sorted_until[Iter, Compare](Iter first, Iter last, Compare comp) except + + Iter is_sorted_until[ExecutionPolicy, Iter, Compare](ExecutionPolicy&& policy, Iter first, Iter last, Compare comp) except + + + void sort[Iter](Iter first, Iter last) except + + void sort[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter last) except + + void sort[Iter, Compare](Iter first, Iter last, Compare comp) except + + void sort[ExecutionPolicy, Iter, Compare](ExecutionPolicy&& policy, Iter first, Iter last, Compare comp) except + + + void partial_sort[Iter](Iter first, Iter middle, Iter last) except + + void partial_sort[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter middle, Iter last) except + + void partial_sort[Iter, Compare](Iter first, Iter middle, Iter last, Compare comp) except + + void partial_sort[ExecutionPolicy, Iter, Compare](ExecutionPolicy&& policy, Iter first, Iter middle, Iter last, Compare comp) except + + + OutputIt partial_sort_copy[InputIt, OutputIt]( + InputIt first, InputIt last, OutputIt d_first, OutputIt d_last) except + + OutputIt partial_sort_copy[ExecutionPolicy, InputIt, OutputIt]( + ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first, OutputIt d_last) except + + OutputIt partial_sort_copy[InputIt, OutputIt, Compare]( + InputIt first, InputIt last, OutputIt d_first, OutputIt d_last, Compare comp) except + + OutputIt partial_sort_copy[ExecutionPolicy, InputIt, OutputIt, Compare]( + ExecutionPolicy&& policy, InputIt first, InputIt last, OutputIt d_first, OutputIt d_last, Compare comp) except + + + void stable_sort[Iter](Iter first, Iter last) except + + void stable_sort[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter last) except + + void stable_sort[Iter, Compare](Iter first, Iter last, Compare comp) except + + void stable_sort[ExecutionPolicy, Iter, Compare](ExecutionPolicy&& policy, Iter first, Iter last, Compare comp) except + + + void nth_element[Iter](Iter first, Iter nth, Iter last) except + + void nth_element[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter nth, Iter last) except + + void nth_element[Iter, Compare](Iter first, Iter nth, Iter last, Compare comp) except + + void nth_element[ExecutionPolicy, Iter, Compare](ExecutionPolicy&& policy, Iter first, Iter nth, Iter last, Compare comp) except + + + # Binary search operations (on sorted ranges) + Iter lower_bound[Iter, T](Iter first, Iter last, const T& value) except + + Iter lower_bound[ExecutionPolicy, Iter, T](ExecutionPolicy&& policy, Iter first, Iter last, const T& value) except + + Iter lower_bound[Iter, T, Compare](Iter first, Iter last, const T& value, Compare comp) except + + Iter lower_bound[ExecutionPolicy, Iter, T, Compare](ExecutionPolicy&& policy, Iter first, Iter last, const T& value, Compare comp) except + + + Iter upper_bound[Iter, T](Iter first, Iter last, const T& value) except + + Iter upper_bound[ExecutionPolicy, Iter, T](ExecutionPolicy&& policy, Iter first, Iter last, const T& value) except + + Iter upper_bound[Iter, T, Compare](Iter first, Iter last, const T& value, Compare comp) except + + Iter upper_bound[ExecutionPolicy, Iter, T, Compare](ExecutionPolicy&& policy, Iter first, Iter last, const T& value, Compare comp) except + + + bool binary_search[Iter, T](Iter first, Iter last, const T& value) except + + bool binary_search[ExecutionPolicy, Iter, T](ExecutionPolicy&& policy, Iter first, Iter last, const T& value) except + + bool binary_search[Iter, T, Compare](Iter first, Iter last, const T& value, Compare comp) except + + bool binary_search[ExecutionPolicy, Iter, T, Compare](ExecutionPolicy&& policy, Iter first, Iter last, const T& value, Compare comp) except + + + # Other operations on sorted ranges + OutputIt merge[InputIt1, InputIt2, OutputIt]( + InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2, OutputIt out) except + + OutputIt merge[InputIt1, InputIt2, OutputIt, Compare]( + InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2, OutputIt out, Compare comp) except + + + void inplace_merge[BidirIt](BidirIt first, BidirIt middle, BidirIt last) except + + void inplace_merge[BidirIt, Compare](BidirIt first, BidirIt middle, BidirIt last, Compare comp) except + + + # Set operations (on sorted ranges) + bool includes[InputIt1, InputIt2]( + InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2) except + + + bool includes[InputIt1, InputIt2, Compare]( + InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2, Compare comp) except + + + OutputIt set_difference[InputIt1, InputIt2, OutputIt]( + InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2, OutputIt out) except + + + OutputIt set_difference[InputIt1, InputIt2, OutputIt, Compare]( + InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2, + OutputIt out, Compare comp) except + + + OutputIt set_intersection[InputIt1, InputIt2, OutputIt]( + InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2, OutputIt out) except + + + OutputIt set_intersection[InputIt1, InputIt2, OutputIt, Compare]( + InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2, OutputIt out, Compare comp) except + + + OutputIt set_symmetric_difference[InputIt1, InputIt2, OutputIt]( + InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2, OutputIt out) except + + + OutputIt set_symmetric_difference[InputIt1, InputIt2, OutputIt, Compare]( + InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2, OutputIt out, Compare comp) except + + + OutputIt set_union[InputIt1, InputIt2, OutputIt]( + InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2, OutputIt out) except + + + OutputIt set_union[InputIt1, InputIt2, OutputIt, Compare]( + InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2, OutputIt out, Compare comp) except + + + # Heap operations + void make_heap[Iter](Iter first, Iter last) except + + void make_heap[Iter, Compare](Iter first, Iter last, Compare comp) except + + + void push_heap[Iter](Iter first, Iter last) except + + void push_heap[Iter, Compare](Iter first, Iter last, Compare comp) except + + + void pop_heap[Iter](Iter first, Iter last) except + + void pop_heap[Iter, Compare](Iter first, Iter last, Compare comp) except + + + void sort_heap[Iter](Iter first, Iter last) except + + void sort_heap[Iter, Compare](Iter first, Iter last, Compare comp) except + + + # Minimum/maximum operations + Iter min_element[Iter](Iter first, Iter last) except + + Iter min_element[Iter, Compare](Iter first, Iter last, Compare comp) except + + Iter min_element[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter last) except + + Iter max_element[Iter](Iter first, Iter last) except + + Iter max_element[Iter, Compare](Iter first, Iter last, Compare comp) except + + Iter max_element[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter last) except + + pair[T, T] minmax[T](const T& a, const T& b) except + + pair[T, T] minmax[T, Compare](const T& a, const T& b, Compare comp) except + + pair[Iter, Iter] minmax_element[Iter](Iter first, Iter last) except + + pair[Iter, Iter] minmax_element[Iter, Compare](Iter first, Iter last, Compare comp) except + + pair[Iter, Iter] minmax_element[ExecutionPolicy, Iter](ExecutionPolicy&& policy, Iter first, Iter last) except + + const T& clamp[T](const T& v, const T& lo, const T& hi) except + + const T& clamp[T, Compare](const T& v, const T& lo, const T& hi, Compare comp) except + + + # Comparison operations + bool equal[InputIt1, InputIt2](InputIt1 first1, InputIt1 last1, InputIt2 first2) except + + bool equal[InputIt1, InputIt2, BinPred](InputIt1 first1, InputIt1 last1, InputIt2 first2, BinPred pred) except + + # ambiguous with previous overload + #bool equal[InputIt1, InputIt2](InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2) except + + bool equal[InputIt1, InputIt2, BinPred](InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2, BinPred pred) except + + + bool lexicographical_compare[InputIt1, InputIt2](InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2) except + + # ambiguous with next overload + #bool lexicographical_compare[InputIt1, InputIt2, ExecutionPolicy](ExecutionPolicy&& policy, InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2) except + + bool lexicographical_compare[InputIt1, InputIt2, Compare](InputIt1 first1, InputIt1 last1, InputIt2 first2, InputIt2 last2, Compare comp) except + + + # Permutation operations + bool is_permutation[ForwardIt1, ForwardIt2](ForwardIt1 first1, ForwardIt1 last1, ForwardIt2 first2) except + + bool is_permutation[ForwardIt1, ForwardIt2, BinaryPred](ForwardIt1 first1, ForwardIt1 last1, ForwardIt2 first2, BinaryPred p) except + + # ambiguous with previous overload + #bool is_permutation[ForwardIt1, ForwardIt2](ForwardIt1 first1, ForwardIt1 last1, ForwardIt2 first2, ForwardIt2 last2) except + + bool is_permutation[ForwardIt1, ForwardIt2, BinaryPred](ForwardIt1 first1, ForwardIt1 last1, ForwardIt2 first2, ForwardIt2 last2, BinaryPred p) except + + bool next_permutation[BidirIt](BidirIt first, BidirIt last) except + + bool next_permutation[BidirIt, Compare](BidirIt first, BidirIt last, Compare comp) except + + bool prev_permutation[BidirIt](BidirIt first, BidirIt last) except + + bool prev_permutation[BidirIt, Compare](BidirIt first, BidirIt last, Compare comp) except + diff --git a/Cython/Includes/libcpp/any.pxd b/Cython/Includes/libcpp/any.pxd new file mode 100644 index 000000000..7c0d000cd --- /dev/null +++ b/Cython/Includes/libcpp/any.pxd @@ -0,0 +1,16 @@ +from libcpp cimport bool +from libcpp.typeinfo cimport type_info + +cdef extern from "<any>" namespace "std" nogil: + cdef cppclass any: + any() + any(any&) except + + void reset() + bool has_value() + type_info& type() + T& emplace[T](...) except + + void swap(any&) + any& operator=(any&) except + + any& operator=[U](U&) except + + + cdef T any_cast[T](any&) except + diff --git a/Cython/Includes/libcpp/atomic.pxd b/Cython/Includes/libcpp/atomic.pxd new file mode 100644 index 000000000..89a8e6ffc --- /dev/null +++ b/Cython/Includes/libcpp/atomic.pxd @@ -0,0 +1,59 @@ + +cdef extern from "<atomic>" namespace "std" nogil: + + cdef enum memory_order: + memory_order_relaxed + memory_order_consume + memory_order_acquire + memory_order_release + memory_order_acq_rel + memory_order_seq_cst + + cdef cppclass atomic[T]: + atomic() + atomic(T) + + bint is_lock_free() + void store(T) + void store(T, memory_order) + T load() + T load(memory_order) + T exchange(T) + T exchange(T, memory_order) + + bint compare_exchange_weak(T&, T, memory_order, memory_order) + bint compare_exchange_weak(T&, T, memory_order) + bint compare_exchange_weak(T&, T) + bint compare_exchange_strong(T&, T, memory_order, memory_order) + bint compare_exchange_strong(T&, T, memory_order) + bint compare_exchange_strong(T&, T) + + T fetch_add(T, memory_order) + T fetch_add(T) + T fetch_sub(T, memory_order) + T fetch_sub(T) + T fetch_and(T, memory_order) + T fetch_and(T) + T fetch_or(T, memory_order) + T fetch_or(T) + T fetch_xor(T, memory_order) + T fetch_xor(T) + + T operator++() + T operator++(int) + T operator--() + T operator--(int) + + # modify-in-place operators not yet supported by Cython: + # T operator+=(T) + # T operator-=(T) + # T operator&=(T) + # T operator|=(T) + # T operator^=(T) + + bint operator==(atomic[T]&, atomic[T]&) + bint operator==(atomic[T]&, T&) + bint operator==(T&, atomic[T]&) + bint operator!=(atomic[T]&, atomic[T]&) + bint operator!=(atomic[T]&, T&) + bint operator!=(T&, atomic[T]&) diff --git a/Cython/Includes/libcpp/bit.pxd b/Cython/Includes/libcpp/bit.pxd new file mode 100644 index 000000000..3b6ffed05 --- /dev/null +++ b/Cython/Includes/libcpp/bit.pxd @@ -0,0 +1,29 @@ +cdef extern from "<bit>" namespace "std" nogil: + # bit_cast (gcc >= 11.0, clang >= 14.0) + cdef To bit_cast[To, From](From&) + + # byteswap (C++23) + #cdef T byteswap[T](T) + + # integral powers of 2 (gcc >= 10.0, clang >= 12.0) + cdef bint has_single_bit[T](T) + cdef T bit_ceil[T](T) + cdef T bit_floor[T](T) + cdef int bit_width[T](T) + + # rotating (gcc >= 9.0, clang >= 9.0) + cdef T rotl[T](T, int shift) + cdef T rotr[T](T, int shift) + + # counting (gcc >= 9.0, clang >= 9.0) + cdef int countl_zero[T](T) + cdef int countl_one[T](T) + cdef int countr_zero[T](T) + cdef int countr_one[T](T) + cdef int popcount[T](T) + + # endian + cpdef enum class endian(int): + little, + big, + native diff --git a/Cython/Includes/libcpp/cmath.pxd b/Cython/Includes/libcpp/cmath.pxd new file mode 100644 index 000000000..edc198383 --- /dev/null +++ b/Cython/Includes/libcpp/cmath.pxd @@ -0,0 +1,518 @@ + +cdef extern from "<cmath>" namespace "std" nogil: + # all C99 functions + float acos(float x) except + + double acos(double x) except + + long double acos(long double x) except + + float acosf(float x) except + + long double acosl(long double x) except + + + float asin(float x) except + + double asin(double x) except + + long double asin(long double x) except + + float asinf(float x) except + + long double asinl(long double x) except + + + float atan(float x) except + + double atan(double x) except + + long double atan(long double x) except + + float atanf(float x) except + + long double atanl(long double x) except + + + float atan2(float y, float x) except + + double atan2(double y, double x) except + + long double atan2(long double y, long double x) except + + float atan2f(float y, float x) except + + long double atan2l(long double y, long double x) except + + + float cos(float x) except + + double cos(double x) except + + long double cos(long double x) except + + float cosf(float x) except + + long double cosl(long double x) except + + + float sin(float x) except + + double sin(double x) except + + long double sin(long double x) except + + float sinf(float x) except + + long double sinl(long double x) except + + + float tan(float x) except + + double tan(double x) except + + long double tan(long double x) except + + float tanf(float x) except + + long double tanl(long double x) except + + + float acosh(float x) except + + double acosh(double x) except + + long double acosh(long double x) except + + float acoshf(float x) except + + long double acoshl(long double x) except + + + float asinh(float x) except + + double asinh(double x) except + + long double asinh(long double x) except + + float asinhf(float x) except + + long double asinhl(long double x) except + + + float atanh(float x) except + + double atanh(double x) except + + long double atanh(long double x) except + + float atanhf(float x) except + + long double atanhl(long double x) except + + + float cosh(float x) except + + double cosh(double x) except + + long double cosh(long double x) except + + float coshf(float x) except + + long double coshl(long double x) except + + + float sinh(float x) except + + double sinh(double x) except + + long double sinh(long double x) except + + float sinhf(float x) except + + long double sinhl(long double x) except + + + float tanh(float x) except + + double tanh(double x) except + + long double tanh(long double x) except + + float tanhf(float x) except + + long double tanhl(long double x) except + + + float exp(float x) except + + double exp(double x) except + + long double exp(long double x) except + + float expf(float x) except + + long double expl(long double x) except + + + float exp2(float x) except + + double exp2(double x) except + + long double exp2(long double x) except + + float exp2f(float x) except + + long double exp2l(long double x) except + + + float expm1(float x) except + + double expm1(double x) except + + long double expm1(long double x) except + + float expm1f(float x) except + + long double expm1l(long double x) except + + + float frexp(float value, int* exp) except + + double frexp(double value, int* exp) except + + long double frexp(long double value, int* exp) except + + float frexpf(float value, int* exp) except + + long double frexpl(long double value, int* exp) except + + + int ilogb(float x) except + + int ilogb(double x) except + + int ilogb(long double x) except + + int ilogbf(float x) except + + int ilogbl(long double x) except + + + float ldexp(float x, int exp) except + + double ldexp(double x, int exp) except + + long double ldexp(long double x, int exp) except + + float ldexpf(float x, int exp) except + + long double ldexpl(long double x, int exp) except + + + float log(float x) except + + double log(double x) except + + long double log(long double x) except + + float logf(float x) except + + long double logl(long double x) except + + + float log10(float x) except + + double log10(double x) except + + long double log10(long double x) except + + float log10f(float x) except + + long double log10l(long double x) except + + + float log1p(float x) except + + double log1p(double x) except + + long double log1p(long double x) except + + float log1pf(float x) except + + long double log1pl(long double x) except + + + float log2(float x) except + + double log2(double x) except + + long double log2(long double x) except + + float log2f(float x) except + + long double log2l(long double x) except + + + float logb(float x) except + + double logb(double x) except + + long double logb(long double x) except + + float logbf(float x) except + + long double logbl(long double x) except + + + float modf(float value, float* iptr) except + + double modf(double value, double* iptr) except + + long double modf(long double value, long double* iptr) except + + float modff(float value, float* iptr) except + + long double modfl(long double value, long double* iptr) except + + + float scalbn(float x, int n) except + + double scalbn(double x, int n) except + + long double scalbn(long double x, int n) except + + float scalbnf(float x, int n) except + + long double scalbnl(long double x, int n) except + + + float scalbln(float x, long int n) except + + double scalbln(double x, long int n) except + + long double scalbln(long double x, long int n) except + + float scalblnf(float x, long int n) except + + long double scalblnl(long double x, long int n) except + + + float cbrt(float x) except + + double cbrt(double x) except + + long double cbrt(long double x) except + + float cbrtf(float x) except + + long double cbrtl(long double x) except + + + # absolute values + int abs(int j) except + + long int abs(long int j) except + + long long int abs(long long int j) except + + float abs(float j) except + + double abs(double j) except + + long double abs(long double j) except + + + float fabs(float x) except + + double fabs(double x) except + + long double fabs(long double x) except + + float fabsf(float x) except + + long double fabsl(long double x) except + + + float hypot(float x, float y) except + + double hypot(double x, double y) except + + long double hypot(long double x, long double y) except + + float hypotf(float x, float y) except + + long double hypotl(long double x, long double y) except + + + # C++17 three-dimensional hypotenuse + float hypot(float x, float y, float z) except + + double hypot(double x, double y, double z) except + + long double hypot(long double x, long double y, long double z) except + + + float pow(float x, float y) except + + double pow(double x, double y) except + + long double pow(long double x, long double y) except + + float powf(float x, float y) except + + long double powl(long double x, long double y) except + + + float sqrt(float x) except + + double sqrt(double x) except + + long double sqrt(long double x) except + + float sqrtf(float x) except + + long double sqrtl(long double x) except + + + float erf(float x) except + + double erf(double x) except + + long double erf(long double x) except + + float erff(float x) except + + long double erfl(long double x) except + + + float erfc(float x) except + + double erfc(double x) except + + long double erfc(long double x) except + + float erfcf(float x) except + + long double erfcl(long double x) except + + + float lgamma(float x) except + + double lgamma(double x) except + + long double lgamma(long double x) except + + float lgammaf(float x) except + + long double lgammal(long double x) except + + + float tgamma(float x) except + + double tgamma(double x) except + + long double tgamma(long double x) except + + float tgammaf(float x) except + + long double tgammal(long double x) except + + + float ceil(float x) except + + double ceil(double x) except + + long double ceil(long double x) except + + float ceilf(float x) except + + long double ceill(long double x) except + + + float floor(float x) except + + double floor(double x) except + + long double floor(long double x) except + + float floorf(float x) except + + long double floorl(long double x) except + + + float nearbyint(float x) except + + double nearbyint(double x) except + + long double nearbyint(long double x) except + + float nearbyintf(float x) except + + long double nearbyintl(long double x) except + + + float rint(float x) except + + double rint(double x) except + + long double rint(long double x) except + + float rintf(float x) except + + long double rintl(long double x) except + + + long int lrint(float x) except + + long int lrint(double x) except + + long int lrint(long double x) except + + long int lrintf(float x) except + + long int lrintl(long double x) except + + + long long int llrint(float x) except + + long long int llrint(double x) except + + long long int llrint(long double x) except + + long long int llrintf(float x) except + + long long int llrintl(long double x) except + + + float round(float x) except + + double round(double x) except + + long double round(long double x) except + + float roundf(float x) except + + long double roundl(long double x) except + + + long int lround(float x) except + + long int lround(double x) except + + long int lround(long double x) except + + long int lroundf(float x) except + + long int lroundl(long double x) except + + + long long int llround(float x) except + + long long int llround(double x) except + + long long int llround(long double x) except + + long long int llroundf(float x) except + + long long int llroundl(long double x) except + + + float trunc(float x) except + + double trunc(double x) except + + long double trunc(long double x) except + + float truncf(float x) except + + long double truncl(long double x) except + + + float fmod(float x, float y) except + + double fmod(double x, double y) except + + long double fmod(long double x, long double y) except + + float fmodf(float x, float y) except + + long double fmodl(long double x, long double y) except + + + float remainder(float x, float y) except + + double remainder(double x, double y) except + + long double remainder(long double x, long double y) except + + float remainderf(float x, float y) except + + long double remainderl(long double x, long double y) except + + + float remquo(float x, float y, int* quo) except + + double remquo(double x, double y, int* quo) except + + long double remquo(long double x, long double y, int* quo) except + + float remquof(float x, float y, int* quo) except + + long double remquol(long double x, long double y, int* quo) except + + + float copysign(float x, float y) except + + double copysign(double x, double y) except + + long double copysign(long double x, long double y) except + + float copysignf(float x, float y) except + + long double copysignl(long double x, long double y) except + + + double nan(const char* tagp) except + + float nanf(const char* tagp) except + + long double nanl(const char* tagp) except + + + float nextafter(float x, float y) except + + double nextafter(double x, double y) except + + long double nextafter(long double x, long double y) except + + float nextafterf(float x, float y) except + + long double nextafterl(long double x, long double y) except + + + float nexttoward(float x, long double y) except + + double nexttoward(double x, long double y) except + + long double nexttoward(long double x, long double y) except + + float nexttowardf(float x, long double y) except + + long double nexttowardl(long double x, long double y) except + + + float fdim(float x, float y) except + + double fdim(double x, double y) except + + long double fdim(long double x, long double y) except + + float fdimf(float x, float y) except + + long double fdiml(long double x, long double y) except + + + float fmax(float x, float y) except + + double fmax(double x, double y) except + + long double fmax(long double x, long double y) except + + float fmaxf(float x, float y) except + + long double fmaxl(long double x, long double y) except + + + float fmin(float x, float y) except + + double fmin(double x, double y) except + + long double fmin(long double x, long double y) except + + float fminf(float x, float y) except + + long double fminl(long double x, long double y) except + + + float fma(float x, float y, float z) except + + double fma(double x, double y, double z) except + + long double fma(long double x, long double y, long double z) except + + float fmaf(float x, float y, float z) except + + long double fmal(long double x, long double y, long double z) except + + + # C++20 linear interpolation + float lerp(float a, float b, float t) + double lerp(double a, double b, double t) + long double lerp(long double a, long double b, long double t) + + # classification / comparison functions + int fpclassify(float x) except + + int fpclassify(double x) except + + int fpclassify(long double x) except + + + bint isfinite(float x) except + + bint isfinite(double x) except + + bint isfinite(long double x) except + + + bint isinf(float x) except + + bint isinf(double x) except + + bint isinf(long double x) except + + + bint isnan(float x) except + + bint isnan(double x) except + + bint isnan(long double x) except + + + bint isnormal(float x) except + + bint isnormal(double x) except + + bint isnormal(long double x) except + + + bint signbit(float x) except + + bint signbit(double x) except + + bint signbit(long double x) except + + + bint isgreater(float x, float y) except + + bint isgreater(double x, double y) except + + bint isgreater(long double x, long double y) except + + + bint isgreaterequal(float x, float y) except + + bint isgreaterequal(double x, double y) except + + bint isgreaterequal(long double x, long double y) except + + + bint isless(float x, float y) except + + bint isless(double x, double y) except + + bint isless(long double x, long double y) except + + + bint islessequal(float x, float y) except + + bint islessequal(double x, double y) except + + bint islessequal(long double x, long double y) except + + + bint islessgreater(float x, float y) except + + bint islessgreater(double x, double y) except + + bint islessgreater(long double x, long double y) except + + + bint isunordered(float x, float y) except + + bint isunordered(double x, double y) except + + bint isunordered(long double x, long double y) except + + + # C++17 mathematical special functions + + # associated Laguerre polynomials + double assoc_laguerre(unsigned int n, unsigned int m, double x) except + + float assoc_laguerref(unsigned int n, unsigned int m, float x) except + + long double assoc_laguerrel(unsigned int n, unsigned int m, long double x) except + + + # associated Legendre functions + double assoc_legendre(unsigned int l, unsigned int m, double x) except + + float assoc_legendref(unsigned int l, unsigned int m, float x) except + + long double assoc_legendrel(unsigned int l, unsigned int m, long double x) except + + + # beta function + double beta(double x, double y) except + + float betaf(float x, float y) except + + long double betal(long double x, long double y) except + + + # complete elliptic integral of the first kind + double comp_ellint_1(double k) except + + float comp_ellint_1f(float k) except + + long double comp_ellint_1l(long double k) except + + + # complete elliptic integral of the second kind + double comp_ellint_2(double k) except + + float comp_ellint_2f(float k) except + + long double comp_ellint_2l(long double k) except + + + # complete elliptic integral of the third kind + double comp_ellint_3(double k, double nu) except + + float comp_ellint_3f(float k, float nu) except + + long double comp_ellint_3l(long double k, long double nu) except + + + # regular modified cylindrical Bessel functions + double cyl_bessel_i(double nu, double x) except + + float cyl_bessel_if(float nu, float x) except + + long double cyl_bessel_il(long double nu, long double x) except + + + # cylindrical Bessel functions of the first kind + double cyl_bessel_j(double nu, double x) except + + float cyl_bessel_jf(float nu, float x) except + + long double cyl_bessel_jl(long double nu, long double x) except + + + # irregular modified cylindrical Bessel functions + double cyl_bessel_k(double nu, double x) except + + float cyl_bessel_kf(float nu, float x) except + + long double cyl_bessel_kl(long double nu, long double x) except + + + # cylindrical Neumann functions + # cylindrical Bessel functions of the second kind + double cyl_neumann(double nu, double x) except + + float cyl_neumannf(float nu, float x) except + + long double cyl_neumannl(long double nu, long double x) except + + + # incomplete elliptic integral of the first kind + double ellint_1(double k, double phi) except + + float ellint_1f(float k, float phi) except + + long double ellint_1l(long double k, long double phi) except + + + # incomplete elliptic integral of the second kind + double ellint_2(double k, double phi) except + + float ellint_2f(float k, float phi) except + + long double ellint_2l(long double k, long double phi) except + + + # incomplete elliptic integral of the third kind + double ellint_3(double k, double nu, double phi) except + + float ellint_3f(float k, float nu, float phi) except + + long double ellint_3l(long double k, long double nu, long double phi) except + + + # exponential integral + double expint(double x) except + + float expintf(float x) except + + long double expintl(long double x) except + + + # Hermite polynomials + double hermite(unsigned int n, double x) except + + float hermitef(unsigned int n, float x) except + + long double hermitel(unsigned int n, long double x) except + + + # Laguerre polynomials + double laguerre(unsigned int n, double x) except + + float laguerref(unsigned int n, float x) except + + long double laguerrel(unsigned int n, long double x) except + + + # Legendre polynomials + double legendre(unsigned int l, double x) except + + float legendref(unsigned int l, float x) except + + long double legendrel(unsigned int l, long double x) except + + + # Riemann zeta function + double riemann_zeta(double x) except + + float riemann_zetaf(float x) except + + long double riemann_zetal(long double x) except + + + # spherical Bessel functions of the first kind + double sph_bessel(unsigned int n, double x) except + + float sph_besself(unsigned int n, float x) except + + long double sph_bessell(unsigned int n, long double x) except + + + # spherical associated Legendre functions + double sph_legendre(unsigned int l, unsigned int m, double theta) except + + float sph_legendref(unsigned int l, unsigned int m, float theta) except + + long double sph_legendrel(unsigned int l, unsigned int m, long double theta) except + + + # spherical Neumann functions + # spherical Bessel functions of the second kind + double sph_neumann(unsigned int n, double x) except + + float sph_neumannf(unsigned int n, float x) except + + long double sph_neumannl(unsigned int n, long double x) except + diff --git a/Cython/Includes/libcpp/deque.pxd b/Cython/Includes/libcpp/deque.pxd index 9e2b2291d..5f189ffd1 100644 --- a/Cython/Includes/libcpp/deque.pxd +++ b/Cython/Includes/libcpp/deque.pxd @@ -9,41 +9,114 @@ cdef extern from "<deque>" namespace "std" nogil: ctypedef size_t size_type ctypedef ptrdiff_t difference_type + cppclass const_iterator cppclass iterator: - T& operator*() + iterator() except + + iterator(iterator&) except + + value_type& operator*() iterator operator++() iterator operator--() + iterator operator++(int) + iterator operator--(int) iterator operator+(size_type) iterator operator-(size_type) difference_type operator-(iterator) + difference_type operator-(const_iterator) bint operator==(iterator) + bint operator==(const_iterator) bint operator!=(iterator) + bint operator!=(const_iterator) bint operator<(iterator) + bint operator<(const_iterator) bint operator>(iterator) + bint operator>(const_iterator) bint operator<=(iterator) + bint operator<=(const_iterator) bint operator>=(iterator) + bint operator>=(const_iterator) + cppclass const_iterator: + const_iterator() except + + const_iterator(iterator&) except + + const_iterator(const_iterator&) except + + operator=(iterator&) except + + const value_type& operator*() + const_iterator operator++() + const_iterator operator--() + const_iterator operator++(int) + const_iterator operator--(int) + const_iterator operator+(size_type) + const_iterator operator-(size_type) + difference_type operator-(iterator) + difference_type operator-(const_iterator) + bint operator==(iterator) + bint operator==(const_iterator) + bint operator!=(iterator) + bint operator!=(const_iterator) + bint operator<(iterator) + bint operator<(const_iterator) + bint operator>(iterator) + bint operator>(const_iterator) + bint operator<=(iterator) + bint operator<=(const_iterator) + bint operator>=(iterator) + bint operator>=(const_iterator) + + cppclass const_reverse_iterator cppclass reverse_iterator: - T& operator*() + reverse_iterator() except + + reverse_iterator(reverse_iterator&) except + + value_type& operator*() reverse_iterator operator++() reverse_iterator operator--() + reverse_iterator operator++(int) + reverse_iterator operator--(int) reverse_iterator operator+(size_type) reverse_iterator operator-(size_type) - difference_type operator-(reverse_iterator) + difference_type operator-(iterator) + difference_type operator-(const_iterator) bint operator==(reverse_iterator) + bint operator==(const_reverse_iterator) bint operator!=(reverse_iterator) + bint operator!=(const_reverse_iterator) bint operator<(reverse_iterator) + bint operator<(const_reverse_iterator) bint operator>(reverse_iterator) + bint operator>(const_reverse_iterator) bint operator<=(reverse_iterator) + bint operator<=(const_reverse_iterator) bint operator>=(reverse_iterator) - cppclass const_iterator(iterator): - pass - cppclass const_reverse_iterator(reverse_iterator): - pass + bint operator>=(const_reverse_iterator) + cppclass const_reverse_iterator: + const_reverse_iterator() except + + const_reverse_iterator(reverse_iterator&) except + + operator=(reverse_iterator&) except + + const value_type& operator*() + const_reverse_iterator operator++() + const_reverse_iterator operator--() + const_reverse_iterator operator++(int) + const_reverse_iterator operator--(int) + const_reverse_iterator operator+(size_type) + const_reverse_iterator operator-(size_type) + difference_type operator-(iterator) + difference_type operator-(const_iterator) + bint operator==(reverse_iterator) + bint operator==(const_reverse_iterator) + bint operator!=(reverse_iterator) + bint operator!=(const_reverse_iterator) + bint operator<(reverse_iterator) + bint operator<(const_reverse_iterator) + bint operator>(reverse_iterator) + bint operator>(const_reverse_iterator) + bint operator<=(reverse_iterator) + bint operator<=(const_reverse_iterator) + bint operator>=(reverse_iterator) + bint operator>=(const_reverse_iterator) + deque() except + deque(deque&) except + deque(size_t) except + deque(size_t, T&) except + - #deque[input_iterator](input_iterator, input_iterator) + #deque[InputIt](InputIt, InputIt) T& operator[](size_t) #deque& operator=(deque&) bint operator==(deque&, deque&) @@ -52,35 +125,39 @@ cdef extern from "<deque>" namespace "std" nogil: bint operator>(deque&, deque&) bint operator<=(deque&, deque&) bint operator>=(deque&, deque&) - void assign(size_t, T&) - void assign(input_iterator, input_iterator) - T& at(size_t) + void assign(size_t, T&) except + + void assign[InputIt](InputIt, InputIt) except + + T& at(size_t) except + T& back() iterator begin() const_iterator const_begin "begin"() + const_iterator cbegin() void clear() bint empty() iterator end() const_iterator const_end "end"() - iterator erase(iterator) - iterator erase(iterator, iterator) + const_iterator cend() + iterator erase(iterator) except + + iterator erase(iterator, iterator) except + T& front() - iterator insert(iterator, T&) - void insert(iterator, size_t, T&) - void insert(iterator, input_iterator, input_iterator) + iterator insert(iterator, T&) except + + void insert(iterator, size_t, T&) except + + void insert[InputIt](iterator, InputIt, InputIt) except + size_t max_size() void pop_back() void pop_front() - void push_back(T&) - void push_front(T&) + void push_back(T&) except + + void push_front(T&) except + reverse_iterator rbegin() #const_reverse_iterator rbegin() + const_reverse_iterator crbegin() reverse_iterator rend() #const_reverse_iterator rend() - void resize(size_t) - void resize(size_t, T&) + const_reverse_iterator crend() + void resize(size_t) except + + void resize(size_t, T&) except + size_t size() void swap(deque&) # C++11 methods - void shrink_to_fit() + void shrink_to_fit() except + diff --git a/Cython/Includes/libcpp/execution.pxd b/Cython/Includes/libcpp/execution.pxd new file mode 100644 index 000000000..eb92e3404 --- /dev/null +++ b/Cython/Includes/libcpp/execution.pxd @@ -0,0 +1,15 @@ + +cdef extern from "<execution>" namespace "std::execution" nogil: + cdef cppclass sequenced_policy: + pass + cdef cppclass parallel_policy: + pass + cdef cppclass parallel_unsequenced_policy: + pass + cdef cppclass unsequenced_policy: + pass + + const sequenced_policy seq "std::execution::seq" + const parallel_policy par "std::execution::par" + const parallel_unsequenced_policy par_unseq "std::execution::par_unseq" + const unsequenced_policy unseq "std::execution::unseq" diff --git a/Cython/Includes/libcpp/forward_list.pxd b/Cython/Includes/libcpp/forward_list.pxd index 8c3b240d0..3e04ce875 100644 --- a/Cython/Includes/libcpp/forward_list.pxd +++ b/Cython/Includes/libcpp/forward_list.pxd @@ -14,6 +14,7 @@ cdef extern from "<forward_list>" namespace "std" nogil: iterator(iterator &) T& operator*() iterator operator++() + iterator operator++(int) bint operator==(iterator) bint operator!=(iterator) cppclass const_iterator(iterator): diff --git a/Cython/Includes/libcpp/functional.pxd b/Cython/Includes/libcpp/functional.pxd index 94cbd9e1d..4786d39eb 100644 --- a/Cython/Includes/libcpp/functional.pxd +++ b/Cython/Includes/libcpp/functional.pxd @@ -1,3 +1,5 @@ +from libcpp cimport bool + cdef extern from "<functional>" namespace "std" nogil: cdef cppclass function[T]: function() except + @@ -10,4 +12,10 @@ cdef extern from "<functional>" namespace "std" nogil: function operator=(void*) function operator=[U](U) - bint operator bool() + bool operator bool() + + # Comparisons + cdef cppclass greater[T=*]: + # https://github.com/cython/cython/issues/3193 + greater() except + + bool operator()(const T& lhs, const T& rhs) except + diff --git a/Cython/Includes/libcpp/iterator.pxd b/Cython/Includes/libcpp/iterator.pxd index e0f8bd8d6..0b50c586d 100644 --- a/Cython/Includes/libcpp/iterator.pxd +++ b/Cython/Includes/libcpp/iterator.pxd @@ -1,6 +1,8 @@ #Basic reference: http://www.cplusplus.com/reference/iterator/ #Most of these classes are in fact empty structs +from libc.stddef import ptrdiff_t + cdef extern from "<iterator>" namespace "std" nogil: cdef cppclass iterator[Category,T,Distance,Pointer,Reference]: pass @@ -29,4 +31,4 @@ cdef extern from "<iterator>" namespace "std" nogil: ##insert_iterator<Container> inserter (Container& x, typename Container::iterator it) insert_iterator[CONTAINER] inserter[CONTAINER,ITERATOR](CONTAINER &, ITERATOR) - + ptrdiff_t distance[It](It first, It last) diff --git a/Cython/Includes/libcpp/limits.pxd b/Cython/Includes/libcpp/limits.pxd index c325263b7..11f5e23ea 100644 --- a/Cython/Includes/libcpp/limits.pxd +++ b/Cython/Includes/libcpp/limits.pxd @@ -1,61 +1,61 @@ cdef extern from "<limits>" namespace "std" nogil: - enum float_round_style: + enum float_round_style: round_indeterminate = -1 round_toward_zero = 0 round_to_nearest = 1 round_toward_infinity = 2 round_toward_neg_infinity = 3 - enum float_denorm_style: + enum float_denorm_style: denorm_indeterminate = -1 denorm_absent = 0 denorm_present = 1 - #The static methods can be called as, e.g. numeric_limits[int].round_error(), etc. - #The const data members should be declared as static. Cython currently doesn't allow that - #and/or I can't figure it out, so you must instantiate an object to access, e.g. - #cdef numeric_limits[double] lm - #print lm.round_style - cdef cppclass numeric_limits[T]: - const bint is_specialized - @staticmethod - T min() - @staticmethod - T max() - const int digits - const int digits10 - const bint is_signed - const bint is_integer - const bint is_exact - const int radix - @staticmethod - T epsilon() - @staticmethod - T round_error() + #The static methods can be called as, e.g. numeric_limits[int].round_error(), etc. + #The const data members should be declared as static. Cython currently doesn't allow that + #and/or I can't figure it out, so you must instantiate an object to access, e.g. + #cdef numeric_limits[double] lm + #print lm.round_style + cdef cppclass numeric_limits[T]: + const bint is_specialized + @staticmethod + T min() + @staticmethod + T max() + const int digits + const int digits10 + const bint is_signed + const bint is_integer + const bint is_exact + const int radix + @staticmethod + T epsilon() + @staticmethod + T round_error() - const int min_exponent - const int min_exponent10 - const int max_exponent - const int max_exponent10 + const int min_exponent + const int min_exponent10 + const int max_exponent + const int max_exponent10 - const bint has_infinity - const bint has_quiet_NaN - const bint has_signaling_NaN - const float_denorm_style has_denorm - const bint has_denorm_loss - @staticmethod - T infinity() - @staticmethod - T quiet_NaN() - @staticmethod - T signaling_NaN() - @staticmethod - T denorm_min() + const bint has_infinity + const bint has_quiet_NaN + const bint has_signaling_NaN + const float_denorm_style has_denorm + const bint has_denorm_loss + @staticmethod + T infinity() + @staticmethod + T quiet_NaN() + @staticmethod + T signaling_NaN() + @staticmethod + T denorm_min() - const bint is_iec559 - const bint is_bounded - const bint is_modulo + const bint is_iec559 + const bint is_bounded + const bint is_modulo - const bint traps - const bint tinyness_before - const float_round_style round_style + const bint traps + const bint tinyness_before + const float_round_style round_style diff --git a/Cython/Includes/libcpp/list.pxd b/Cython/Includes/libcpp/list.pxd index b5b0410ad..b69cd573e 100644 --- a/Cython/Includes/libcpp/list.pxd +++ b/Cython/Includes/libcpp/list.pxd @@ -9,26 +9,61 @@ cdef extern from "<list>" namespace "std" nogil: ctypedef size_t size_type ctypedef ptrdiff_t difference_type + cppclass const_iterator cppclass iterator: - iterator() - iterator(iterator &) - T& operator*() + iterator() except + + iterator(iterator&) except + + value_type& operator*() iterator operator++() iterator operator--() + iterator operator++(int) + iterator operator--(int) bint operator==(iterator) + bint operator==(const_iterator) bint operator!=(iterator) + bint operator!=(const_iterator) + cppclass const_iterator: + const_iterator() except + + const_iterator(iterator&) except + + const_iterator(const_iterator&) except + + operator=(iterator&) except + + const value_type& operator*() + const_iterator operator++() + const_iterator operator--() + const_iterator operator++(int) + const_iterator operator--(int) + bint operator==(iterator) + bint operator==(const_iterator) + bint operator!=(iterator) + bint operator!=(const_iterator) + + cppclass const_reverse_iterator cppclass reverse_iterator: - reverse_iterator() - reverse_iterator(iterator &) - T& operator*() + reverse_iterator() except + + reverse_iterator(reverse_iterator&) except + + value_type& operator*() reverse_iterator operator++() reverse_iterator operator--() + reverse_iterator operator++(int) + reverse_iterator operator--(int) bint operator==(reverse_iterator) + bint operator==(const_reverse_iterator) bint operator!=(reverse_iterator) - cppclass const_iterator(iterator): - pass - cppclass const_reverse_iterator(reverse_iterator): - pass + bint operator!=(const_reverse_iterator) + cppclass const_reverse_iterator: + const_reverse_iterator() except + + const_reverse_iterator(reverse_iterator&) except + + operator=(reverse_iterator&) except + + const value_type& operator*() + const_reverse_iterator operator++() + const_reverse_iterator operator--() + const_reverse_iterator operator++(int) + const_reverse_iterator operator--(int) + bint operator==(reverse_iterator) + bint operator==(const_reverse_iterator) + bint operator!=(reverse_iterator) + bint operator!=(const_reverse_iterator) + list() except + list(list&) except + list(size_t, T&) except + @@ -39,36 +74,40 @@ cdef extern from "<list>" namespace "std" nogil: bint operator>(list&, list&) bint operator<=(list&, list&) bint operator>=(list&, list&) - void assign(size_t, T&) + void assign(size_t, T&) except + T& back() iterator begin() const_iterator const_begin "begin"() + const_iterator cbegin() void clear() bint empty() iterator end() const_iterator const_end "end"() + const_iterator cend() iterator erase(iterator) iterator erase(iterator, iterator) T& front() iterator insert(iterator, T&) void insert(iterator, size_t, T&) size_t max_size() - void merge(list&) + void merge(list&) except + #void merge(list&, BinPred) void pop_back() void pop_front() - void push_back(T&) - void push_front(T&) + void push_back(T&) except + + void push_front(T&) except + reverse_iterator rbegin() const_reverse_iterator const_rbegin "rbegin"() - void remove(T&) + const_reverse_iterator crbegin() + void remove(T&) except + #void remove_if(UnPred) reverse_iterator rend() const_reverse_iterator const_rend "rend"() - void resize(size_t, T&) + const_reverse_iterator crend() + void resize(size_t, T&) except + void reverse() size_t size() - void sort() + void sort() except + #void sort(BinPred) void splice(iterator, list&) void splice(iterator, list&, iterator) diff --git a/Cython/Includes/libcpp/map.pxd b/Cython/Includes/libcpp/map.pxd index 624a7ac02..d81af66e0 100644 --- a/Cython/Includes/libcpp/map.pxd +++ b/Cython/Includes/libcpp/map.pxd @@ -7,26 +7,76 @@ cdef extern from "<map>" namespace "std" nogil: ctypedef pair[const T, U] value_type ctypedef COMPARE key_compare ctypedef ALLOCATOR allocator_type + + # these should really be allocator_type.size_type and + # allocator_type.difference_type to be true to the C++ definition + # but cython doesn't support deferred access on template arguments + ctypedef size_t size_type + ctypedef ptrdiff_t difference_type + + cppclass const_iterator cppclass iterator: + iterator() except + + iterator(iterator&) except + + # correct would be value_type& but this does not work + # well with cython's code gen pair[T, U]& operator*() iterator operator++() iterator operator--() + iterator operator++(int) + iterator operator--(int) + bint operator==(iterator) + bint operator==(const_iterator) + bint operator!=(iterator) + bint operator!=(const_iterator) + cppclass const_iterator: + const_iterator() except + + const_iterator(iterator&) except + + const_iterator(const_iterator&) except + + operator=(iterator&) except + + # correct would be const value_type& but this does not work + # well with cython's code gen + const pair[T, U]& operator*() + const_iterator operator++() + const_iterator operator--() + const_iterator operator++(int) + const_iterator operator--(int) bint operator==(iterator) + bint operator==(const_iterator) bint operator!=(iterator) + bint operator!=(const_iterator) + + cppclass const_reverse_iterator cppclass reverse_iterator: - pair[T, U]& operator*() - iterator operator++() - iterator operator--() + reverse_iterator() except + + reverse_iterator(reverse_iterator&) except + + value_type& operator*() + reverse_iterator operator++() + reverse_iterator operator--() + reverse_iterator operator++(int) + reverse_iterator operator--(int) + bint operator==(reverse_iterator) + bint operator==(const_reverse_iterator) + bint operator!=(reverse_iterator) + bint operator!=(const_reverse_iterator) + cppclass const_reverse_iterator: + const_reverse_iterator() except + + const_reverse_iterator(reverse_iterator&) except + + operator=(reverse_iterator&) except + + const value_type& operator*() + const_reverse_iterator operator++() + const_reverse_iterator operator--() + const_reverse_iterator operator++(int) + const_reverse_iterator operator--(int) bint operator==(reverse_iterator) + bint operator==(const_reverse_iterator) bint operator!=(reverse_iterator) - cppclass const_iterator(iterator): - pass - cppclass const_reverse_iterator(reverse_iterator): - pass + bint operator!=(const_reverse_iterator) + map() except + map(map&) except + #map(key_compare&) - U& operator[](T&) + U& operator[](const T&) #map& operator=(map&) bint operator==(map&, map&) bint operator!=(map&, map&) @@ -38,31 +88,157 @@ cdef extern from "<map>" namespace "std" nogil: const U& const_at "at"(const T&) except + iterator begin() const_iterator const_begin "begin" () + const_iterator cbegin() void clear() size_t count(const T&) bint empty() iterator end() const_iterator const_end "end" () + const_iterator cend() pair[iterator, iterator] equal_range(const T&) - #pair[const_iterator, const_iterator] equal_range(key_type&) - void erase(iterator) - void erase(iterator, iterator) + pair[const_iterator, const_iterator] const_equal_range "equal_range"(const T&) + iterator erase(iterator) + iterator const_erase "erase"(const_iterator) + iterator erase(const_iterator, const_iterator) size_t erase(const T&) iterator find(const T&) const_iterator const_find "find" (const T&) - pair[iterator, bint] insert(pair[T, U]) except + # XXX pair[T,U]& - iterator insert(iterator, pair[T, U]) except + # XXX pair[T,U]& - #void insert(input_iterator, input_iterator) + pair[iterator, bint] insert(const pair[T, U]&) except + + iterator insert(const_iterator, const pair[T, U]&) except + + void insert[InputIt](InputIt, InputIt) except + #key_compare key_comp() iterator lower_bound(const T&) const_iterator const_lower_bound "lower_bound"(const T&) size_t max_size() reverse_iterator rbegin() const_reverse_iterator const_rbegin "rbegin"() + const_reverse_iterator crbegin() reverse_iterator rend() const_reverse_iterator const_rend "rend"() + const_reverse_iterator crend() size_t size() void swap(map&) iterator upper_bound(const T&) const_iterator const_upper_bound "upper_bound"(const T&) #value_compare value_comp() + # C++20 + bint contains(const T&) + + cdef cppclass multimap[T, U, COMPARE=*, ALLOCATOR=*]: + ctypedef T key_type + ctypedef U mapped_type + ctypedef pair[const T, U] value_type + ctypedef COMPARE key_compare + ctypedef ALLOCATOR allocator_type + + # these should really be allocator_type.size_type and + # allocator_type.difference_type to be true to the C++ definition + # but cython doesn't support deferred access on template arguments + ctypedef size_t size_type + ctypedef ptrdiff_t difference_type + + cppclass const_iterator + cppclass iterator: + iterator() except + + iterator(iterator&) except + + # correct would be value_type& but this does not work + # well with cython's code gen + pair[T, U]& operator*() + iterator operator++() + iterator operator--() + iterator operator++(int) + iterator operator--(int) + bint operator==(iterator) + bint operator==(const_iterator) + bint operator!=(iterator) + bint operator!=(const_iterator) + cppclass const_iterator: + const_iterator() except + + const_iterator(iterator&) except + + const_iterator(const_iterator&) except + + operator=(iterator&) except + + # correct would be const value_type& but this does not work + # well with cython's code gen + const pair[T, U]& operator*() + const_iterator operator++() + const_iterator operator--() + const_iterator operator++(int) + const_iterator operator--(int) + bint operator==(iterator) + bint operator==(const_iterator) + bint operator!=(iterator) + bint operator!=(const_iterator) + + cppclass const_reverse_iterator + cppclass reverse_iterator: + reverse_iterator() except + + reverse_iterator(reverse_iterator&) except + + value_type& operator*() + reverse_iterator operator++() + reverse_iterator operator--() + reverse_iterator operator++(int) + reverse_iterator operator--(int) + bint operator==(reverse_iterator) + bint operator==(const_reverse_iterator) + bint operator!=(reverse_iterator) + bint operator!=(const_reverse_iterator) + cppclass const_reverse_iterator: + const_reverse_iterator() except + + const_reverse_iterator(reverse_iterator&) except + + operator=(reverse_iterator&) except + + const value_type& operator*() + const_reverse_iterator operator++() + const_reverse_iterator operator--() + const_reverse_iterator operator++(int) + const_reverse_iterator operator--(int) + bint operator==(reverse_iterator) + bint operator==(const_reverse_iterator) + bint operator!=(reverse_iterator) + bint operator!=(const_reverse_iterator) + + multimap() except + + multimap(const multimap&) except + + #multimap(key_compare&) + #multimap& operator=(multimap&) + bint operator==(const multimap&, const multimap&) + bint operator!=(const multimap&, const multimap&) + bint operator<(const multimap&, const multimap&) + bint operator>(const multimap&, const multimap&) + bint operator<=(const multimap&, const multimap&) + bint operator>=(const multimap&, const multimap&) + iterator begin() + const_iterator const_begin "begin"() + const_iterator cbegin() + void clear() + size_t count(const T&) + bint empty() + iterator end() + const_iterator const_end "end"() + const_iterator cend() + pair[iterator, iterator] equal_range(const T&) + pair[const_iterator, const_iterator] const_equal_range "equal_range"(const T&) + iterator erase(iterator) + iterator const_erase "erase"(const_iterator) + iterator erase(const_iterator, const_iterator) + size_t erase(const T&) + iterator find(const T&) + const_iterator const_find "find"(const T&) + iterator insert(const pair[T, U]&) except + + iterator insert(const_iterator, const pair[T, U]&) except + + void insert[InputIt](InputIt, InputIt) except + + #key_compare key_comp() + iterator lower_bound(const T&) + const_iterator const_lower_bound "lower_bound"(const T&) + size_t max_size() + reverse_iterator rbegin() + const_reverse_iterator const_rbegin "rbegin"() + const_reverse_iterator crbegin() + reverse_iterator rend() + const_reverse_iterator const_rend "rend"() + const_reverse_iterator crend() + size_t size() + void swap(multimap&) + iterator upper_bound(const T&) + const_iterator const_upper_bound "upper_bound"(const T&) + #value_compare value_comp() + bint contains(const T&) diff --git a/Cython/Includes/libcpp/memory.pxd b/Cython/Includes/libcpp/memory.pxd index 2151c1ec7..c477c93fe 100644 --- a/Cython/Includes/libcpp/memory.pxd +++ b/Cython/Includes/libcpp/memory.pxd @@ -59,6 +59,7 @@ cdef extern from "<memory>" namespace "std" nogil: shared_ptr(shared_ptr[T]&, T*) shared_ptr(unique_ptr[T]&) #shared_ptr(weak_ptr[T]&) # Not Supported + shared_ptr[T]& operator=[Y](const shared_ptr[Y]& ptr) # Modifiers void reset() diff --git a/Cython/Includes/libcpp/numbers.pxd b/Cython/Includes/libcpp/numbers.pxd new file mode 100644 index 000000000..abd3069a1 --- /dev/null +++ b/Cython/Includes/libcpp/numbers.pxd @@ -0,0 +1,15 @@ +cdef extern from "<numbers>" namespace "std::numbers" nogil: + # C++20 mathematical constants + const double e + const double log2e + const double log10e + const double pi + const double inv_pi + const double inv_sqrtpi + const double ln2 + const double ln10 + const double sqrt2 + const double sqrt3 + const double inv_sqrt3 + const double egamma + const double phi diff --git a/Cython/Includes/libcpp/numeric.pxd b/Cython/Includes/libcpp/numeric.pxd new file mode 100644 index 000000000..a9fb37205 --- /dev/null +++ b/Cython/Includes/libcpp/numeric.pxd @@ -0,0 +1,131 @@ +cdef extern from "<numeric>" namespace "std" nogil: + T inner_product[InputIt1, InputIt2, T](InputIt1 first1, InputIt1 last1, InputIt2 first2, T init) + + T inner_product[InputIt1, InputIt2, T, BinaryOperation1, BinaryOperation2](InputIt1 first1, InputIt1 last1, + InputIt2 first2, T init, + BinaryOperation1 op1, + BinaryOperation2 op2) + + void iota[ForwardIt, T](ForwardIt first, ForwardIt last, T value) + + T accumulate[InputIt, T](InputIt first, InputIt last, T init) + + T accumulate[InputIt, T, BinaryOperation](InputIt first, InputIt last, T init, BinaryOperation op) + + void adjacent_difference[InputIt, OutputIt](InputIt in_first, InputIt in_last, OutputIt out_first) + + void adjacent_difference[InputIt, OutputIt, BinaryOperation](InputIt in_first, InputIt in_last, OutputIt out_first, + BinaryOperation op) + + void partial_sum[InputIt, OutputIt](InputIt in_first, OutputIt in_last, OutputIt out_first) + + void partial_sum[InputIt, OutputIt, BinaryOperation](InputIt in_first, InputIt in_last, OutputIt out_first, + BinaryOperation op) + + + T reduce[InputIt, T](InputIt first, InputIt last, T init) + + # ambiguous with next overload + #T reduce[ExecutionPolicy, ForwardIt, T](ExecutionPolicy&& policy, + # ForwardIt first, ForwardIt last, T init) + + T reduce[InputIt, T, BinaryOp](InputIt first, InputIt last, T init, BinaryOp binary_op) + + T reduce[ExecutionPolicy, ForwardIt, T, BinaryOp](ExecutionPolicy&& policy, + ForwardIt first, ForwardIt last, T init, BinaryOp binary_op) + + T transform_reduce[InputIt1, InputIt2, T](InputIt1 first1, InputIt1 last1, + InputIt2 first2, T init) + + T transform_reduce[InputIt1, InputIt2, T, BinaryReductionOp, BinaryTransformOp]( + InputIt1 first1, InputIt1 last1, InputIt2 first2, T init, + BinaryReductionOp reduce, BinaryTransformOp transform) + + T transform_reduce[InputIt, T, BinaryReductionOp, UnaryTransformOp]( + InputIt first, InputIt last, T init, BinaryReductionOp reduce, + UnaryTransformOp transform) + + # ambiguous with previous overload + #T transform_reduce[ExecutionPolicy, ForwardIt1, ForwardIt2, T]( + # ExecutionPolicy&& policy, ForwardIt1 first1, ForwardIt1 last1, + # ForwardIt2 first2, T init) + + T transform_reduce[ExecutionPolicy, ForwardIt1, ForwardIt2, T, BinaryReductionOp, BinaryTransformOp]( + ExecutionPolicy&& policy, ForwardIt1 first1, ForwardIt1 last1, ForwardIt2 first2, T init, + BinaryReductionOp reduce, BinaryTransformOp transform) + + # ambiguous with second overload + #T transform_reduce[ExecutionPolicy, ForwardIt, T, BinaryReductionOp, UnaryTransformOp]( + # ExecutionPolicy&& policy, ForwardIt first, ForwardIt last, T init, BinaryReductionOp reduce, + # UnaryTransformOp transform) + + OutputIt inclusive_scan[InputIt, OutputIt](InputIt first, InputIt last, OutputIt d_first) + + # ambiguous with next overload + # ForwardIt2 inclusive_scan[ExecutionPolicy, ForwardIt1, ForwardIt2]( + # ExecutionPolicy&& policy, ForwardIt1 first, ForwardIt1 last, + # ForwardIt2 d_first) + + OutputIt inclusive_scan[InputIt, OutputIt, BinaryOperation]( + InputIt first, InputIt last, OutputIt d_first, BinaryOperation binary_op) + + # ambiguous with next overload + # ForwardIt2 inclusive_scan[ExecutionPolicy, ForwardIt1, ForwardIt2, BinaryOperation]( + # ExecutionPolicy&& policy, ForwardIt1 first, ForwardIt1 last, ForwardIt2 d_first, + # BinaryOperation binary_op) + + OutputIt inclusive_scan[InputIt, OutputIt, BinaryOperation, T]( + InputIt first, InputIt last, OutputIt d_first, BinaryOperation binary_op, + T init) + + # + # ForwardIt2 inclusive_scan[ExecutionPolicy, ForwardIt1, ForwardIt2, BinaryOperation, T]( + # ExecutionPolicy&& policy, ForwardIt1 first, ForwardIt1 last, ForwardIt2 d_first, + # BinaryOperation binary_op, T init) + + OutputIt exclusive_scan[InputIt, OutputIt, T](InputIt first, InputIt last, + OutputIt d_first, T init) + + # ambiguous with next overload + #ForwardIt2 exclusive_scan[ExecutionPolicy, ForwardIt1, ForwardIt2, T]( + # ExecutionPolicy&& policy, ForwardIt1 first, ForwardIt1 last, + # ForwardIt2 d_first, T init) + + OutputIt exclusive_scan[InputIt, OutputIt, T, BinaryOperation]( + InputIt first, InputIt last, OutputIt d_first, T init, BinaryOperation binary_op) + + ForwardIt2 exclusive_scan[ExecutionPolicy, ForwardIt1, ForwardIt2, T, BinaryOperation]( + ExecutionPolicy&& policy, ForwardIt1 first, ForwardIt1 last, ForwardIt2 d_first, + T init, BinaryOperation binary_op) + + OutputIt transform_inclusive_scan[InputIt, OutputIt, BinaryOperation, UnaryOperation]( + InputIt first, InputIt last, OutputIt d_first, BinaryOperation binary_op, + UnaryOperation unary_op) + + # ambiguous with next overload + # ForwardIt2 transform_inclusive_scan[ExecutionPolicy, ForwardIt1, ForwardIt2, BinaryOperation, UnaryOperation]( + # ExecutionPolicy&& policy, ForwardIt1 first, ForwardIt1 last, ForwardIt2 d_first, + # BinaryOperation binary_op, UnaryOperation unary_op) + + OutputIt transform_inclusive_scan[InputIt, OutputIt, BinaryOperation, UnaryOperation, T]( + InputIt first, InputIt last, OutputIt d_first, BinaryOperation binary_op, + UnaryOperation unary_op, T init) + + ForwardIt2 transform_inclusive_scan[ExecutionPolicy, ForwardIt1, ForwardIt2, BinaryOperation, UnaryOperation, T]( + ExecutionPolicy&& policy, ForwardIt1 first, ForwardIt1 last, ForwardIt2 d_first, + BinaryOperation binary_op, UnaryOperation unary_op, T init) + + OutputIt transform_exclusive_scan[InputIt, OutputIt, T, BinaryOperation, UnaryOperation]( + InputIt first, InputIt last, OutputIt d_first, T init, BinaryOperation binary_op, + UnaryOperation unary_op) + + ForwardIt2 transform_exclusive_scan[ExecutionPolicy, ForwardIt1, ForwardIt2, T, BinaryOperation, UnaryOperation]( + ExecutionPolicy&& policy, ForwardIt1 first, ForwardIt1 last, ForwardIt2 d_first, + T init, BinaryOperation binary_op, UnaryOperation unary_op) + + # C++17 + T gcd[T](T a, T b) + T lcm[T](T a, T b) + + # C++20 + T midpoint[T](T a, T b) except + diff --git a/Cython/Includes/libcpp/optional.pxd b/Cython/Includes/libcpp/optional.pxd new file mode 100644 index 000000000..284dfcd6e --- /dev/null +++ b/Cython/Includes/libcpp/optional.pxd @@ -0,0 +1,34 @@ +from libcpp cimport bool + +cdef extern from "<optional>" namespace "std" nogil: + cdef cppclass nullopt_t: + nullopt_t() + + cdef nullopt_t nullopt + + cdef cppclass optional[T]: + ctypedef T value_type + optional() + optional(nullopt_t) + optional(optional&) except + + optional(T&) except + + bool has_value() + T& value() + T& value_or[U](U& default_value) + void swap(optional&) + void reset() + T& emplace(...) + T& operator*() + #T* operator->() # Not Supported + optional& operator=(optional&) + optional& operator=[U](U&) + bool operator bool() + bool operator!() + bool operator==[U](optional&, U&) + bool operator!=[U](optional&, U&) + bool operator<[U](optional&, U&) + bool operator>[U](optional&, U&) + bool operator<=[U](optional&, U&) + bool operator>=[U](optional&, U&) + + optional[T] make_optional[T](...) except + diff --git a/Cython/Includes/libcpp/random.pxd b/Cython/Includes/libcpp/random.pxd new file mode 100644 index 000000000..9e48bb27f --- /dev/null +++ b/Cython/Includes/libcpp/random.pxd @@ -0,0 +1,166 @@ +from libc.stdint cimport uint_fast32_t, uint_fast64_t + + +cdef extern from "<random>" namespace "std" nogil: + cdef cppclass random_device: + ctypedef uint_fast32_t result_type + random_device() except + + result_type operator()() except + + + cdef cppclass mt19937: + ctypedef uint_fast32_t result_type + mt19937() except + + mt19937(result_type seed) except + + result_type operator()() except + + result_type min() except + + result_type max() except + + void discard(size_t z) except + + void seed(result_type seed) except + + + cdef cppclass mt19937_64: + ctypedef uint_fast64_t result_type + + mt19937_64() except + + mt19937_64(result_type seed) except + + result_type operator()() except + + result_type min() except + + result_type max() except + + void discard(size_t z) except + + void seed(result_type seed) except + + + cdef cppclass uniform_int_distribution[T]: + ctypedef T result_type + uniform_int_distribution() except + + uniform_int_distribution(T, T) except + + result_type operator()[Generator](Generator&) except + + result_type min() except + + result_type max() except + + + cdef cppclass uniform_real_distribution[T]: + ctypedef T result_type + uniform_real_distribution() except + + uniform_real_distribution(T, T) except + + result_type operator()[Generator](Generator&) except + + result_type min() except + + result_type max() except + + + cdef cppclass bernoulli_distribution: + ctypedef bint result_type + bernoulli_distribution() except + + bernoulli_distribution(double) except + + result_type operator()[Generator](Generator&) except + + result_type min() except + + result_type max() except + + + cdef cppclass binomial_distribution[T]: + ctypedef T result_type + binomial_distribution() except + + binomial_distribution(T, double) except + + result_type operator()[Generator](Generator&) except + + result_type min() except + + result_type max() except + + + cdef cppclass geometric_distribution[T]: + ctypedef T result_type + geometric_distribution() except + + geometric_distribution(double) except + + result_type operator()[Generator](Generator&) except + + result_type min() except + + result_type max() except + + + + cdef cppclass negative_binomial_distribution[T]: + ctypedef T result_type + negative_binomial_distribution() except + + negative_binomial_distribution(T, double) except + + result_type operator()[Generator](Generator&) except + + result_type min() except + + result_type max() except + + + cdef cppclass poisson_distribution[T]: + ctypedef T result_type + poisson_distribution() except + + poisson_distribution(double) except + + result_type operator()[Generator](Generator&) except + + result_type min() except + + result_type max() except + + + cdef cppclass exponential_distribution[T]: + ctypedef T result_type + exponential_distribution() except + + exponential_distribution(result_type) except + + result_type operator()[Generator](Generator&) except + + result_type min() except + + result_type max() except + + + cdef cppclass gamma_distribution[T]: + ctypedef T result_type + gamma_distribution() except + + gamma_distribution(result_type, result_type) except + + result_type operator()[Generator](Generator&) except + + result_type min() except + + result_type max() except + + + cdef cppclass weibull_distribution[T]: + ctypedef T result_type + weibull_distribution() except + + weibull_distribution(result_type, result_type) except + + result_type operator()[Generator](Generator&) except + + result_type min() except + + result_type max() except + + + cdef cppclass extreme_value_distribution[T]: + ctypedef T result_type + extreme_value_distribution() except + + extreme_value_distribution(result_type, result_type) except + + result_type operator()[Generator](Generator&) except + + result_type min() except + + result_type max() except + + + cdef cppclass normal_distribution[T]: + ctypedef T result_type + normal_distribution() except + + normal_distribution(result_type, result_type) except + + result_type operator()[Generator](Generator&) except + + result_type min() except + + result_type max() except + + + cdef cppclass lognormal_distribution[T]: + ctypedef T result_type + lognormal_distribution() except + + lognormal_distribution(result_type, result_type) except + + result_type operator()[Generator](Generator&) except + + result_type min() except + + result_type max() except + + + cdef cppclass chi_squared_distribution[T]: + ctypedef T result_type + chi_squared_distribution() except + + chi_squared_distribution(result_type) except + + result_type operator()[Generator](Generator&) except + + result_type min() except + + result_type max() except + + + cdef cppclass cauchy_distribution[T]: + ctypedef T result_type + cauchy_distribution() except + + cauchy_distribution(result_type, result_type) except + + result_type operator()[Generator](Generator&) except + + result_type min() except + + result_type max() except + + + cdef cppclass fisher_f_distribution[T]: + ctypedef T result_type + fisher_f_distribution() except + + fisher_f_distribution(result_type, result_type) except + + result_type operator()[Generator](Generator&) except + + result_type min() except + + result_type max() except + + + cdef cppclass student_t_distribution[T]: + ctypedef T result_type + student_t_distribution() except + + student_t_distribution(result_type) except + + result_type operator()[Generator](Generator&) except + + result_type min() except + + result_type max() except + diff --git a/Cython/Includes/libcpp/set.pxd b/Cython/Includes/libcpp/set.pxd index 1069be746..444b1ce9f 100644 --- a/Cython/Includes/libcpp/set.pxd +++ b/Cython/Includes/libcpp/set.pxd @@ -3,22 +3,68 @@ from .utility cimport pair cdef extern from "<set>" namespace "std" nogil: cdef cppclass set[T]: ctypedef T value_type + + # these should really be allocator_type.size_type and + # allocator_type.difference_type to be true to the C++ definition + # but cython doesn't support deferred access on template arguments + ctypedef size_t size_type + ctypedef ptrdiff_t difference_type + + cppclass const_iterator cppclass iterator: - T& operator*() + iterator() except + + iterator(iterator&) except + + value_type& operator*() iterator operator++() iterator operator--() + iterator operator++(int) + iterator operator--(int) + bint operator==(iterator) + bint operator==(const_iterator) + bint operator!=(iterator) + bint operator!=(const_iterator) + cppclass const_iterator: + const_iterator() except + + const_iterator(iterator&) except + + const_iterator(const_iterator&) except + + operator=(iterator&) except + + const value_type& operator*() + const_iterator operator++() + const_iterator operator--() + const_iterator operator++(int) + const_iterator operator--(int) bint operator==(iterator) + bint operator==(const_iterator) bint operator!=(iterator) + bint operator!=(const_iterator) + + cppclass const_reverse_iterator cppclass reverse_iterator: - T& operator*() - iterator operator++() - iterator operator--() + reverse_iterator() except + + reverse_iterator(reverse_iterator&) except + + value_type& operator*() + reverse_iterator operator++() + reverse_iterator operator--() + reverse_iterator operator++(int) + reverse_iterator operator--(int) + bint operator==(reverse_iterator) + bint operator==(const_reverse_iterator) + bint operator!=(reverse_iterator) + bint operator!=(const_reverse_iterator) + cppclass const_reverse_iterator: + const_reverse_iterator() except + + const_reverse_iterator(reverse_iterator&) except + + operator=(reverse_iterator&) except + + const value_type& operator*() + const_reverse_iterator operator++() + const_reverse_iterator operator--() + const_reverse_iterator operator++(int) + const_reverse_iterator operator--(int) bint operator==(reverse_iterator) + bint operator==(const_reverse_iterator) bint operator!=(reverse_iterator) - cppclass const_iterator(iterator): - pass - cppclass const_reverse_iterator(reverse_iterator): - pass + bint operator!=(const_reverse_iterator) + set() except + set(set&) except + #set(key_compare&) @@ -31,31 +77,152 @@ cdef extern from "<set>" namespace "std" nogil: bint operator>=(set&, set&) iterator begin() const_iterator const_begin "begin"() + const_iterator cbegin() void clear() size_t count(const T&) bint empty() iterator end() const_iterator const_end "end"() + const_iterator cend() pair[iterator, iterator] equal_range(const T&) - #pair[const_iterator, const_iterator] equal_range(T&) + pair[const_iterator, const_iterator] const_equal_range "equal_range"(const T&) iterator erase(iterator) - iterator erase(iterator, iterator) - size_t erase(T&) - iterator find(T&) - const_iterator const_find "find"(T&) + iterator const_erase "erase"(const_iterator) + iterator erase(const_iterator, const_iterator) + size_t erase(const T&) + iterator find(const T&) + const_iterator const_find "find"(const T&) pair[iterator, bint] insert(const T&) except + iterator insert(iterator, const T&) except + - void insert(iterator, iterator) except + + iterator insert(const_iterator, const T&) except + + iterator const_insert "insert"(const_iterator, const T&) except + + void insert[InputIt](InputIt, InputIt) except + #key_compare key_comp() - iterator lower_bound(T&) - const_iterator const_lower_bound "lower_bound"(T&) + iterator lower_bound(const T&) + const_iterator const_lower_bound "lower_bound"(const T&) size_t max_size() reverse_iterator rbegin() const_reverse_iterator const_rbegin "rbegin"() + const_reverse_iterator crbegin() reverse_iterator rend() const_reverse_iterator const_rend "rend"() + const_reverse_iterator crend() size_t size() void swap(set&) iterator upper_bound(const T&) const_iterator const_upper_bound "upper_bound"(const T&) #value_compare value_comp() + # C++20 + bint contains(const T&) + + cdef cppclass multiset[T]: + ctypedef T value_type + + # these should really be allocator_type.size_type and + # allocator_type.difference_type to be true to the C++ definition + # but cython doesn't support deferred access on template arguments + ctypedef size_t size_type + ctypedef ptrdiff_t difference_type + + cppclass const_iterator + cppclass iterator: + iterator() except + + iterator(iterator&) except + + value_type& operator*() + iterator operator++() + iterator operator--() + iterator operator++(int) + iterator operator--(int) + bint operator==(iterator) + bint operator==(const_iterator) + bint operator!=(iterator) + bint operator!=(const_iterator) + cppclass const_iterator: + const_iterator() except + + const_iterator(iterator&) except + + const_iterator(const_iterator&) except + + operator=(iterator&) except + + const value_type& operator*() + const_iterator operator++() + const_iterator operator--() + const_iterator operator++(int) + const_iterator operator--(int) + bint operator==(iterator) + bint operator==(const_iterator) + bint operator!=(iterator) + bint operator!=(const_iterator) + + cppclass const_reverse_iterator + cppclass reverse_iterator: + reverse_iterator() except + + reverse_iterator(reverse_iterator&) except + + value_type& operator*() + reverse_iterator operator++() + reverse_iterator operator--() + reverse_iterator operator++(int) + reverse_iterator operator--(int) + bint operator==(reverse_iterator) + bint operator==(const_reverse_iterator) + bint operator!=(reverse_iterator) + bint operator!=(const_reverse_iterator) + cppclass const_reverse_iterator: + const_reverse_iterator() except + + const_reverse_iterator(reverse_iterator&) except + + operator=(reverse_iterator&) except + + const value_type& operator*() + const_reverse_iterator operator++() + const_reverse_iterator operator--() + const_reverse_iterator operator++(int) + const_reverse_iterator operator--(int) + bint operator==(reverse_iterator) + bint operator==(const_reverse_iterator) + bint operator!=(reverse_iterator) + bint operator!=(const_reverse_iterator) + + multiset() except + + multiset(multiset&) except + + #multiset(key_compare&) + #multiset& operator=(multiset&) + bint operator==(multiset&, multiset&) + bint operator!=(multiset&, multiset&) + bint operator<(multiset&, multiset&) + bint operator>(multiset&, multiset&) + bint operator<=(multiset&, multiset&) + bint operator>=(multiset&, multiset&) + iterator begin() + const_iterator const_begin "begin"() + const_iterator cbegin() + void clear() + size_t count(const T&) + bint empty() + iterator end() + const_iterator const_end "end"() + const_iterator cend() + pair[iterator, iterator] equal_range(const T&) + pair[const_iterator, const_iterator] const_equal_range "equal_range"(const T&) + iterator erase(iterator) + iterator const_erase "erase"(const_iterator) + iterator erase(const_iterator, const_iterator) + size_t erase(const T&) + iterator find(const T&) + const_iterator const_find "find"(const T&) + iterator insert(const T&) except + + iterator insert(iterator, const T&) except + + iterator const_insert "insert"(const_iterator, const T&) except + + void insert[InputIt](InputIt, InputIt) except + + #key_compare key_comp() + iterator lower_bound(const T&) + const_iterator const_lower_bound "lower_bound"(const T&) + size_t max_size() + reverse_iterator rbegin() + const_reverse_iterator const_rbegin "rbegin"() + const_reverse_iterator crbegin() + reverse_iterator rend() + const_reverse_iterator const_rend "rend"() + const_reverse_iterator crend() + size_t size() + void swap(multiset&) + iterator upper_bound(const T&) + const_iterator const_upper_bound "upper_bound"(const T&) + # C++20 + bint contains(const T&) diff --git a/Cython/Includes/libcpp/stack.pxd b/Cython/Includes/libcpp/stack.pxd index 2dc80992b..f92240f66 100644 --- a/Cython/Includes/libcpp/stack.pxd +++ b/Cython/Includes/libcpp/stack.pxd @@ -6,6 +6,6 @@ cdef extern from "<stack>" namespace "std" nogil: #stack(Container&) bint empty() void pop() - void push(T&) + void push(T&) except + size_t size() T& top() diff --git a/Cython/Includes/libcpp/string.pxd b/Cython/Includes/libcpp/string.pxd index a894144f1..23518806a 100644 --- a/Cython/Includes/libcpp/string.pxd +++ b/Cython/Includes/libcpp/string.pxd @@ -7,34 +7,116 @@ cdef extern from "<string>" namespace "std::string" nogil: cdef extern from "<string>" namespace "std" nogil: cdef cppclass string: + ctypedef char value_type + # these should really be allocator_type.size_type and + # allocator_type.difference_type to be true to the C++ definition + # but cython doesn't support deferred access on template arguments + ctypedef size_t size_type + ctypedef ptrdiff_t difference_type + + cppclass const_iterator cppclass iterator: - iterator() - char& operator*() - iterator(iterator&) + iterator() except + + iterator(iterator&) except + + value_type& operator*() iterator operator++() iterator operator--() + iterator operator++(int) + iterator operator--(int) + iterator operator+(size_type) + iterator operator-(size_type) + difference_type operator-(iterator) + difference_type operator-(const_iterator) bint operator==(iterator) + bint operator==(const_iterator) bint operator!=(iterator) - + bint operator!=(const_iterator) + bint operator<(iterator) + bint operator<(const_iterator) + bint operator>(iterator) + bint operator>(const_iterator) + bint operator<=(iterator) + bint operator<=(const_iterator) + bint operator>=(iterator) + bint operator>=(const_iterator) + cppclass const_iterator: + const_iterator() except + + const_iterator(iterator&) except + + const_iterator(const_iterator&) except + + operator=(iterator&) except + + const value_type& operator*() + const_iterator operator++() + const_iterator operator--() + const_iterator operator++(int) + const_iterator operator--(int) + const_iterator operator+(size_type) + const_iterator operator-(size_type) + difference_type operator-(iterator) + difference_type operator-(const_iterator) + bint operator==(iterator) + bint operator==(const_iterator) + bint operator!=(iterator) + bint operator!=(const_iterator) + bint operator<(iterator) + bint operator<(const_iterator) + bint operator>(iterator) + bint operator>(const_iterator) + bint operator<=(iterator) + bint operator<=(const_iterator) + bint operator>=(iterator) + bint operator>=(const_iterator) + + cppclass const_reverse_iterator cppclass reverse_iterator: - char& operator*() - iterator operator++() - iterator operator--() - iterator operator+(size_t) - iterator operator-(size_t) + reverse_iterator() except + + reverse_iterator(reverse_iterator&) except + + value_type& operator*() + reverse_iterator operator++() + reverse_iterator operator--() + reverse_iterator operator++(int) + reverse_iterator operator--(int) + reverse_iterator operator+(size_type) + reverse_iterator operator-(size_type) + difference_type operator-(iterator) + difference_type operator-(const_iterator) bint operator==(reverse_iterator) + bint operator==(const_reverse_iterator) bint operator!=(reverse_iterator) + bint operator!=(const_reverse_iterator) bint operator<(reverse_iterator) + bint operator<(const_reverse_iterator) bint operator>(reverse_iterator) + bint operator>(const_reverse_iterator) bint operator<=(reverse_iterator) + bint operator<=(const_reverse_iterator) bint operator>=(reverse_iterator) - - cppclass const_iterator(iterator): - pass - - cppclass const_reverse_iterator(reverse_iterator): - pass + bint operator>=(const_reverse_iterator) + cppclass const_reverse_iterator: + const_reverse_iterator() except + + const_reverse_iterator(reverse_iterator&) except + + operator=(reverse_iterator&) except + + const value_type& operator*() + const_reverse_iterator operator++() + const_reverse_iterator operator--() + const_reverse_iterator operator++(int) + const_reverse_iterator operator--(int) + const_reverse_iterator operator+(size_type) + const_reverse_iterator operator-(size_type) + difference_type operator-(iterator) + difference_type operator-(const_iterator) + bint operator==(reverse_iterator) + bint operator==(const_reverse_iterator) + bint operator!=(reverse_iterator) + bint operator!=(const_reverse_iterator) + bint operator<(reverse_iterator) + bint operator<(const_reverse_iterator) + bint operator>(reverse_iterator) + bint operator>(const_reverse_iterator) + bint operator<=(reverse_iterator) + bint operator<=(const_reverse_iterator) + bint operator>=(reverse_iterator) + bint operator>=(const_reverse_iterator) string() except + string(const string& s) except + @@ -47,12 +129,16 @@ cdef extern from "<string>" namespace "std" nogil: iterator begin() const_iterator const_begin "begin"() + const_iterator cbegin() iterator end() const_iterator const_end "end"() + const_iterator cend() reverse_iterator rbegin() const_reverse_iterator const_rbegin "rbegin"() + const_reverse_iterator crbegin() reverse_iterator rend() const_reverse_iterator const_rend "rend"() + const_reverse_iterator crend() const char* c_str() const char* data() @@ -62,6 +148,7 @@ cdef extern from "<string>" namespace "std" nogil: void resize(size_t) except + void resize(size_t, char) except + void shrink_to_fit() except + + void swap(string& other) size_t capacity() void reserve(size_t) except + void clear() @@ -164,6 +251,15 @@ cdef extern from "<string>" namespace "std" nogil: string substr(size_t pos) except + string substr() + # C++20 + bint starts_with(char c) except + + bint starts_with(const char* s) + bint ends_with(char c) except + + bint ends_with(const char* s) + # C++23 + bint contains(char c) except + + bint contains(const char* s) + #string& operator= (const string&) #string& operator= (const char*) #string& operator= (char) diff --git a/Cython/Includes/libcpp/unordered_map.pxd b/Cython/Includes/libcpp/unordered_map.pxd index a00fbbed2..90f5a2d06 100644 --- a/Cython/Includes/libcpp/unordered_map.pxd +++ b/Cython/Includes/libcpp/unordered_map.pxd @@ -5,26 +5,49 @@ cdef extern from "<unordered_map>" namespace "std" nogil: ctypedef T key_type ctypedef U mapped_type ctypedef pair[const T, U] value_type + ctypedef ALLOCATOR allocator_type + + # these should really be allocator_type.size_type and + # allocator_type.difference_type to be true to the C++ definition + # but cython doesn't support deferred access on template arguments + ctypedef size_t size_type + ctypedef ptrdiff_t difference_type + + cppclass iterator cppclass iterator: + iterator() except + + iterator(iterator&) except + + # correct would be value_type& but this does not work + # well with cython's code gen pair[T, U]& operator*() iterator operator++() iterator operator--() + iterator operator++(int) + iterator operator--(int) bint operator==(iterator) + bint operator==(const_iterator) bint operator!=(iterator) - cppclass reverse_iterator: - pair[T, U]& operator*() - iterator operator++() - iterator operator--() - bint operator==(reverse_iterator) - bint operator!=(reverse_iterator) - cppclass const_iterator(iterator): - pass - cppclass const_reverse_iterator(reverse_iterator): - pass + bint operator!=(const_iterator) + cppclass const_iterator: + const_iterator() except + + const_iterator(iterator&) except + + operator=(iterator&) except + + # correct would be const value_type& but this does not work + # well with cython's code gen + const pair[T, U]& operator*() + const_iterator operator++() + const_iterator operator--() + const_iterator operator++(int) + const_iterator operator--(int) + bint operator==(iterator) + bint operator==(const_iterator) + bint operator!=(iterator) + bint operator!=(const_iterator) + unordered_map() except + unordered_map(unordered_map&) except + #unordered_map(key_compare&) - U& operator[](T&) + U& operator[](const T&) #unordered_map& operator=(unordered_map&) bint operator==(unordered_map&, unordered_map&) bint operator!=(unordered_map&, unordered_map&) @@ -32,43 +55,139 @@ cdef extern from "<unordered_map>" namespace "std" nogil: bint operator>(unordered_map&, unordered_map&) bint operator<=(unordered_map&, unordered_map&) bint operator>=(unordered_map&, unordered_map&) - U& at(const T&) - const U& const_at "at"(const T&) + U& at(const T&) except + + const U& const_at "at"(const T&) except + iterator begin() const_iterator const_begin "begin"() + const_iterator cbegin() void clear() - size_t count(T&) + size_t count(const T&) bint empty() iterator end() const_iterator const_end "end"() - pair[iterator, iterator] equal_range(T&) + const_iterator cend() + pair[iterator, iterator] equal_range(const T&) pair[const_iterator, const_iterator] const_equal_range "equal_range"(const T&) iterator erase(iterator) - iterator erase(iterator, iterator) - size_t erase(T&) - iterator find(T&) - const_iterator const_find "find"(T&) - pair[iterator, bint] insert(pair[T, U]) # XXX pair[T,U]& - iterator insert(iterator, pair[T, U]) # XXX pair[T,U]& - iterator insert(iterator, iterator) + iterator const_erase "erase"(const_iterator) + iterator erase(const_iterator, const_iterator) + size_t erase(const T&) + iterator find(const T&) + const_iterator const_find "find"(const T&) + pair[iterator, bint] insert(const pair[T, U]&) except + + iterator insert(const_iterator, const pair[T, U]&) except + + void insert[InputIt](InputIt, InputIt) except + #key_compare key_comp() - iterator lower_bound(T&) - const_iterator const_lower_bound "lower_bound"(T&) + iterator lower_bound(const T&) + const_iterator const_lower_bound "lower_bound"(const T&) size_t max_size() - reverse_iterator rbegin() - const_reverse_iterator const_rbegin "rbegin"() - reverse_iterator rend() - const_reverse_iterator const_rend "rend"() size_t size() void swap(unordered_map&) - iterator upper_bound(T&) - const_iterator const_upper_bound "upper_bound"(T&) + iterator upper_bound(const T&) + const_iterator const_upper_bound "upper_bound"(const T&) + #value_compare value_comp() + void max_load_factor(float) + float max_load_factor() + float load_factor() + void rehash(size_t) + void reserve(size_t) + size_t bucket_count() + size_t max_bucket_count() + size_t bucket_size(size_t) + size_t bucket(const T&) + # C++20 + bint contains(const T&) + + cdef cppclass unordered_multimap[T, U, HASH=*, PRED=*, ALLOCATOR=*]: + ctypedef T key_type + ctypedef U mapped_type + ctypedef pair[const T, U] value_type + ctypedef ALLOCATOR allocator_type + + # these should really be allocator_type.size_type and + # allocator_type.difference_type to be true to the C++ definition + # but cython doesn't support deferred access on template arguments + ctypedef size_t size_type + ctypedef ptrdiff_t difference_type + + cppclass const_iterator + cppclass iterator: + iterator() except + + iterator(iterator&) except + + # correct would be value_type& but this does not work + # well with cython's code gen + pair[T, U]& operator*() + iterator operator++() + iterator operator++(int) + bint operator==(iterator) + bint operator==(const_iterator) + bint operator!=(iterator) + bint operator!=(const_iterator) + cppclass const_iterator: + const_iterator() except + + const_iterator(iterator&) except + + operator=(iterator&) except + + # correct would be const value_type& but this does not work + # well with cython's code gen + const pair[T, U]& operator*() + const_iterator operator++() + const_iterator operator++(int) + bint operator==(iterator) + bint operator==(const_iterator) + bint operator!=(iterator) + bint operator!=(const_iterator) + + unordered_multimap() except + + unordered_multimap(const unordered_multimap&) except + + #unordered_multimap(key_compare&) + #unordered_map& operator=(unordered_multimap&) + bint operator==(const unordered_multimap&, const unordered_multimap&) + bint operator!=(const unordered_multimap&, const unordered_multimap&) + bint operator<(const unordered_multimap&, const unordered_multimap&) + bint operator>(const unordered_multimap&, const unordered_multimap&) + bint operator<=(const unordered_multimap&, const unordered_multimap&) + bint operator>=(const unordered_multimap&, const unordered_multimap&) + iterator begin() + const_iterator const_begin "begin"() + const_iterator cbegin() + #local_iterator begin(size_t) + #const_local_iterator const_begin "begin"(size_t) + void clear() + size_t count(const T&) + bint empty() + iterator end() + const_iterator const_end "end"() + const_iterator cend() + #local_iterator end(size_t) + #const_local_iterator const_end "end"(size_t) + pair[iterator, iterator] equal_range(const T&) + pair[const_iterator, const_iterator] const_equal_range "equal_range"(const T&) + iterator erase(iterator) + iterator const_erase "erase"(const_iterator) + iterator erase(const_iterator, const_iterator) + size_t erase(const T&) + iterator find(const T&) + const_iterator const_find "find"(const T&) + iterator insert(const pair[T, U]&) except + + iterator insert(const_iterator, const pair[T, U]&) except + + void insert[InputIt](InputIt, InputIt) except + + #key_compare key_comp() + iterator lower_bound(const T&) + const_iterator const_lower_bound "lower_bound"(const T&) + size_t max_size() + size_t size() + void swap(unordered_multimap&) + iterator upper_bound(const T&) + const_iterator const_upper_bound "upper_bound"(const T&) #value_compare value_comp() void max_load_factor(float) float max_load_factor() + float load_factor() void rehash(size_t) void reserve(size_t) size_t bucket_count() size_t max_bucket_count() size_t bucket_size(size_t) size_t bucket(const T&) + # C++20 + bint contains(const T&) diff --git a/Cython/Includes/libcpp/unordered_set.pxd b/Cython/Includes/libcpp/unordered_set.pxd index 5aa241752..6aae890d9 100644 --- a/Cython/Includes/libcpp/unordered_set.pxd +++ b/Cython/Includes/libcpp/unordered_set.pxd @@ -3,67 +3,150 @@ from .utility cimport pair cdef extern from "<unordered_set>" namespace "std" nogil: cdef cppclass unordered_set[T,HASH=*,PRED=*,ALLOCATOR=*]: ctypedef T value_type + + # these should really be allocator_type.size_type and + # allocator_type.difference_type to be true to the C++ definition + # but cython doesn't support deferred access on template arguments + ctypedef size_t size_type + ctypedef ptrdiff_t difference_type + + cppclass const_iterator cppclass iterator: - T& operator*() + iterator() except + + iterator(iterator&) except + + value_type& operator*() iterator operator++() iterator operator--() + iterator operator++(int) + iterator operator--(int) bint operator==(iterator) + bint operator==(const_iterator) bint operator!=(iterator) - cppclass reverse_iterator: - T& operator*() - iterator operator++() - iterator operator--() - bint operator==(reverse_iterator) - bint operator!=(reverse_iterator) - cppclass const_iterator(iterator): - pass - cppclass const_reverse_iterator(reverse_iterator): - pass + bint operator!=(const_iterator) + cppclass const_iterator: + const_iterator() except + + const_iterator(iterator&) except + + operator=(iterator&) except + + const value_type& operator*() + const_iterator operator++() + const_iterator operator--() + const_iterator operator++(int) + const_iterator operator--(int) + bint operator==(iterator) + bint operator==(const_iterator) + bint operator!=(iterator) + bint operator!=(const_iterator) + unordered_set() except + unordered_set(unordered_set&) except + - #unordered_set(key_compare&) #unordered_set& operator=(unordered_set&) bint operator==(unordered_set&, unordered_set&) bint operator!=(unordered_set&, unordered_set&) - bint operator<(unordered_set&, unordered_set&) - bint operator>(unordered_set&, unordered_set&) - bint operator<=(unordered_set&, unordered_set&) - bint operator>=(unordered_set&, unordered_set&) iterator begin() const_iterator const_begin "begin"() + const_iterator cbegin() void clear() - size_t count(T&) + size_t count(const T&) bint empty() iterator end() const_iterator const_end "end"() - pair[iterator, iterator] equal_range(T&) - pair[const_iterator, const_iterator] const_equal_range "equal_range"(T&) + const_iterator cend() + pair[iterator, iterator] equal_range(const T&) + pair[const_iterator, const_iterator] const_equal_range "equal_range"(const T&) iterator erase(iterator) - iterator erase(iterator, iterator) - size_t erase(T&) - iterator find(T&) - const_iterator const_find "find"(T&) - pair[iterator, bint] insert(T&) - iterator insert(iterator, T&) - #key_compare key_comp() - iterator insert(iterator, iterator) - iterator lower_bound(T&) - const_iterator const_lower_bound "lower_bound"(T&) + iterator const_erase "erase"(const_iterator) + iterator erase(const_iterator, const_iterator) + size_t erase(const T&) + iterator find(const T&) + const_iterator const_find "find"(const T&) + pair[iterator, bint] insert(const T&) except + + iterator insert(const_iterator, const T&) except + + void insert[InputIt](InputIt, InputIt) except + size_t max_size() - reverse_iterator rbegin() - const_reverse_iterator const_rbegin "rbegin"() - reverse_iterator rend() - const_reverse_iterator const_rend "rend"() size_t size() void swap(unordered_set&) - iterator upper_bound(T&) - const_iterator const_upper_bound "upper_bound"(T&) #value_compare value_comp() void max_load_factor(float) float max_load_factor() + float load_factor() + void rehash(size_t) + void reserve(size_t) + size_t bucket_count() + size_t max_bucket_count() + size_t bucket_size(size_t) + size_t bucket(const T&) + # C++20 + bint contains(const T&) + + cdef cppclass unordered_multiset[T,HASH=*,PRED=*,ALLOCATOR=*]: + ctypedef T value_type + + # these should really be allocator_type.size_type and + # allocator_type.difference_type to be true to the C++ definition + # but cython doesn't support deferred access on template arguments + ctypedef size_t size_type + ctypedef ptrdiff_t difference_type + + cppclass const_iterator + cppclass iterator: + iterator() except + + iterator(iterator&) except + + value_type& operator*() + iterator operator++() + iterator operator++(int) + bint operator==(iterator) + bint operator==(const_iterator) + bint operator!=(iterator) + bint operator!=(const_iterator) + cppclass const_iterator: + const_iterator() except + + const_iterator(iterator&) except + + operator=(iterator&) except + + const value_type& operator*() + const_iterator operator++() + const_iterator operator++(int) + bint operator==(iterator) + bint operator==(const_iterator) + bint operator!=(iterator) + bint operator!=(const_iterator) + + unordered_multiset() except + + unordered_multiset(unordered_multiset&) except + + #unordered_multiset& operator=(unordered_multiset&) + bint operator==(unordered_multiset&, unordered_multiset&) + bint operator!=(unordered_multiset&, unordered_multiset&) + iterator begin() + const_iterator const_begin "begin"() + const_iterator cbegin() + void clear() + size_t count(const T&) + bint empty() + iterator end() + const_iterator const_end "end"() + const_iterator cend() + pair[iterator, iterator] equal_range(const T&) + pair[const_iterator, const_iterator] const_equal_range "equal_range"(const T&) + iterator erase(iterator) + iterator const_erase "erase"(const_iterator) + iterator erase(const_iterator, const_iterator) + size_t erase(const T&) + iterator find(const T&) + const_iterator const_find "find"(const T&) + iterator insert(const T&) except + + iterator insert(const_iterator, const T&) except + + void insert[InputIt](InputIt, InputIt) except + + size_t max_size() + size_t size() + void swap(unordered_multiset&) + #value_compare value_comp() + void max_load_factor(float) + float max_load_factor() + float load_factor() void rehash(size_t) void reserve(size_t) size_t bucket_count() size_t max_bucket_count() size_t bucket_size(size_t) size_t bucket(const T&) + # C++20 + bint contains(const T&) diff --git a/Cython/Includes/libcpp/vector.pxd b/Cython/Includes/libcpp/vector.pxd index 9b007dd0c..3def8a568 100644 --- a/Cython/Includes/libcpp/vector.pxd +++ b/Cython/Includes/libcpp/vector.pxd @@ -9,41 +9,114 @@ cdef extern from "<vector>" namespace "std" nogil: ctypedef size_t size_type ctypedef ptrdiff_t difference_type + cppclass const_iterator cppclass iterator: + iterator() except + + iterator(iterator&) except + T& operator*() iterator operator++() iterator operator--() + iterator operator++(int) + iterator operator--(int) iterator operator+(size_type) iterator operator-(size_type) difference_type operator-(iterator) + difference_type operator-(const_iterator) bint operator==(iterator) + bint operator==(const_iterator) bint operator!=(iterator) + bint operator!=(const_iterator) bint operator<(iterator) + bint operator<(const_iterator) bint operator>(iterator) + bint operator>(const_iterator) bint operator<=(iterator) + bint operator<=(const_iterator) bint operator>=(iterator) + bint operator>=(const_iterator) + cppclass const_iterator: + const_iterator() except + + const_iterator(iterator&) except + + const_iterator(const_iterator&) except + + operator=(iterator&) except + + const T& operator*() + const_iterator operator++() + const_iterator operator--() + const_iterator operator++(int) + const_iterator operator--(int) + const_iterator operator+(size_type) + const_iterator operator-(size_type) + difference_type operator-(iterator) + difference_type operator-(const_iterator) + bint operator==(iterator) + bint operator==(const_iterator) + bint operator!=(iterator) + bint operator!=(const_iterator) + bint operator<(iterator) + bint operator<(const_iterator) + bint operator>(iterator) + bint operator>(const_iterator) + bint operator<=(iterator) + bint operator<=(const_iterator) + bint operator>=(iterator) + bint operator>=(const_iterator) + + cppclass const_reverse_iterator cppclass reverse_iterator: + reverse_iterator() except + + reverse_iterator(reverse_iterator&) except + T& operator*() reverse_iterator operator++() reverse_iterator operator--() + reverse_iterator operator++(int) + reverse_iterator operator--(int) reverse_iterator operator+(size_type) reverse_iterator operator-(size_type) - difference_type operator-(reverse_iterator) + difference_type operator-(iterator) + difference_type operator-(const_iterator) bint operator==(reverse_iterator) + bint operator==(const_reverse_iterator) bint operator!=(reverse_iterator) + bint operator!=(const_reverse_iterator) bint operator<(reverse_iterator) + bint operator<(const_reverse_iterator) bint operator>(reverse_iterator) + bint operator>(const_reverse_iterator) bint operator<=(reverse_iterator) + bint operator<=(const_reverse_iterator) bint operator>=(reverse_iterator) - cppclass const_iterator(iterator): - pass - cppclass const_reverse_iterator(reverse_iterator): - pass + bint operator>=(const_reverse_iterator) + cppclass const_reverse_iterator: + const_reverse_iterator() except + + const_reverse_iterator(reverse_iterator&) except + + operator=(reverse_iterator&) except + + const T& operator*() + const_reverse_iterator operator++() + const_reverse_iterator operator--() + const_reverse_iterator operator++(int) + const_reverse_iterator operator--(int) + const_reverse_iterator operator+(size_type) + const_reverse_iterator operator-(size_type) + difference_type operator-(iterator) + difference_type operator-(const_iterator) + bint operator==(reverse_iterator) + bint operator==(const_reverse_iterator) + bint operator!=(reverse_iterator) + bint operator!=(const_reverse_iterator) + bint operator<(reverse_iterator) + bint operator<(const_reverse_iterator) + bint operator>(reverse_iterator) + bint operator>(const_reverse_iterator) + bint operator<=(reverse_iterator) + bint operator<=(const_reverse_iterator) + bint operator>=(reverse_iterator) + bint operator>=(const_reverse_iterator) + vector() except + vector(vector&) except + vector(size_type) except + vector(size_type, T&) except + - #vector[input_iterator](input_iterator, input_iterator) + #vector[InputIt](InputIt, InputIt) T& operator[](size_type) #vector& operator=(vector&) bint operator==(vector&, vector&) @@ -53,30 +126,34 @@ cdef extern from "<vector>" namespace "std" nogil: bint operator<=(vector&, vector&) bint operator>=(vector&, vector&) void assign(size_type, const T&) - void assign[input_iterator](input_iterator, input_iterator) except + + void assign[InputIt](InputIt, InputIt) except + T& at(size_type) except + T& back() iterator begin() const_iterator const_begin "begin"() + const_iterator cbegin() size_type capacity() void clear() bint empty() iterator end() const_iterator const_end "end"() + const_iterator cend() iterator erase(iterator) iterator erase(iterator, iterator) T& front() iterator insert(iterator, const T&) except + iterator insert(iterator, size_type, const T&) except + - iterator insert[Iter](iterator, Iter, Iter) except + + iterator insert[InputIt](iterator, InputIt, InputIt) except + size_type max_size() void pop_back() void push_back(T&) except + reverse_iterator rbegin() - const_reverse_iterator const_rbegin "crbegin"() + const_reverse_iterator const_rbegin "rbegin"() + const_reverse_iterator crbegin() reverse_iterator rend() - const_reverse_iterator const_rend "crend"() - void reserve(size_type) + const_reverse_iterator const_rend "rend"() + const_reverse_iterator crend() + void reserve(size_type) except + void resize(size_type) except + void resize(size_type, T&) except + size_type size() @@ -85,4 +162,6 @@ cdef extern from "<vector>" namespace "std" nogil: # C++11 methods T* data() const T* const_data "data"() - void shrink_to_fit() + void shrink_to_fit() except + + iterator emplace(const_iterator, ...) except + + T& emplace_back(...) except + diff --git a/Cython/Includes/numpy/__init__.pxd b/Cython/Includes/numpy/__init__.pxd index 15700c05e..3d9aff71b 100644 --- a/Cython/Includes/numpy/__init__.pxd +++ b/Cython/Includes/numpy/__init__.pxd @@ -1,30 +1,31 @@ # NumPy static imports for Cython # -# If any of the PyArray_* functions are called, import_array must be -# called first. -# -# This also defines backwards-compatibility buffer acquisition -# code for use in Python 2.x (or Python <= 2.5 when NumPy starts -# implementing PEP-3118 directly). +# NOTE: Do not make incompatible local changes to this file without contacting the NumPy project. +# This file is maintained by the NumPy project at +# https://github.com/numpy/numpy/tree/master/numpy # -# Because of laziness, the format string of the buffer is statically -# allocated. Increase the size if this is not enough, or submit a -# patch to do this properly. +# If any of the PyArray_* functions are called, import_array must be +# called first. This is done automatically by Cython 3.0+ if a call +# is not detected inside of the module. # # Author: Dag Sverre Seljebotn # -DEF _buffer_format_string_len = 255 - -cimport cpython.buffer as pybuf from cpython.ref cimport Py_INCREF -from cpython.mem cimport PyObject_Malloc, PyObject_Free -from cpython.object cimport PyObject, PyTypeObject -from cpython.type cimport type +from cpython.object cimport PyObject, PyTypeObject, PyObject_TypeCheck cimport libc.stdio as stdio + +cdef extern from *: + # Leave a marker that the NumPy declarations came from Cython and not from NumPy itself. + # See https://github.com/cython/cython/issues/3573 + """ + /* Using NumPy API declarations from "Cython/Includes/numpy/" */ + """ + + cdef extern from "Python.h": - ctypedef int Py_intptr_t + ctypedef Py_ssize_t Py_intptr_t cdef extern from "numpy/arrayobject.h": ctypedef Py_intptr_t npy_intp @@ -219,7 +220,7 @@ cdef extern from "numpy/arrayobject.h": cdef int type_num cdef int itemsize "elsize" cdef int alignment - cdef dict fields + cdef object fields cdef tuple names # Use PyDataType_HASSUBARRAY to test whether this field is # valid (the pointer can be NULL). Most users should access @@ -242,104 +243,56 @@ cdef extern from "numpy/arrayobject.h": ctypedef class numpy.ndarray [object PyArrayObject, check_size ignore]: cdef __cythonbufferdefaults__ = {"mode": "strided"} - cdef: - # Only taking a few of the most commonly used and stable fields. - # One should use PyArray_* macros instead to access the C fields. - char *data - int ndim "nd" - npy_intp *shape "dimensions" - npy_intp *strides - dtype descr # deprecated since NumPy 1.7 ! - PyObject* base - - # Note: This syntax (function definition in pxd files) is an - # experimental exception made for __getbuffer__ and __releasebuffer__ - # -- the details of this may change. - def __getbuffer__(ndarray self, Py_buffer* info, int flags): - # This implementation of getbuffer is geared towards Cython - # requirements, and does not yet fulfill the PEP. - # In particular strided access is always provided regardless - # of flags - - cdef int i, ndim - cdef int endian_detector = 1 - cdef bint little_endian = ((<char*>&endian_detector)[0] != 0) - - ndim = PyArray_NDIM(self) - - if ((flags & pybuf.PyBUF_C_CONTIGUOUS == pybuf.PyBUF_C_CONTIGUOUS) - and not PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)): - raise ValueError(u"ndarray is not C contiguous") - - if ((flags & pybuf.PyBUF_F_CONTIGUOUS == pybuf.PyBUF_F_CONTIGUOUS) - and not PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)): - raise ValueError(u"ndarray is not Fortran contiguous") - - info.buf = PyArray_DATA(self) - info.ndim = ndim - if sizeof(npy_intp) != sizeof(Py_ssize_t): - # Allocate new buffer for strides and shape info. - # This is allocated as one block, strides first. - info.strides = <Py_ssize_t*>PyObject_Malloc(sizeof(Py_ssize_t) * 2 * <size_t>ndim) - info.shape = info.strides + ndim - for i in range(ndim): - info.strides[i] = PyArray_STRIDES(self)[i] - info.shape[i] = PyArray_DIMS(self)[i] - else: - info.strides = <Py_ssize_t*>PyArray_STRIDES(self) - info.shape = <Py_ssize_t*>PyArray_DIMS(self) - info.suboffsets = NULL - info.itemsize = PyArray_ITEMSIZE(self) - info.readonly = not PyArray_ISWRITEABLE(self) - - cdef int t - cdef char* f = NULL - cdef dtype descr = <dtype>PyArray_DESCR(self) - cdef int offset - - info.obj = self - - if not PyDataType_HASFIELDS(descr): - t = descr.type_num - if ((descr.byteorder == c'>' and little_endian) or - (descr.byteorder == c'<' and not little_endian)): - raise ValueError(u"Non-native byte order not supported") - if t == NPY_BYTE: f = "b" - elif t == NPY_UBYTE: f = "B" - elif t == NPY_SHORT: f = "h" - elif t == NPY_USHORT: f = "H" - elif t == NPY_INT: f = "i" - elif t == NPY_UINT: f = "I" - elif t == NPY_LONG: f = "l" - elif t == NPY_ULONG: f = "L" - elif t == NPY_LONGLONG: f = "q" - elif t == NPY_ULONGLONG: f = "Q" - elif t == NPY_FLOAT: f = "f" - elif t == NPY_DOUBLE: f = "d" - elif t == NPY_LONGDOUBLE: f = "g" - elif t == NPY_CFLOAT: f = "Zf" - elif t == NPY_CDOUBLE: f = "Zd" - elif t == NPY_CLONGDOUBLE: f = "Zg" - elif t == NPY_OBJECT: f = "O" - else: - raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) - info.format = f - return - else: - info.format = <char*>PyObject_Malloc(_buffer_format_string_len) - info.format[0] = c'^' # Native data types, manual alignment - offset = 0 - f = _util_dtypestring(descr, info.format + 1, - info.format + _buffer_format_string_len, - &offset) - f[0] = c'\0' # Terminate format string - - def __releasebuffer__(ndarray self, Py_buffer* info): - if PyArray_HASFIELDS(self): - PyObject_Free(info.format) - if sizeof(npy_intp) != sizeof(Py_ssize_t): - PyObject_Free(info.strides) - # info.shape was stored after info.strides in the same block + # NOTE: no field declarations since direct access is deprecated since NumPy 1.7 + # Instead, we use properties that map to the corresponding C-API functions. + + @property + cdef inline PyObject* base(self) nogil: + """Returns a borrowed reference to the object owning the data/memory. + """ + return PyArray_BASE(self) + + @property + cdef inline dtype descr(self): + """Returns an owned reference to the dtype of the array. + """ + return <dtype>PyArray_DESCR(self) + + @property + cdef inline int ndim(self) nogil: + """Returns the number of dimensions in the array. + """ + return PyArray_NDIM(self) + + @property + cdef inline npy_intp *shape(self) nogil: + """Returns a pointer to the dimensions/shape of the array. + The number of elements matches the number of dimensions of the array (ndim). + Can return NULL for 0-dimensional arrays. + """ + return PyArray_DIMS(self) + + @property + cdef inline npy_intp *strides(self) nogil: + """Returns a pointer to the strides of the array. + The number of elements matches the number of dimensions of the array (ndim). + """ + return PyArray_STRIDES(self) + + @property + cdef inline npy_intp size(self) nogil: + """Returns the total size (in number of elements) of the array. + """ + return PyArray_SIZE(self) + + @property + cdef inline char* data(self) nogil: + """The pointer to the data buffer as a char*. + This is provided for legacy reasons to avoid direct struct field access. + For new code that needs this access, you probably want to cast the result + of `PyArray_DATA()` instead, which returns a 'void*'. + """ + return PyArray_BYTES(self) ctypedef unsigned char npy_bool @@ -416,103 +369,110 @@ cdef extern from "numpy/arrayobject.h": int len int _import_array() except -1 + # A second definition so _import_array isn't marked as used when we use it here. + # Do not use - subject to change any time. + int __pyx_import_array "_import_array"() except -1 # # Macros from ndarrayobject.h # - bint PyArray_CHKFLAGS(ndarray m, int flags) - bint PyArray_IS_C_CONTIGUOUS(ndarray arr) - bint PyArray_IS_F_CONTIGUOUS(ndarray arr) - bint PyArray_ISCONTIGUOUS(ndarray m) - bint PyArray_ISWRITEABLE(ndarray m) - bint PyArray_ISALIGNED(ndarray m) - - int PyArray_NDIM(ndarray) - bint PyArray_ISONESEGMENT(ndarray) - bint PyArray_ISFORTRAN(ndarray) - int PyArray_FORTRANIF(ndarray) - - void* PyArray_DATA(ndarray) - char* PyArray_BYTES(ndarray) - npy_intp* PyArray_DIMS(ndarray) - npy_intp* PyArray_STRIDES(ndarray) - npy_intp PyArray_DIM(ndarray, size_t) - npy_intp PyArray_STRIDE(ndarray, size_t) - - PyObject *PyArray_BASE(ndarray) # returns borrowed reference! - PyArray_Descr *PyArray_DESCR(ndarray) # returns borrowed reference to dtype! - int PyArray_FLAGS(ndarray) - npy_intp PyArray_ITEMSIZE(ndarray) - int PyArray_TYPE(ndarray arr) + bint PyArray_CHKFLAGS(ndarray m, int flags) nogil + bint PyArray_IS_C_CONTIGUOUS(ndarray arr) nogil + bint PyArray_IS_F_CONTIGUOUS(ndarray arr) nogil + bint PyArray_ISCONTIGUOUS(ndarray m) nogil + bint PyArray_ISWRITEABLE(ndarray m) nogil + bint PyArray_ISALIGNED(ndarray m) nogil + + int PyArray_NDIM(ndarray) nogil + bint PyArray_ISONESEGMENT(ndarray) nogil + bint PyArray_ISFORTRAN(ndarray) nogil + int PyArray_FORTRANIF(ndarray) nogil + + void* PyArray_DATA(ndarray) nogil + char* PyArray_BYTES(ndarray) nogil + + npy_intp* PyArray_DIMS(ndarray) nogil + npy_intp* PyArray_STRIDES(ndarray) nogil + npy_intp PyArray_DIM(ndarray, size_t) nogil + npy_intp PyArray_STRIDE(ndarray, size_t) nogil + + PyObject *PyArray_BASE(ndarray) nogil # returns borrowed reference! + PyArray_Descr *PyArray_DESCR(ndarray) nogil # returns borrowed reference to dtype! + PyArray_Descr *PyArray_DTYPE(ndarray) nogil # returns borrowed reference to dtype! NP 1.7+ alias for descr. + int PyArray_FLAGS(ndarray) nogil + void PyArray_CLEARFLAGS(ndarray, int flags) nogil # Added in NumPy 1.7 + void PyArray_ENABLEFLAGS(ndarray, int flags) nogil # Added in NumPy 1.7 + npy_intp PyArray_ITEMSIZE(ndarray) nogil + int PyArray_TYPE(ndarray arr) nogil object PyArray_GETITEM(ndarray arr, void *itemptr) int PyArray_SETITEM(ndarray arr, void *itemptr, object obj) - bint PyTypeNum_ISBOOL(int) - bint PyTypeNum_ISUNSIGNED(int) - bint PyTypeNum_ISSIGNED(int) - bint PyTypeNum_ISINTEGER(int) - bint PyTypeNum_ISFLOAT(int) - bint PyTypeNum_ISNUMBER(int) - bint PyTypeNum_ISSTRING(int) - bint PyTypeNum_ISCOMPLEX(int) - bint PyTypeNum_ISPYTHON(int) - bint PyTypeNum_ISFLEXIBLE(int) - bint PyTypeNum_ISUSERDEF(int) - bint PyTypeNum_ISEXTENDED(int) - bint PyTypeNum_ISOBJECT(int) - - bint PyDataType_ISBOOL(dtype) - bint PyDataType_ISUNSIGNED(dtype) - bint PyDataType_ISSIGNED(dtype) - bint PyDataType_ISINTEGER(dtype) - bint PyDataType_ISFLOAT(dtype) - bint PyDataType_ISNUMBER(dtype) - bint PyDataType_ISSTRING(dtype) - bint PyDataType_ISCOMPLEX(dtype) - bint PyDataType_ISPYTHON(dtype) - bint PyDataType_ISFLEXIBLE(dtype) - bint PyDataType_ISUSERDEF(dtype) - bint PyDataType_ISEXTENDED(dtype) - bint PyDataType_ISOBJECT(dtype) - bint PyDataType_HASFIELDS(dtype) - bint PyDataType_HASSUBARRAY(dtype) - - bint PyArray_ISBOOL(ndarray) - bint PyArray_ISUNSIGNED(ndarray) - bint PyArray_ISSIGNED(ndarray) - bint PyArray_ISINTEGER(ndarray) - bint PyArray_ISFLOAT(ndarray) - bint PyArray_ISNUMBER(ndarray) - bint PyArray_ISSTRING(ndarray) - bint PyArray_ISCOMPLEX(ndarray) - bint PyArray_ISPYTHON(ndarray) - bint PyArray_ISFLEXIBLE(ndarray) - bint PyArray_ISUSERDEF(ndarray) - bint PyArray_ISEXTENDED(ndarray) - bint PyArray_ISOBJECT(ndarray) - bint PyArray_HASFIELDS(ndarray) - - bint PyArray_ISVARIABLE(ndarray) - - bint PyArray_SAFEALIGNEDCOPY(ndarray) - bint PyArray_ISNBO(char) # works on ndarray.byteorder - bint PyArray_IsNativeByteOrder(char) # works on ndarray.byteorder - bint PyArray_ISNOTSWAPPED(ndarray) - bint PyArray_ISBYTESWAPPED(ndarray) - - bint PyArray_FLAGSWAP(ndarray, int) - - bint PyArray_ISCARRAY(ndarray) - bint PyArray_ISCARRAY_RO(ndarray) - bint PyArray_ISFARRAY(ndarray) - bint PyArray_ISFARRAY_RO(ndarray) - bint PyArray_ISBEHAVED(ndarray) - bint PyArray_ISBEHAVED_RO(ndarray) - - - bint PyDataType_ISNOTSWAPPED(dtype) - bint PyDataType_ISBYTESWAPPED(dtype) + bint PyTypeNum_ISBOOL(int) nogil + bint PyTypeNum_ISUNSIGNED(int) nogil + bint PyTypeNum_ISSIGNED(int) nogil + bint PyTypeNum_ISINTEGER(int) nogil + bint PyTypeNum_ISFLOAT(int) nogil + bint PyTypeNum_ISNUMBER(int) nogil + bint PyTypeNum_ISSTRING(int) nogil + bint PyTypeNum_ISCOMPLEX(int) nogil + bint PyTypeNum_ISPYTHON(int) nogil + bint PyTypeNum_ISFLEXIBLE(int) nogil + bint PyTypeNum_ISUSERDEF(int) nogil + bint PyTypeNum_ISEXTENDED(int) nogil + bint PyTypeNum_ISOBJECT(int) nogil + + bint PyDataType_ISBOOL(dtype) nogil + bint PyDataType_ISUNSIGNED(dtype) nogil + bint PyDataType_ISSIGNED(dtype) nogil + bint PyDataType_ISINTEGER(dtype) nogil + bint PyDataType_ISFLOAT(dtype) nogil + bint PyDataType_ISNUMBER(dtype) nogil + bint PyDataType_ISSTRING(dtype) nogil + bint PyDataType_ISCOMPLEX(dtype) nogil + bint PyDataType_ISPYTHON(dtype) nogil + bint PyDataType_ISFLEXIBLE(dtype) nogil + bint PyDataType_ISUSERDEF(dtype) nogil + bint PyDataType_ISEXTENDED(dtype) nogil + bint PyDataType_ISOBJECT(dtype) nogil + bint PyDataType_HASFIELDS(dtype) nogil + bint PyDataType_HASSUBARRAY(dtype) nogil + + bint PyArray_ISBOOL(ndarray) nogil + bint PyArray_ISUNSIGNED(ndarray) nogil + bint PyArray_ISSIGNED(ndarray) nogil + bint PyArray_ISINTEGER(ndarray) nogil + bint PyArray_ISFLOAT(ndarray) nogil + bint PyArray_ISNUMBER(ndarray) nogil + bint PyArray_ISSTRING(ndarray) nogil + bint PyArray_ISCOMPLEX(ndarray) nogil + bint PyArray_ISPYTHON(ndarray) nogil + bint PyArray_ISFLEXIBLE(ndarray) nogil + bint PyArray_ISUSERDEF(ndarray) nogil + bint PyArray_ISEXTENDED(ndarray) nogil + bint PyArray_ISOBJECT(ndarray) nogil + bint PyArray_HASFIELDS(ndarray) nogil + + bint PyArray_ISVARIABLE(ndarray) nogil + + bint PyArray_SAFEALIGNEDCOPY(ndarray) nogil + bint PyArray_ISNBO(char) nogil # works on ndarray.byteorder + bint PyArray_IsNativeByteOrder(char) nogil # works on ndarray.byteorder + bint PyArray_ISNOTSWAPPED(ndarray) nogil + bint PyArray_ISBYTESWAPPED(ndarray) nogil + + bint PyArray_FLAGSWAP(ndarray, int) nogil + + bint PyArray_ISCARRAY(ndarray) nogil + bint PyArray_ISCARRAY_RO(ndarray) nogil + bint PyArray_ISFARRAY(ndarray) nogil + bint PyArray_ISFARRAY_RO(ndarray) nogil + bint PyArray_ISBEHAVED(ndarray) nogil + bint PyArray_ISBEHAVED_RO(ndarray) nogil + + + bint PyDataType_ISNOTSWAPPED(dtype) nogil + bint PyDataType_ISBYTESWAPPED(dtype) nogil bint PyArray_DescrCheck(object) @@ -532,10 +492,11 @@ cdef extern from "numpy/arrayobject.h": bint PyArray_IsPythonScalar(object) bint PyArray_IsAnyScalar(object) bint PyArray_CheckAnyScalar(object) + ndarray PyArray_GETCONTIGUOUS(ndarray) - bint PyArray_SAMESHAPE(ndarray, ndarray) - npy_intp PyArray_SIZE(ndarray) - npy_intp PyArray_NBYTES(ndarray) + bint PyArray_SAMESHAPE(ndarray, ndarray) nogil + npy_intp PyArray_SIZE(ndarray) nogil + npy_intp PyArray_NBYTES(ndarray) nogil object PyArray_FROM_O(object) object PyArray_FROM_OF(object m, int flags) @@ -548,16 +509,16 @@ cdef extern from "numpy/arrayobject.h": npy_intp PyArray_REFCOUNT(object) object PyArray_ContiguousFromAny(op, int, int min_depth, int max_depth) unsigned char PyArray_EquivArrTypes(ndarray a1, ndarray a2) - bint PyArray_EquivByteorders(int b1, int b2) + bint PyArray_EquivByteorders(int b1, int b2) nogil object PyArray_SimpleNew(int nd, npy_intp* dims, int typenum) object PyArray_SimpleNewFromData(int nd, npy_intp* dims, int typenum, void* data) #object PyArray_SimpleNewFromDescr(int nd, npy_intp* dims, dtype descr) object PyArray_ToScalar(void* data, ndarray arr) - void* PyArray_GETPTR1(ndarray m, npy_intp i) - void* PyArray_GETPTR2(ndarray m, npy_intp i, npy_intp j) - void* PyArray_GETPTR3(ndarray m, npy_intp i, npy_intp j, npy_intp k) - void* PyArray_GETPTR4(ndarray m, npy_intp i, npy_intp j, npy_intp k, npy_intp l) + void* PyArray_GETPTR1(ndarray m, npy_intp i) nogil + void* PyArray_GETPTR2(ndarray m, npy_intp i, npy_intp j) nogil + void* PyArray_GETPTR3(ndarray m, npy_intp i, npy_intp j, npy_intp k) nogil + void* PyArray_GETPTR4(ndarray m, npy_intp i, npy_intp j, npy_intp k, npy_intp l) nogil void PyArray_XDECREF_ERR(ndarray) # Cannot be supported due to out arg @@ -685,7 +646,7 @@ cdef extern from "numpy/arrayobject.h": object PyArray_Choose (ndarray, object, ndarray, NPY_CLIPMODE) int PyArray_Sort (ndarray, int, NPY_SORTKIND) object PyArray_ArgSort (ndarray, int, NPY_SORTKIND) - object PyArray_SearchSorted (ndarray, object, NPY_SEARCHSIDE, PyObject*) + object PyArray_SearchSorted (ndarray, object, NPY_SEARCHSIDE, PyObject *) object PyArray_ArgMax (ndarray, int, ndarray) object PyArray_ArgMin (ndarray, int, ndarray) object PyArray_Reshape (ndarray, object) @@ -838,72 +799,67 @@ cdef inline tuple PyDataType_SHAPE(dtype d): else: return () -cdef inline char* _util_dtypestring(dtype descr, char* f, char* end, int* offset) except NULL: - # Recursive utility function used in __getbuffer__ to get format - # string. The new location in the format string is returned. - - cdef dtype child - cdef int endian_detector = 1 - cdef bint little_endian = ((<char*>&endian_detector)[0] != 0) - cdef tuple fields - - for childname in descr.names: - fields = descr.fields[childname] - child, new_offset = fields - - if (end - f) - <int>(new_offset - offset[0]) < 15: - raise RuntimeError(u"Format string allocated too short, see comment in numpy.pxd") - - if ((child.byteorder == c'>' and little_endian) or - (child.byteorder == c'<' and not little_endian)): - raise ValueError(u"Non-native byte order not supported") - # One could encode it in the format string and have Cython - # complain instead, BUT: < and > in format strings also imply - # standardized sizes for datatypes, and we rely on native in - # order to avoid reencoding data types based on their size. - # - # A proper PEP 3118 exporter for other clients than Cython - # must deal properly with this! - - # Output padding bytes - while offset[0] < new_offset: - f[0] = 120 # "x"; pad byte - f += 1 - offset[0] += 1 - - offset[0] += child.itemsize - - if not PyDataType_HASFIELDS(child): - t = child.type_num - if end - f < 5: - raise RuntimeError(u"Format string allocated too short.") - - # Until ticket #99 is fixed, use integers to avoid warnings - if t == NPY_BYTE: f[0] = 98 #"b" - elif t == NPY_UBYTE: f[0] = 66 #"B" - elif t == NPY_SHORT: f[0] = 104 #"h" - elif t == NPY_USHORT: f[0] = 72 #"H" - elif t == NPY_INT: f[0] = 105 #"i" - elif t == NPY_UINT: f[0] = 73 #"I" - elif t == NPY_LONG: f[0] = 108 #"l" - elif t == NPY_ULONG: f[0] = 76 #"L" - elif t == NPY_LONGLONG: f[0] = 113 #"q" - elif t == NPY_ULONGLONG: f[0] = 81 #"Q" - elif t == NPY_FLOAT: f[0] = 102 #"f" - elif t == NPY_DOUBLE: f[0] = 100 #"d" - elif t == NPY_LONGDOUBLE: f[0] = 103 #"g" - elif t == NPY_CFLOAT: f[0] = 90; f[1] = 102; f += 1 # Zf - elif t == NPY_CDOUBLE: f[0] = 90; f[1] = 100; f += 1 # Zd - elif t == NPY_CLONGDOUBLE: f[0] = 90; f[1] = 103; f += 1 # Zg - elif t == NPY_OBJECT: f[0] = 79 #"O" - else: - raise ValueError(u"unknown dtype code in numpy.pxd (%d)" % t) - f += 1 - else: - # Cython ignores struct boundary information ("T{...}"), - # so don't output it - f = _util_dtypestring(child, f, end, offset) - return f + +cdef extern from "numpy/ndarrayobject.h": + PyTypeObject PyTimedeltaArrType_Type + PyTypeObject PyDatetimeArrType_Type + ctypedef int64_t npy_timedelta + ctypedef int64_t npy_datetime + +cdef extern from "numpy/ndarraytypes.h": + ctypedef struct PyArray_DatetimeMetaData: + NPY_DATETIMEUNIT base + int64_t num + +cdef extern from "numpy/arrayscalars.h": + + # abstract types + ctypedef class numpy.generic [object PyObject]: + pass + ctypedef class numpy.number [object PyObject]: + pass + ctypedef class numpy.integer [object PyObject]: + pass + ctypedef class numpy.signedinteger [object PyObject]: + pass + ctypedef class numpy.unsignedinteger [object PyObject]: + pass + ctypedef class numpy.inexact [object PyObject]: + pass + ctypedef class numpy.floating [object PyObject]: + pass + ctypedef class numpy.complexfloating [object PyObject]: + pass + ctypedef class numpy.flexible [object PyObject]: + pass + ctypedef class numpy.character [object PyObject]: + pass + + ctypedef struct PyDatetimeScalarObject: + # PyObject_HEAD + npy_datetime obval + PyArray_DatetimeMetaData obmeta + + ctypedef struct PyTimedeltaScalarObject: + # PyObject_HEAD + npy_timedelta obval + PyArray_DatetimeMetaData obmeta + + ctypedef enum NPY_DATETIMEUNIT: + NPY_FR_Y + NPY_FR_M + NPY_FR_W + NPY_FR_D + NPY_FR_B + NPY_FR_h + NPY_FR_m + NPY_FR_s + NPY_FR_ms + NPY_FR_us + NPY_FR_ns + NPY_FR_ps + NPY_FR_fs + NPY_FR_as # @@ -1032,7 +988,7 @@ cdef inline object get_array_base(ndarray arr): # Cython code. cdef inline int import_array() except -1: try: - _import_array() + __pyx_import_array() except Exception: raise ImportError("numpy.core.multiarray failed to import") @@ -1047,3 +1003,57 @@ cdef inline int import_ufunc() except -1: _import_umath() except Exception: raise ImportError("numpy.core.umath failed to import") + + +cdef inline bint is_timedelta64_object(object obj): + """ + Cython equivalent of `isinstance(obj, np.timedelta64)` + + Parameters + ---------- + obj : object + + Returns + ------- + bool + """ + return PyObject_TypeCheck(obj, &PyTimedeltaArrType_Type) + + +cdef inline bint is_datetime64_object(object obj): + """ + Cython equivalent of `isinstance(obj, np.datetime64)` + + Parameters + ---------- + obj : object + + Returns + ------- + bool + """ + return PyObject_TypeCheck(obj, &PyDatetimeArrType_Type) + + +cdef inline npy_datetime get_datetime64_value(object obj) nogil: + """ + returns the int64 value underlying scalar numpy datetime64 object + + Note that to interpret this as a datetime, the corresponding unit is + also needed. That can be found using `get_datetime64_unit`. + """ + return (<PyDatetimeScalarObject*>obj).obval + + +cdef inline npy_timedelta get_timedelta64_value(object obj) nogil: + """ + returns the int64 value underlying scalar numpy timedelta64 object + """ + return (<PyTimedeltaScalarObject*>obj).obval + + +cdef inline NPY_DATETIMEUNIT get_datetime64_unit(object obj) nogil: + """ + returns the unit part of the dtype for a numpy datetime64 object. + """ + return <NPY_DATETIMEUNIT>(<PyDatetimeScalarObject*>obj).obmeta.base diff --git a/Cython/Includes/openmp.pxd b/Cython/Includes/openmp.pxd index 30873a588..40f2f17a3 100644 --- a/Cython/Includes/openmp.pxd +++ b/Cython/Includes/openmp.pxd @@ -48,4 +48,3 @@ cdef extern from "<omp.h>": int omp_get_ancestor_thread_num(int) nogil int omp_get_team_size(int) nogil int omp_get_active_level() nogil - diff --git a/Cython/Includes/posix/dlfcn.pxd b/Cython/Includes/posix/dlfcn.pxd index cff5bea15..bf61997f3 100644 --- a/Cython/Includes/posix/dlfcn.pxd +++ b/Cython/Includes/posix/dlfcn.pxd @@ -1,5 +1,5 @@ # POSIX dynamic linking/loading interface. -# http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/dlfcn.h.html +# https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/dlfcn.h.html cdef extern from "<dlfcn.h>" nogil: void *dlopen(const char *, int) diff --git a/Cython/Includes/posix/fcntl.pxd b/Cython/Includes/posix/fcntl.pxd index 9afc33a36..f7bec9e37 100644 --- a/Cython/Includes/posix/fcntl.pxd +++ b/Cython/Includes/posix/fcntl.pxd @@ -1,8 +1,11 @@ -# http://www.opengroup.org/onlinepubs/009695399/basedefs/fcntl.h.html +# https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/fcntl.h.html + +from posix.types cimport mode_t, off_t, pid_t cdef extern from "<fcntl.h>" nogil: enum: F_DUPFD + enum: F_DUPFD_CLOEXEC enum: F_GETFD enum: F_SETFD enum: F_GETFL @@ -23,11 +26,14 @@ cdef extern from "<fcntl.h>" nogil: enum: SEEK_CUR enum: SEEK_END + enum: O_CLOEXEC enum: O_CREAT enum: O_DIRECT + enum: O_DIRECTORY enum: O_EXCL enum: O_NOCTTY enum: O_TRUNC + enum: O_TTY_INIT enum: O_APPEND enum: O_DSYNC @@ -37,9 +43,17 @@ cdef extern from "<fcntl.h>" nogil: enum: O_ACCMODE # O_RDONLY|O_WRONLY|O_RDWR + enum: O_EXEC enum: O_RDONLY enum: O_WRONLY enum: O_RDWR + enum: O_SEARCH + + enum: AT_FDCWD + enum: AT_EACCESS + enum: AT_SYMLINK_NOFOLLOW + enum: AT_SYMLINK_FOLLOW + enum: AT_REMOVEDIR enum: S_IFMT enum: S_IFBLK @@ -50,9 +64,12 @@ cdef extern from "<fcntl.h>" nogil: enum: S_IFLNK enum: S_IFSOCK - ctypedef int mode_t - ctypedef signed pid_t - ctypedef signed off_t + enum: POSIX_FADV_DONTNEED + enum: POSIX_FADV_NOREUSE + enum: POSIX_FADV_NORMAL + enum: POSIX_FADV_RANDOM + enum: POSIX_FADV_SEQUENTIAL + enum: POSIX_FADV_WILLNEED struct flock: short l_type @@ -61,8 +78,9 @@ cdef extern from "<fcntl.h>" nogil: off_t l_len pid_t l_pid - int creat(char *, mode_t) + int creat(const char *, mode_t) int fcntl(int, int, ...) - int open(char *, int, ...) - #int open (char *, int, mode_t) - + int open(const char *, int, ...) + int openat(int, const char *, int, ...) + int posix_fadvise(int, off_t, off_t, int) + int posix_fallocate(int, off_t, off_t) diff --git a/Cython/Includes/posix/mman.pxd b/Cython/Includes/posix/mman.pxd index c810f431b..9f26f7615 100644 --- a/Cython/Includes/posix/mman.pxd +++ b/Cython/Includes/posix/mman.pxd @@ -1,4 +1,6 @@ -# http://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/mman.h.html +# https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/sys_mman.h.html +# https://man7.org/linux/man-pages/man2/mmap.2.html +# https://www.freebsd.org/cgi/man.cgi?query=mmap&sektion=2 from posix.types cimport off_t, mode_t diff --git a/Cython/Includes/posix/resource.pxd b/Cython/Includes/posix/resource.pxd index 9f55c6ab4..b9628c66b 100644 --- a/Cython/Includes/posix/resource.pxd +++ b/Cython/Includes/posix/resource.pxd @@ -1,4 +1,5 @@ -# http://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/resource.h.html +# https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/sys_resource.h.html +# https://man7.org/linux/man-pages/man2/getrusage.2.html from posix.time cimport timeval from posix.types cimport id_t @@ -33,6 +34,7 @@ cdef extern from "<sys/resource.h>" nogil: cdef struct rusage: timeval ru_utime timeval ru_stime + # Linux-specific long ru_maxrss long ru_ixrss long ru_idrss diff --git a/Cython/Includes/posix/select.pxd b/Cython/Includes/posix/select.pxd index 46703df10..803c492d4 100644 --- a/Cython/Includes/posix/select.pxd +++ b/Cython/Includes/posix/select.pxd @@ -1,3 +1,5 @@ +# https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/sys_select.h.html + from .types cimport sigset_t from .time cimport timeval, timespec @@ -12,7 +14,7 @@ cdef extern from "<sys/select.h>" nogil: void FD_ZERO(fd_set*) int select(int nfds, fd_set *readfds, fd_set *writefds, - fd_set *exceptfds, const timeval *timeout) + fd_set *exceptfds, timeval *timeout) int pselect(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, const timespec *timeout, diff --git a/Cython/Includes/posix/stat.pxd b/Cython/Includes/posix/stat.pxd index 69c2eca16..9247423f8 100644 --- a/Cython/Includes/posix/stat.pxd +++ b/Cython/Includes/posix/stat.pxd @@ -1,5 +1,9 @@ +# https://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/stat.h.html +# https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/sys_stat.h.html + from posix.types cimport (blkcnt_t, blksize_t, dev_t, gid_t, ino_t, mode_t, nlink_t, off_t, time_t, uid_t) +from posix.time cimport timespec cdef extern from "<sys/stat.h>" nogil: @@ -14,9 +18,14 @@ cdef extern from "<sys/stat.h>" nogil: off_t st_size blksize_t st_blksize blkcnt_t st_blocks + # POSIX.1-2001 time_t st_atime time_t st_mtime time_t st_ctime + # POSIX.1-2008 + timespec st_atim + timespec st_mtim + timespec st_ctim # st_birthtime exists on *BSD and OS X. # Under Linux, defining it here does not hurt. Compilation under Linux @@ -25,12 +34,24 @@ cdef extern from "<sys/stat.h>" nogil: # POSIX prescribes including both <sys/stat.h> and <unistd.h> for these cdef extern from "<unistd.h>" nogil: - int fchmod(int, mode_t) int chmod(const char *, mode_t) + int fchmod(int, mode_t) + int fchmodat(int, const char *, mode_t, int flags) - int fstat(int, struct_stat *) - int lstat(const char *, struct_stat *) int stat(const char *, struct_stat *) + int lstat(const char *, struct_stat *) + int fstat(int, struct_stat *) + int fstatat(int, const char *, struct_stat *, int flags) + + int mkdir(const char *, mode_t) + int mkdirat(int, const char *, mode_t) + int mkfifo(const char *, mode_t) + int mkfifoat(int, const char *, mode_t) + int mknod(const char *, mode_t, dev_t) + int mknodat(int, const char *, mode_t, dev_t) + + int futimens(int, const timespec *) + int utimensat(int, const char *, const timespec *, int flags) # Macros for st_mode mode_t S_ISREG(mode_t) @@ -69,3 +90,9 @@ cdef extern from "<unistd.h>" nogil: mode_t S_IROTH mode_t S_IWOTH mode_t S_IXOTH + + # test file types + bint S_TYPEISMQ(struct_stat *buf) + bint S_TYPEISSEM(struct_stat *buf) + bint S_TYPEISSHM(struct_stat *buf) + bint S_TYPEISTMO(struct_stat *buf) diff --git a/Cython/Includes/posix/stdio.pxd b/Cython/Includes/posix/stdio.pxd index 53913fdf4..38b815559 100644 --- a/Cython/Includes/posix/stdio.pxd +++ b/Cython/Includes/posix/stdio.pxd @@ -1,5 +1,5 @@ # POSIX additions to <stdio.h>. -# http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/stdio.h.html +# https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/stdio.h.html from libc.stdio cimport FILE from libc.stddef cimport wchar_t diff --git a/Cython/Includes/posix/stdlib.pxd b/Cython/Includes/posix/stdlib.pxd index 513de938a..188e2e501 100644 --- a/Cython/Includes/posix/stdlib.pxd +++ b/Cython/Includes/posix/stdlib.pxd @@ -1,5 +1,5 @@ # POSIX additions to <stdlib.h> -# http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/stdlib.h.html +# https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/stdlib.h.html cdef extern from "<stdlib.h>" nogil: void _Exit(int) diff --git a/Cython/Includes/posix/time.pxd b/Cython/Includes/posix/time.pxd index 6bc81bfea..a90cab577 100644 --- a/Cython/Includes/posix/time.pxd +++ b/Cython/Includes/posix/time.pxd @@ -1,4 +1,4 @@ -# http://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/time.h.html +# https://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/time.h.html from posix.types cimport suseconds_t, time_t, clockid_t, timer_t from posix.signal cimport sigevent diff --git a/Cython/Includes/posix/uio.pxd b/Cython/Includes/posix/uio.pxd new file mode 100644 index 000000000..d9971bd4a --- /dev/null +++ b/Cython/Includes/posix/uio.pxd @@ -0,0 +1,26 @@ +# https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/sys_uio.h.html + +from posix.types cimport off_t + + +cdef extern from "<sys/uio.h>" nogil: + + cdef struct iovec: + void *iov_base + size_t iov_len + + ssize_t readv (int fd, const iovec *iov, int iovcnt) + ssize_t writev(int fd, const iovec *iov, int iovcnt) + + # Linux-specific, https://man7.org/linux/man-pages/man2/readv.2.html + ssize_t preadv (int fd, const iovec *iov, int iovcnt, off_t offset) + ssize_t pwritev(int fd, const iovec *iov, int iovcnt, off_t offset) + + enum: RWF_DSYNC + enum: RWF_HIPRI + enum: RWF_SYNC + enum: RWF_NOWAIT + enum: RWF_APPEND + + ssize_t preadv2 (int fd, const iovec *iov, int iovcnt, off_t offset, int flags) + ssize_t pwritev2(int fd, const iovec *iov, int iovcnt, off_t offset, int flags) diff --git a/Cython/Includes/posix/wait.pxd b/Cython/Includes/posix/wait.pxd index d18cff9cf..f30be06df 100644 --- a/Cython/Includes/posix/wait.pxd +++ b/Cython/Includes/posix/wait.pxd @@ -1,4 +1,4 @@ -# http://pubs.opengroup.org/onlinepubs/009695399/basedefs/sys/wait.h.html +# https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/sys_wait.h.html from posix.types cimport pid_t, id_t from posix.signal cimport siginfo_t diff --git a/Cython/Parser/ConcreteSyntaxTree.pyx b/Cython/Parser/ConcreteSyntaxTree.pyx index f9888c561..90969fee7 100644 --- a/Cython/Parser/ConcreteSyntaxTree.pyx +++ b/Cython/Parser/ConcreteSyntaxTree.pyx @@ -32,13 +32,14 @@ import re def extract_names(path): # All parse tree types are #defined in these files as ints. type_names = {} - for line in open(path): - if line.startswith('#define'): - try: - _, name, value = line.strip().split() - type_names[int(value)] = name - except: - pass + with open(path) as fid: + for line in fid: + if line.startswith('#define'): + try: + _, name, value = line.strip().split() + type_names[int(value)] = name + except: + pass return type_names cdef dict type_names = {} @@ -61,7 +62,8 @@ def handle_includes(source, path): included = os.path.join(os.path.dirname(path), include_line.group(1)[1:-1]) if not os.path.exists(included): return include_line.group(0) + ' # no such path: ' + included - return handle_includes(open(included).read(), path) + with open(included) as fid: + return handle_includes(fid.read(), path) # TODO: Proper string tokenizing. return re.sub(r'^include\s+([^\n]+[\'"])\s*(#.*)?$', include_here, source, flags=re.M) @@ -69,7 +71,8 @@ def p_module(path): cdef perrdetail err cdef int flags cdef node* n - source = open(path).read() + with open(path) as fid: + source = fid.read() if '\ninclude ' in source: # TODO: Tokanizer needs to understand includes. source = handle_includes(source, path) diff --git a/Cython/Plex/Actions.pxd b/Cython/Plex/Actions.pxd index 34660a2d9..cd884ced8 100644 --- a/Cython/Plex/Actions.pxd +++ b/Cython/Plex/Actions.pxd @@ -1,25 +1,26 @@ +# cython: language_level=3 cdef class Action: - cdef perform(self, token_stream, text) - cpdef same_as(self, other) + cdef perform(self, token_stream, text) cdef class Return(Action): - cdef object value - cdef perform(self, token_stream, text) - cpdef same_as(self, other) + cdef object value + cdef perform(self, token_stream, text) cdef class Call(Action): - cdef object function - cdef perform(self, token_stream, text) - cpdef same_as(self, other) + cdef object function + cdef perform(self, token_stream, text) + +cdef class Method(Action): + cdef str name + cdef dict kwargs cdef class Begin(Action): - cdef object state_name - cdef perform(self, token_stream, text) - cpdef same_as(self, other) + cdef object state_name + cdef perform(self, token_stream, text) cdef class Ignore(Action): - cdef perform(self, token_stream, text) + cdef perform(self, token_stream, text) cdef class Text(Action): - cdef perform(self, token_stream, text) + cdef perform(self, token_stream, text) diff --git a/Cython/Plex/Actions.py b/Cython/Plex/Actions.py index c88176e71..725278ddf 100644 --- a/Cython/Plex/Actions.py +++ b/Cython/Plex/Actions.py @@ -1,18 +1,20 @@ +# cython: language_level=3str # cython: auto_pickle=False -#======================================================================= -# -# Python Lexical Analyser -# -# Actions for use in token specifications -# -#======================================================================= +""" +Python Lexical Analyser + +Actions for use in token specifications +""" class Action(object): def perform(self, token_stream, text): pass # abstract - def same_as(self, other): - return self is other + def __copy__(self): + return self # immutable, no need to copy + + def __deepcopy__(self, memo): + return self # immutable, no need to copy class Return(Action): @@ -27,11 +29,8 @@ class Return(Action): def perform(self, token_stream, text): return self.value - def same_as(self, other): - return isinstance(other, Return) and self.value == other.value - def __repr__(self): - return "Return(%s)" % repr(self.value) + return "Return(%r)" % self.value class Call(Action): @@ -48,8 +47,27 @@ class Call(Action): def __repr__(self): return "Call(%s)" % self.function.__name__ - def same_as(self, other): - return isinstance(other, Call) and self.function is other.function + +class Method(Action): + """ + Plex action that calls a specific method on the token stream, + passing the matched text and any provided constant keyword arguments. + """ + + def __init__(self, name, **kwargs): + self.name = name + self.kwargs = kwargs or None + + def perform(self, token_stream, text): + method = getattr(token_stream, self.name) + # self.kwargs is almost always unused => avoid call overhead + return method(text, **self.kwargs) if self.kwargs is not None else method(text) + + def __repr__(self): + kwargs = ( + ', '.join(sorted(['%s=%r' % item for item in self.kwargs.items()])) + if self.kwargs is not None else '') + return "Method(%s%s%s)" % (self.name, ', ' if kwargs else '', kwargs) class Begin(Action): @@ -68,9 +86,6 @@ class Begin(Action): def __repr__(self): return "Begin(%s)" % self.state_name - def same_as(self, other): - return isinstance(other, Begin) and self.state_name == other.state_name - class Ignore(Action): """ @@ -87,7 +102,6 @@ class Ignore(Action): IGNORE = Ignore() -#IGNORE.__doc__ = Ignore.__doc__ class Text(Action): @@ -105,6 +119,3 @@ class Text(Action): TEXT = Text() -#TEXT.__doc__ = Text.__doc__ - - diff --git a/Cython/Plex/DFA.pxd b/Cython/Plex/DFA.pxd new file mode 100644 index 000000000..226d07567 --- /dev/null +++ b/Cython/Plex/DFA.pxd @@ -0,0 +1,30 @@ +# cython: auto_pickle=False + +cimport cython + +from . cimport Machines +from .Transitions cimport TransitionMap + + +@cython.final +cdef class StateMap: + cdef Machines.FastMachine new_machine + cdef dict old_to_new_dict + cdef dict new_to_old_dict + + cdef old_to_new(self, dict old_state_set) + + @cython.locals(state=Machines.Node) + cdef highest_priority_action(self, dict state_set) + + cdef make_key(self, dict state_set) + + +@cython.locals(new_machine=Machines.FastMachine, transitions=TransitionMap) +cpdef nfa_to_dfa(Machines.Machine old_machine, debug=*) + +cdef set_epsilon_closure(dict state_set) +cdef dict epsilon_closure(Machines.Node state) + +@cython.locals(state_set_2=dict, state2=Machines.Node) +cdef add_to_epsilon_closure(dict state_set, Machines.Node state) diff --git a/Cython/Plex/DFA.py b/Cython/Plex/DFA.py index 76324621f..66dc4a379 100644 --- a/Cython/Plex/DFA.py +++ b/Cython/Plex/DFA.py @@ -1,11 +1,9 @@ -#======================================================================= -# -# Python Lexical Analyser -# -# Converting NFA to DFA -# -#======================================================================= +# cython: auto_cpdef=True +""" +Python Lexical Analyser +Converting NFA to DFA +""" from __future__ import absolute_import from . import Machines @@ -29,12 +27,14 @@ def nfa_to_dfa(old_machine, debug=None): # is reached. new_machine = Machines.FastMachine() state_map = StateMap(new_machine) + # Seed the process using the initial states of the old machine. # Make the corresponding new states into initial states of the new # machine with the same names. for (key, old_state) in old_machine.initial_states.items(): new_state = state_map.old_to_new(epsilon_closure(old_state)) new_machine.make_initial_state(key, new_state) + # Tricky bit here: we add things to the end of this list while we're # iterating over it. The iteration stops when closure is achieved. for new_state in new_machine.states: @@ -45,6 +45,7 @@ def nfa_to_dfa(old_machine, debug=None): transitions.add_set(event, set_epsilon_closure(old_target_states)) for event, old_states in transitions.items(): new_machine.add_transitions(new_state, event, state_map.old_to_new(old_states)) + if debug: debug.write("\n===== State Mapping =====\n") state_map.dump(debug) @@ -95,14 +96,11 @@ class StateMap(object): Helper class used by nfa_to_dfa() to map back and forth between sets of states from the old machine and states of the new machine. """ - new_machine = None # Machine - old_to_new_dict = None # {(old_state,...) : new_state} - new_to_old_dict = None # {id(new_state) : old_state_set} def __init__(self, new_machine): - self.new_machine = new_machine - self.old_to_new_dict = {} - self.new_to_old_dict = {} + self.new_machine = new_machine # Machine + self.old_to_new_dict = {} # {(old_state,...) : new_state} + self.new_to_old_dict = {} # {id(new_state) : old_state_set} def old_to_new(self, old_state_set): """ @@ -119,8 +117,6 @@ class StateMap(object): new_state = self.new_machine.new_state(action) self.old_to_new_dict[key] = new_state self.new_to_old_dict[id(new_state)] = old_state_set - #for old_state in old_state_set.keys(): - #new_state.merge_actions(old_state) return new_state def highest_priority_action(self, state_set): @@ -133,13 +129,6 @@ class StateMap(object): best_priority = priority return best_action - # def old_to_new_set(self, old_state_set): - # """ - # Return the new state corresponding to a set of old states as - # a singleton set. - # """ - # return {self.old_to_new(old_state_set):1} - def new_to_old(self, new_state): """Given a new state, return a set of corresponding old states.""" return self.new_to_old_dict[id(new_state)] @@ -149,9 +138,7 @@ class StateMap(object): Convert a set of states into a uniquified sorted tuple suitable for use as a dictionary key. """ - lst = list(state_set) - lst.sort() - return tuple(lst) + return tuple(sorted(state_set)) def dump(self, file): from .Transitions import state_set_str @@ -160,5 +147,3 @@ class StateMap(object): old_state_set = self.new_to_old_dict[id(new_state)] file.write(" State %s <-- %s\n" % ( new_state['number'], state_set_str(old_state_set))) - - diff --git a/Cython/Plex/Errors.py b/Cython/Plex/Errors.py index f460100d7..fa10374f8 100644 --- a/Cython/Plex/Errors.py +++ b/Cython/Plex/Errors.py @@ -1,10 +1,8 @@ -#======================================================================= -# -# Python Lexical Analyser -# -# Exception classes -# -#======================================================================= +""" +Python Lexical Analyser + +Exception classes +""" class PlexError(Exception): @@ -19,10 +17,6 @@ class PlexValueError(PlexError, ValueError): pass -class InvalidRegex(PlexError): - pass - - class InvalidToken(PlexError): def __init__(self, token_number, message): PlexError.__init__(self, "Token number %d: %s" % (token_number, message)) diff --git a/Cython/Plex/Lexicons.py b/Cython/Plex/Lexicons.py index 787f5854b..438e44bda 100644 --- a/Cython/Plex/Lexicons.py +++ b/Cython/Plex/Lexicons.py @@ -1,15 +1,10 @@ -#======================================================================= -# -# Python Lexical Analyser -# -# Lexical Analyser Specification -# -#======================================================================= +""" +Python Lexical Analyser +Lexical Analyser Specification +""" from __future__ import absolute_import -import types - from . import Actions from . import DFA from . import Errors @@ -114,17 +109,14 @@ class Lexicon(object): machine = None # Machine tables = None # StateTableMachine - def __init__(self, specifications, debug=None, debug_flags=7, timings=None): + def __init__(self, specifications, debug=None, debug_flags=7): if not isinstance(specifications, list): raise Errors.InvalidScanner("Scanner definition is not a list") - if timings: - from .Timing import time - total_time = 0.0 - time1 = time() nfa = Machines.Machine() default_initial_state = nfa.new_initial_state('') token_number = 1 + for spec in specifications: if isinstance(spec, State): user_initial_state = nfa.new_initial_state(spec.name) @@ -140,33 +132,22 @@ class Lexicon(object): raise Errors.InvalidToken( token_number, "Expected a token definition (tuple) or State instance") - if timings: - time2 = time() - total_time = total_time + (time2 - time1) - time3 = time() + if debug and (debug_flags & 1): debug.write("\n============= NFA ===========\n") nfa.dump(debug) + dfa = DFA.nfa_to_dfa(nfa, debug=(debug_flags & 3) == 3 and debug) - if timings: - time4 = time() - total_time = total_time + (time4 - time3) + if debug and (debug_flags & 2): debug.write("\n============= DFA ===========\n") dfa.dump(debug) - if timings: - timings.write("Constructing NFA : %5.2f\n" % (time2 - time1)) - timings.write("Converting to DFA: %5.2f\n" % (time4 - time3)) - timings.write("TOTAL : %5.2f\n" % total_time) + self.machine = dfa def add_token_to_machine(self, machine, initial_state, token_spec, token_number): try: (re, action_spec) = self.parse_token_definition(token_spec) - # Disabled this -- matching empty strings can be useful - #if re.nullable: - # raise Errors.InvalidToken( - # token_number, "Pattern can match 0 input symbols") if isinstance(action_spec, Actions.Action): action = action_spec else: @@ -188,6 +169,7 @@ class Lexicon(object): raise Errors.InvalidToken("Token definition is not a tuple") if len(token_spec) != 2: raise Errors.InvalidToken("Wrong number of items in token definition") + pattern, action = token_spec if not isinstance(pattern, Regexps.RE): raise Errors.InvalidToken("Pattern is not an RE instance") @@ -195,6 +177,3 @@ class Lexicon(object): def get_initial_state(self, name): return self.machine.get_initial_state(name) - - - diff --git a/Cython/Plex/Machines.pxd b/Cython/Plex/Machines.pxd new file mode 100644 index 000000000..13b43a234 --- /dev/null +++ b/Cython/Plex/Machines.pxd @@ -0,0 +1,33 @@ +cimport cython + +from .Actions cimport Action +from .Transitions cimport TransitionMap + +cdef int maxint + + +@cython.final +cdef class Machine: + cdef readonly list states + cdef readonly dict initial_states + cdef readonly Py_ssize_t next_state_number + + cpdef new_state(self) + cpdef new_initial_state(self, name) + + +@cython.final +cdef class Node: + cdef readonly TransitionMap transitions + cdef readonly Action action + cdef public dict epsilon_closure + cdef readonly Py_ssize_t number + cdef readonly int action_priority + + +@cython.final +cdef class FastMachine: + cdef readonly dict initial_states + cdef readonly dict new_state_template + cdef readonly list states + cdef readonly Py_ssize_t next_number diff --git a/Cython/Plex/Machines.py b/Cython/Plex/Machines.py index 398850976..77b65c1a9 100644 --- a/Cython/Plex/Machines.py +++ b/Cython/Plex/Machines.py @@ -1,42 +1,33 @@ -#======================================================================= -# -# Python Lexical Analyser -# -# Classes for building NFAs and DFAs -# -#======================================================================= +# cython: auto_pickle=False +""" +Python Lexical Analyser +Classes for building NFAs and DFAs +""" from __future__ import absolute_import -import sys - +import cython from .Transitions import TransitionMap -try: - from sys import maxsize as maxint -except ImportError: - from sys import maxint +maxint = 2**31-1 # sentinel value -try: - unichr -except NameError: - unichr = chr +if not cython.compiled: + try: + unichr + except NameError: + unichr = chr LOWEST_PRIORITY = -maxint class Machine(object): """A collection of Nodes representing an NFA or DFA.""" - states = None # [Node] - next_state_number = 1 - initial_states = None # {(name, bol): Node} - def __init__(self): - self.states = [] - self.initial_states = {} + self.states = [] # [Node] + self.initial_states = {} # {(name, bol): Node} + self.next_state_number = 1 def __del__(self): - #print "Destroying", self ### for state in self.states: state.destroy() @@ -72,21 +63,17 @@ class Machine(object): class Node(object): """A state of an NFA or DFA.""" - transitions = None # TransitionMap - action = None # Action - action_priority = None # integer - number = 0 # for debug output - epsilon_closure = None # used by nfa_to_dfa() def __init__(self): # Preinitialise the list of empty transitions, because # the nfa-to-dfa algorithm needs it - #self.transitions = {'':[]} - self.transitions = TransitionMap() - self.action_priority = LOWEST_PRIORITY + self.transitions = TransitionMap() # TransitionMap + self.action_priority = LOWEST_PRIORITY # integer + self.action = None # Action + self.number = 0 # for debug output + self.epsilon_closure = None # used by nfa_to_dfa() def destroy(self): - #print "Destroying", self ### self.transitions = None self.action = None self.epsilon_closure = None @@ -133,23 +120,23 @@ class Node(object): def __lt__(self, other): return self.number < other.number + def __hash__(self): + # Prevent overflowing hash values due to arbitrarily large unsigned addresses. + return id(self) & maxint + class FastMachine(object): """ FastMachine is a deterministic machine represented in a way that allows fast scanning. """ - initial_states = None # {state_name:state} - states = None # [state] where state = {event:state, 'else':state, 'action':Action} - next_number = 1 # for debugging - - new_state_template = { - '': None, 'bol': None, 'eol': None, 'eof': None, 'else': None - } - def __init__(self): - self.initial_states = {} - self.states = [] + self.initial_states = {} # {state_name:state} + self.states = [] # [state] where state = {event:state, 'else':state, 'action':Action} + self.next_number = 1 # for debugging + self.new_state_template = { + '': None, 'bol': None, 'eol': None, 'eof': None, 'else': None + } def __del__(self): for state in self.states: @@ -167,6 +154,7 @@ class FastMachine(object): def make_initial_state(self, name, state): self.initial_states[name] = state + @cython.locals(code0=cython.int, code1=cython.int, maxint=cython.int, state=dict) def add_transitions(self, state, event, new_state, maxint=maxint): if type(event) is tuple: code0, code1 = event @@ -218,9 +206,7 @@ class FastMachine(object): if char_list: ranges = self.chars_to_ranges(char_list) ranges_to_state[ranges] = state - ranges_list = ranges_to_state.keys() - ranges_list.sort() - for ranges in ranges_list: + for ranges in sorted(ranges_to_state): key = self.ranges_to_string(ranges) state = ranges_to_state[ranges] file.write(" %s --> State %d\n" % (key, state['number'])) @@ -229,6 +215,7 @@ class FastMachine(object): if state: file.write(" %s --> State %d\n" % (key, state['number'])) + @cython.locals(char_list=list, i=cython.Py_ssize_t, n=cython.Py_ssize_t, c1=cython.long, c2=cython.long) def chars_to_ranges(self, char_list): char_list.sort() i = 0 diff --git a/Cython/Plex/Regexps.py b/Cython/Plex/Regexps.py index 41816c939..99d8c994a 100644 --- a/Cython/Plex/Regexps.py +++ b/Cython/Plex/Regexps.py @@ -1,21 +1,16 @@ -#======================================================================= -# -# Python Lexical Analyser -# -# Regular Expressions -# -#======================================================================= +""" +Python Lexical Analyser +Regular Expressions +""" from __future__ import absolute_import import types -try: - from sys import maxsize as maxint -except ImportError: - from sys import maxint from . import Errors +maxint = 2**31-1 # sentinel value + # # Constants # @@ -186,37 +181,6 @@ class RE(object): # These are the basic REs from which all others are built. # -## class Char(RE): -## """ -## Char(c) is an RE which matches the character |c|. -## """ - -## nullable = 0 - -## def __init__(self, char): -## self.char = char -## self.match_nl = char == '\n' - -## def build_machine(self, m, initial_state, final_state, match_bol, nocase): -## c = self.char -## if match_bol and c != BOL: -## s1 = self.build_opt(m, initial_state, BOL) -## else: -## s1 = initial_state -## if c == '\n' or c == EOF: -## s1 = self.build_opt(m, s1, EOL) -## if len(c) == 1: -## code = ord(self.char) -## s1.add_transition((code, code+1), final_state) -## if nocase and is_letter_code(code): -## code2 = other_case_code(code) -## s1.add_transition((code2, code2+1), final_state) -## else: -## s1.add_transition(c, final_state) - -## def calc_str(self): -## return "Char(%s)" % repr(self.char) - def Char(c): """ @@ -428,6 +392,7 @@ class SwitchCase(RE): name = "Case" return "%s(%s)" % (name, self.re) + # # Composite RE constructors # ------------------------- @@ -469,7 +434,6 @@ def Any(s): """ Any(s) is an RE which matches any character in the string |s|. """ - #result = apply(Alt, tuple(map(Char, s))) result = CodeRanges(chars_to_ranges(s)) result.str = "Any(%s)" % repr(s) return result @@ -549,6 +513,7 @@ def Case(re): """ return SwitchCase(re, nocase=0) + # # RE Constants # @@ -573,4 +538,3 @@ Eof.__doc__ = \ Eof is an RE which matches the end of the file. """ Eof.str = "Eof" - diff --git a/Cython/Plex/Scanners.pxd b/Cython/Plex/Scanners.pxd index 6e75f55e6..664b1a6f0 100644 --- a/Cython/Plex/Scanners.pxd +++ b/Cython/Plex/Scanners.pxd @@ -16,8 +16,8 @@ cdef class Scanner: cdef public Py_ssize_t cur_line cdef public Py_ssize_t cur_line_start cdef public Py_ssize_t start_pos - cdef public Py_ssize_t start_line - cdef public Py_ssize_t start_col + cdef tuple current_scanner_position_tuple + cdef public tuple last_token_position_tuple cdef public text cdef public initial_state # int? cdef public state_name @@ -28,13 +28,13 @@ cdef class Scanner: cdef public level - @cython.final @cython.locals(input_state=long) - cdef next_char(self) + cdef inline next_char(self) @cython.locals(action=Action) cpdef tuple read(self) - @cython.final - cdef tuple scan_a_token(self) + cdef inline unread(self, token, value, position) + cdef inline get_current_scan_pos(self) + cdef inline tuple scan_a_token(self) ##cdef tuple position(self) # used frequently by Parsing.py @cython.final @@ -44,7 +44,5 @@ cdef class Scanner: trace=bint, discard=Py_ssize_t, data=unicode, buffer=unicode) cdef run_machine_inlined(self) - @cython.final - cdef begin(self, state) - @cython.final - cdef produce(self, value, text = *) + cdef inline begin(self, state) + cdef inline produce(self, value, text = *) diff --git a/Cython/Plex/Scanners.py b/Cython/Plex/Scanners.py index 88f7e2da3..5729e3a3f 100644 --- a/Cython/Plex/Scanners.py +++ b/Cython/Plex/Scanners.py @@ -1,18 +1,15 @@ +# cython: language_level=3str # cython: auto_pickle=False -#======================================================================= -# -# Python Lexical Analyser -# -# -# Scanning an input stream -# -#======================================================================= +""" +Python Lexical Analyser +Scanning an input stream +""" from __future__ import absolute_import import cython -cython.declare(BOL=object, EOL=object, EOF=object, NOT_FOUND=object) +cython.declare(BOL=object, EOL=object, EOF=object, NOT_FOUND=object) # noqa:E402 from . import Errors from .Regexps import BOL, EOL, EOF @@ -56,18 +53,25 @@ class Scanner(object): # stream = None # file-like object # name = '' # buffer = '' + # + # These positions are used by the scanner to track its internal state: # buf_start_pos = 0 # position in input of start of buffer # next_pos = 0 # position in input of next char to read # cur_pos = 0 # position in input of current char # cur_line = 1 # line number of current char # cur_line_start = 0 # position in input of start of current line # start_pos = 0 # position in input of start of token - # start_line = 0 # line number of start of token - # start_col = 0 # position in line of start of token + # current_scanner_position_tuple = ("", 0, 0) + # tuple of filename, line number and position in line, really mainly for error reporting + # + # These positions are used to track what was read from the queue + # (which may differ from the internal state when tokens are replaced onto the queue) + # last_token_position_tuple = ("", 0, 0) # tuple of filename, line number and position in line + # text = None # text of last token read # initial_state = None # Node # state_name = '' # Name of initial state - # queue = None # list of tokens to be returned + # queue = None # list of tokens and positions to be returned # trace = 0 def __init__(self, lexicon, stream, name='', initial_pos=None): @@ -91,8 +95,8 @@ class Scanner(object): self.cur_pos = 0 self.cur_line = 1 self.start_pos = 0 - self.start_line = 0 - self.start_col = 0 + self.current_scanner_position_tuple = ("", 0, 0) + self.last_token_position_tuple = ("", 0, 0) self.text = None self.state_name = None @@ -127,10 +131,17 @@ class Scanner(object): value = action.perform(self, self.text) if value is not None: self.produce(value) - result = queue[0] + result, self.last_token_position_tuple = queue[0] del queue[0] return result + def unread(self, token, value, position): + self.queue.insert(0, ((token, value), position)) + + def get_current_scan_pos(self): + # distinct from the position of the last token due to the queue + return self.current_scanner_position_tuple + def scan_a_token(self): """ Read the next input sequence recognised by the machine @@ -138,8 +149,9 @@ class Scanner(object): file. """ self.start_pos = self.cur_pos - self.start_line = self.cur_line - self.start_col = self.cur_pos - self.cur_line_start + self.current_scanner_position_tuple = ( + self.name, self.cur_line, self.cur_pos - self.cur_line_start + ) action = self.run_machine_inlined() if action is not None: if self.trace: @@ -173,26 +185,28 @@ class Scanner(object): buf_len = len(buffer) b_action, b_cur_pos, b_cur_line, b_cur_line_start, b_cur_char, b_input_state, b_next_pos = \ None, 0, 0, 0, u'', 0, 0 + trace = self.trace while 1: - if trace: #TRACE# - print("State %d, %d/%d:%s -->" % ( #TRACE# - state['number'], input_state, cur_pos, repr(cur_char))) #TRACE# + if trace: + print("State %d, %d/%d:%s -->" % ( + state['number'], input_state, cur_pos, repr(cur_char))) + # Begin inlined self.save_for_backup() - #action = state.action #@slow - action = state['action'] #@fast + action = state['action'] if action is not None: b_action, b_cur_pos, b_cur_line, b_cur_line_start, b_cur_char, b_input_state, b_next_pos = \ action, cur_pos, cur_line, cur_line_start, cur_char, input_state, next_pos # End inlined self.save_for_backup() + c = cur_char - #new_state = state.new_state(c) #@slow - new_state = state.get(c, NOT_FOUND) #@fast - if new_state is NOT_FOUND: #@fast - new_state = c and state.get('else') #@fast + new_state = state.get(c, NOT_FOUND) + if new_state is NOT_FOUND: + new_state = c and state.get('else') + if new_state: - if trace: #TRACE# - print("State %d" % new_state['number']) #TRACE# + if trace: + print("State %d" % new_state['number']) state = new_state # Begin inlined: self.next_char() if input_state == 1: @@ -240,8 +254,8 @@ class Scanner(object): cur_char = u'' # End inlined self.next_char() else: # not new_state - if trace: #TRACE# - print("blocked") #TRACE# + if trace: + print("blocked") # Begin inlined: action = self.back_up() if b_action is not None: (action, cur_pos, cur_line, cur_line_start, @@ -252,15 +266,16 @@ class Scanner(object): action = None break # while 1 # End inlined: action = self.back_up() + self.cur_pos = cur_pos self.cur_line = cur_line self.cur_line_start = cur_line_start self.cur_char = cur_char self.input_state = input_state self.next_pos = next_pos - if trace: #TRACE# - if action is not None: #TRACE# - print("Doing %s" % action) #TRACE# + if trace: + if action is not None: + print("Doing %s" % action) return action def next_char(self): @@ -303,10 +318,11 @@ class Scanner(object): position within the line of the first character of the token (0-based). """ - return (self.name, self.start_line, self.start_col) + return self.last_token_position_tuple def get_position(self): - """Python accessible wrapper around position(), only for error reporting. + """ + Python accessible wrapper around position(), only for error reporting. """ return self.position() @@ -329,10 +345,15 @@ class Scanner(object): """ if text is None: text = self.text - self.queue.append((value, text)) + self.queue.append(((value, text), self.current_scanner_position_tuple)) def eof(self): """ Override this method if you want something to be done at end of file. """ + pass + + @property + def start_line(self): + return self.last_token_position_tuple[1] diff --git a/Cython/Plex/Timing.py b/Cython/Plex/Timing.py deleted file mode 100644 index 5c3692693..000000000 --- a/Cython/Plex/Timing.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Get time in platform-dependent way -# - -from __future__ import absolute_import - -import os -from sys import platform, exit, stderr - -if platform == 'mac': - import MacOS - def time(): - return MacOS.GetTicks() / 60.0 - timekind = "real" -elif hasattr(os, 'times'): - def time(): - t = os.times() - return t[0] + t[1] - timekind = "cpu" -else: - stderr.write( - "Don't know how to get time on platform %s\n" % repr(platform)) - exit(1) diff --git a/Cython/Plex/Traditional.py b/Cython/Plex/Traditional.py deleted file mode 100644 index ec7252dae..000000000 --- a/Cython/Plex/Traditional.py +++ /dev/null @@ -1,158 +0,0 @@ -#======================================================================= -# -# Python Lexical Analyser -# -# Traditional Regular Expression Syntax -# -#======================================================================= - -from __future__ import absolute_import - -from .Regexps import Alt, Seq, Rep, Rep1, Opt, Any, AnyBut, Bol, Eol, Char -from .Errors import PlexError - - -class RegexpSyntaxError(PlexError): - pass - - -def re(s): - """ - Convert traditional string representation of regular expression |s| - into Plex representation. - """ - return REParser(s).parse_re() - - -class REParser(object): - def __init__(self, s): - self.s = s - self.i = -1 - self.end = 0 - self.next() - - def parse_re(self): - re = self.parse_alt() - if not self.end: - self.error("Unexpected %s" % repr(self.c)) - return re - - def parse_alt(self): - """Parse a set of alternative regexps.""" - re = self.parse_seq() - if self.c == '|': - re_list = [re] - while self.c == '|': - self.next() - re_list.append(self.parse_seq()) - re = Alt(*re_list) - return re - - def parse_seq(self): - """Parse a sequence of regexps.""" - re_list = [] - while not self.end and not self.c in "|)": - re_list.append(self.parse_mod()) - return Seq(*re_list) - - def parse_mod(self): - """Parse a primitive regexp followed by *, +, ? modifiers.""" - re = self.parse_prim() - while not self.end and self.c in "*+?": - if self.c == '*': - re = Rep(re) - elif self.c == '+': - re = Rep1(re) - else: # self.c == '?' - re = Opt(re) - self.next() - return re - - def parse_prim(self): - """Parse a primitive regexp.""" - c = self.get() - if c == '.': - re = AnyBut("\n") - elif c == '^': - re = Bol - elif c == '$': - re = Eol - elif c == '(': - re = self.parse_alt() - self.expect(')') - elif c == '[': - re = self.parse_charset() - self.expect(']') - else: - if c == '\\': - c = self.get() - re = Char(c) - return re - - def parse_charset(self): - """Parse a charset. Does not include the surrounding [].""" - char_list = [] - invert = 0 - if self.c == '^': - invert = 1 - self.next() - if self.c == ']': - char_list.append(']') - self.next() - while not self.end and self.c != ']': - c1 = self.get() - if self.c == '-' and self.lookahead(1) != ']': - self.next() - c2 = self.get() - for a in range(ord(c1), ord(c2) + 1): - char_list.append(chr(a)) - else: - char_list.append(c1) - chars = ''.join(char_list) - if invert: - return AnyBut(chars) - else: - return Any(chars) - - def next(self): - """Advance to the next char.""" - s = self.s - i = self.i = self.i + 1 - if i < len(s): - self.c = s[i] - else: - self.c = '' - self.end = 1 - - def get(self): - if self.end: - self.error("Premature end of string") - c = self.c - self.next() - return c - - def lookahead(self, n): - """Look ahead n chars.""" - j = self.i + n - if j < len(self.s): - return self.s[j] - else: - return '' - - def expect(self, c): - """ - Expect to find character |c| at current position. - Raises an exception otherwise. - """ - if self.c == c: - self.next() - else: - self.error("Missing %s" % repr(c)) - - def error(self, mess): - """Raise exception to signal syntax error in regexp.""" - raise RegexpSyntaxError("Syntax error in regexp %s at position %d: %s" % ( - repr(self.s), self.i, mess)) - - - diff --git a/Cython/Plex/Transitions.pxd b/Cython/Plex/Transitions.pxd new file mode 100644 index 000000000..53dd4d58e --- /dev/null +++ b/Cython/Plex/Transitions.pxd @@ -0,0 +1,22 @@ +cimport cython + +cdef long maxint + +@cython.final +cdef class TransitionMap: + cdef list map + cdef dict special + + @cython.locals(i=cython.Py_ssize_t, j=cython.Py_ssize_t) + cpdef add(self, event, new_state) + + @cython.locals(i=cython.Py_ssize_t, j=cython.Py_ssize_t) + cpdef add_set(self, event, new_set) + + @cython.locals(i=cython.Py_ssize_t, n=cython.Py_ssize_t, else_set=cython.bint) + cpdef iteritems(self) + + @cython.locals(map=list, lo=cython.Py_ssize_t, mid=cython.Py_ssize_t, hi=cython.Py_ssize_t) + cdef split(self, long code) + + cdef get_special(self, event) diff --git a/Cython/Plex/Transitions.py b/Cython/Plex/Transitions.py index 383381794..f58dd538e 100644 --- a/Cython/Plex/Transitions.py +++ b/Cython/Plex/Transitions.py @@ -1,15 +1,11 @@ -# -# Plex - Transition Maps -# -# This version represents state sets directly as dicts for speed. -# +# cython: auto_pickle=False +""" +Plex - Transition Maps -from __future__ import absolute_import +This version represents state sets directly as dicts for speed. +""" -try: - from sys import maxsize as maxint -except ImportError: - from sys import maxint +maxint = 2**31-1 # sentinel value class TransitionMap(object): @@ -40,24 +36,19 @@ class TransitionMap(object): kept separately in a dictionary. """ - map = None # The list of codes and states - special = None # Mapping for special events - def __init__(self, map=None, special=None): if not map: map = [-maxint, {}, maxint] if not special: special = {} - self.map = map - self.special = special - #self.check() ### + self.map = map # The list of codes and states + self.special = special # Mapping for special events - def add(self, event, new_state, - TupleType=tuple): + def add(self, event, new_state): """ Add transition to |new_state| on |event|. """ - if type(event) is TupleType: + if type(event) is tuple: code0, code1 = event i = self.split(code0) j = self.split(code1) @@ -68,12 +59,11 @@ class TransitionMap(object): else: self.get_special(event)[new_state] = 1 - def add_set(self, event, new_set, - TupleType=tuple): + def add_set(self, event, new_set): """ Add transitions to the states in |new_set| on |event|. """ - if type(event) is TupleType: + if type(event) is tuple: code0, code1 = event i = self.split(code0) j = self.split(code1) @@ -84,15 +74,13 @@ class TransitionMap(object): else: self.get_special(event).update(new_set) - def get_epsilon(self, - none=None): + def get_epsilon(self): """ Return the mapping for epsilon, or None. """ - return self.special.get('', none) + return self.special.get('') - def iteritems(self, - len=len): + def iteritems(self): """ Return the mapping as an iterable of ((code1, code2), state_set) and (special_event, state_set) pairs. @@ -119,8 +107,7 @@ class TransitionMap(object): # ------------------- Private methods -------------------- - def split(self, code, - len=len, maxint=maxint): + def split(self, code): """ Search the list for the position of the split point for |code|, inserting a new split point if necessary. Returns index |i| such @@ -132,6 +119,7 @@ class TransitionMap(object): # Special case: code == map[-1] if code == maxint: return hi + # General case lo = 0 # loop invariant: map[lo] <= code < map[hi] and hi - lo >= 2 @@ -147,7 +135,6 @@ class TransitionMap(object): return lo else: map[hi:hi] = [code, map[hi - 1].copy()] - #self.check() ### return hi def get_special(self, event): @@ -243,9 +230,5 @@ class TransitionMap(object): # State set manipulation functions # -#def merge_state_sets(set1, set2): -# for state in set2.keys(): -# set1[state] = 1 - def state_set_str(set): return "[%s]" % ','.join(["S%d" % state.number for state in set]) diff --git a/Cython/Plex/__init__.py b/Cython/Plex/__init__.py index 81a066f78..83bb9239a 100644 --- a/Cython/Plex/__init__.py +++ b/Cython/Plex/__init__.py @@ -1,10 +1,6 @@ -#======================================================================= -# -# Python Lexical Analyser -# -#======================================================================= - """ +Python Lexical Analyser + The Plex module provides lexical analysers with similar capabilities to GNU Flex. The following classes and functions are exported; see the attached docstrings for more information. @@ -29,10 +25,10 @@ see the attached docstrings for more information. Actions for associating with patterns when creating a Lexicon. """ - +# flake8: noqa:F401 from __future__ import absolute_import -from .Actions import TEXT, IGNORE, Begin +from .Actions import TEXT, IGNORE, Begin, Method from .Lexicons import Lexicon, State from .Regexps import RE, Seq, Alt, Rep1, Empty, Str, Any, AnyBut, AnyChar, Range from .Regexps import Opt, Rep, Bol, Eol, Eof, Case, NoCase diff --git a/Cython/Runtime/refnanny.pyx b/Cython/Runtime/refnanny.pyx index d4b873fe9..bc72f62c6 100644 --- a/Cython/Runtime/refnanny.pyx +++ b/Cython/Runtime/refnanny.pyx @@ -1,6 +1,6 @@ # cython: language_level=3, auto_pickle=False -from cpython.ref cimport PyObject, Py_INCREF, Py_DECREF, Py_XDECREF, Py_XINCREF +from cpython.ref cimport PyObject, Py_INCREF, Py_CLEAR, Py_XDECREF, Py_XINCREF from cpython.exc cimport PyErr_Fetch, PyErr_Restore from cpython.pystate cimport PyThreadState_Get @@ -10,6 +10,9 @@ loglevel = 0 reflog = [] cdef log(level, action, obj, lineno): + if reflog is None: + # can happen during finalisation + return if loglevel >= level: reflog.append((lineno, action, id(obj))) @@ -29,7 +32,7 @@ cdef class Context(object): self.refs = {} # id -> (count, [lineno]) self.errors = [] - cdef regref(self, obj, lineno, bint is_null): + cdef regref(self, obj, Py_ssize_t lineno, bint is_null): log(LOG_ALL, u'regref', u"<NULL>" if is_null else obj, lineno) if is_null: self.errors.append(f"NULL argument on line {lineno}") @@ -39,7 +42,7 @@ cdef class Context(object): self.refs[id_] = (count + 1, linenumbers) linenumbers.append(lineno) - cdef bint delref(self, obj, lineno, bint is_null) except -1: + cdef bint delref(self, obj, Py_ssize_t lineno, bint is_null) except -1: # returns whether it is ok to do the decref operation log(LOG_ALL, u'delref', u"<NULL>" if is_null else obj, lineno) if is_null: @@ -50,12 +53,11 @@ cdef class Context(object): if count == 0: self.errors.append(f"Too many decrefs on line {lineno}, reference acquired on lines {linenumbers!r}") return False - elif count == 1: + if count == 1: del self.refs[id_] - return True else: self.refs[id_] = (count - 1, linenumbers) - return True + return True cdef end(self): if self.refs: @@ -63,122 +65,118 @@ cdef class Context(object): for count, linenos in self.refs.itervalues(): msg += f"\n ({count}) acquired on lines: {u', '.join([f'{x}' for x in linenos])}" self.errors.append(msg) - if self.errors: - return u"\n".join([u'REFNANNY: '+error for error in self.errors]) - else: - return None + return u"\n".join([f'REFNANNY: {error}' for error in self.errors]) if self.errors else None + -cdef void report_unraisable(object e=None): +cdef void report_unraisable(filename, Py_ssize_t lineno, object e=None): try: if e is None: import sys e = sys.exc_info()[1] - print(f"refnanny raised an exception: {e}") - except: - pass # We absolutely cannot exit with an exception + print(f"refnanny raised an exception from {filename}:{lineno}: {e}") + finally: + return # We absolutely cannot exit with an exception + # All Python operations must happen after any existing # exception has been fetched, in case we are called from # exception-handling code. -cdef PyObject* SetupContext(char* funcname, int lineno, char* filename) except NULL: +cdef PyObject* SetupContext(char* funcname, Py_ssize_t lineno, char* filename) except NULL: if Context is None: # Context may be None during finalize phase. # In that case, we don't want to be doing anything fancy # like caching and resetting exceptions. return NULL cdef (PyObject*) type = NULL, value = NULL, tb = NULL, result = NULL - PyThreadState_Get() + PyThreadState_Get() # Check that we hold the GIL PyErr_Fetch(&type, &value, &tb) try: ctx = Context(funcname, lineno, filename) Py_INCREF(ctx) result = <PyObject*>ctx except Exception, e: - report_unraisable(e) + report_unraisable(filename, lineno, e) PyErr_Restore(type, value, tb) return result -cdef void GOTREF(PyObject* ctx, PyObject* p_obj, int lineno): +cdef void GOTREF(PyObject* ctx, PyObject* p_obj, Py_ssize_t lineno): if ctx == NULL: return cdef (PyObject*) type = NULL, value = NULL, tb = NULL PyErr_Fetch(&type, &value, &tb) try: - try: - if p_obj is NULL: - (<Context>ctx).regref(None, lineno, True) - else: - (<Context>ctx).regref(<object>p_obj, lineno, False) - except: - report_unraisable() + (<Context>ctx).regref( + <object>p_obj if p_obj is not NULL else None, + lineno, + is_null=p_obj is NULL, + ) except: - # __Pyx_GetException may itself raise errors - pass - PyErr_Restore(type, value, tb) + report_unraisable((<Context>ctx).filename, lineno=(<Context>ctx).start) + finally: + PyErr_Restore(type, value, tb) + return # swallow any exceptions -cdef int GIVEREF_and_report(PyObject* ctx, PyObject* p_obj, int lineno): +cdef bint GIVEREF_and_report(PyObject* ctx, PyObject* p_obj, Py_ssize_t lineno): if ctx == NULL: return 1 cdef (PyObject*) type = NULL, value = NULL, tb = NULL cdef bint decref_ok = False PyErr_Fetch(&type, &value, &tb) try: - try: - if p_obj is NULL: - decref_ok = (<Context>ctx).delref(None, lineno, True) - else: - decref_ok = (<Context>ctx).delref(<object>p_obj, lineno, False) - except: - report_unraisable() + decref_ok = (<Context>ctx).delref( + <object>p_obj if p_obj is not NULL else None, + lineno, + is_null=p_obj is NULL, + ) except: - # __Pyx_GetException may itself raise errors - pass - PyErr_Restore(type, value, tb) - return decref_ok + report_unraisable((<Context>ctx).filename, lineno=(<Context>ctx).start) + finally: + PyErr_Restore(type, value, tb) + return decref_ok # swallow any exceptions -cdef void GIVEREF(PyObject* ctx, PyObject* p_obj, int lineno): +cdef void GIVEREF(PyObject* ctx, PyObject* p_obj, Py_ssize_t lineno): GIVEREF_and_report(ctx, p_obj, lineno) -cdef void INCREF(PyObject* ctx, PyObject* obj, int lineno): +cdef void INCREF(PyObject* ctx, PyObject* obj, Py_ssize_t lineno): Py_XINCREF(obj) - PyThreadState_Get() + PyThreadState_Get() # Check that we hold the GIL GOTREF(ctx, obj, lineno) -cdef void DECREF(PyObject* ctx, PyObject* obj, int lineno): +cdef void DECREF(PyObject* ctx, PyObject* obj, Py_ssize_t lineno): if GIVEREF_and_report(ctx, obj, lineno): Py_XDECREF(obj) - PyThreadState_Get() + PyThreadState_Get() # Check that we hold the GIL cdef void FinishContext(PyObject** ctx): if ctx == NULL or ctx[0] == NULL: return cdef (PyObject*) type = NULL, value = NULL, tb = NULL cdef object errors = None cdef Context context - PyThreadState_Get() + PyThreadState_Get() # Check that we hold the GIL PyErr_Fetch(&type, &value, &tb) try: - try: - context = <Context>ctx[0] - errors = context.end() - if errors: - print(f"{context.filename.decode('latin1')}: {context.name.decode('latin1')}()") - print(errors) - context = None - except: - report_unraisable() + context = <Context>ctx[0] + errors = context.end() + if errors: + print(f"{context.filename.decode('latin1')}: {context.name.decode('latin1')}()") + print(errors) + context = None except: - # __Pyx_GetException may itself raise errors - pass - Py_XDECREF(ctx[0]) - ctx[0] = NULL - PyErr_Restore(type, value, tb) + report_unraisable( + context.filename if context is not None else None, + lineno=context.start if context is not None else 0, + ) + finally: + Py_CLEAR(ctx[0]) + PyErr_Restore(type, value, tb) + return # swallow any exceptions ctypedef struct RefNannyAPIStruct: - void (*INCREF)(PyObject*, PyObject*, int) - void (*DECREF)(PyObject*, PyObject*, int) - void (*GOTREF)(PyObject*, PyObject*, int) - void (*GIVEREF)(PyObject*, PyObject*, int) - PyObject* (*SetupContext)(char*, int, char*) except NULL - void (*FinishContext)(PyObject**) + void (*INCREF)(PyObject*, PyObject*, Py_ssize_t) + void (*DECREF)(PyObject*, PyObject*, Py_ssize_t) + void (*GOTREF)(PyObject*, PyObject*, Py_ssize_t) + void (*GIVEREF)(PyObject*, PyObject*, Py_ssize_t) + PyObject* (*SetupContext)(char*, Py_ssize_t, char*) except NULL + void (*FinishContext)(PyObject**) cdef RefNannyAPIStruct api api.INCREF = INCREF diff --git a/Cython/Shadow.py b/Cython/Shadow.py index 96296070e..6371ecb11 100644 --- a/Cython/Shadow.py +++ b/Cython/Shadow.py @@ -1,7 +1,8 @@ # cython.* namespace for pure mode. from __future__ import absolute_import -__version__ = "0.29.34" +# Possible version formats: "3.1.0", "3.1.0a1", "3.1.0a1.dev0" +__version__ = "3.0.0b2" try: from __builtin__ import basestring @@ -71,7 +72,7 @@ def index_type(base_type, item): else: # int[8] etc. assert int(item) == item # array size must be a plain integer - array(base_type, item) + return array(base_type, item) # END shameless copy @@ -107,8 +108,8 @@ class _Optimization(object): cclass = ccall = cfunc = _EmptyDecoratorAndManager() -returns = wraparound = boundscheck = initializedcheck = nonecheck = \ - embedsignature = cdivision = cdivision_warnings = \ +annotation_typing = returns = wraparound = boundscheck = initializedcheck = \ + nonecheck = embedsignature = cdivision = cdivision_warnings = \ always_allows_keywords = profile = linetrace = infer_types = \ unraisable_tracebacks = freelist = \ lambda _: _EmptyDecoratorAndManager() @@ -116,12 +117,12 @@ returns = wraparound = boundscheck = initializedcheck = nonecheck = \ exceptval = lambda _=None, check=True: _EmptyDecoratorAndManager() overflowcheck = lambda _: _EmptyDecoratorAndManager() -optimization = _Optimization() +optimize = _Optimization() -overflowcheck.fold = optimization.use_switch = \ - optimization.unpack_method_calls = lambda arg: _EmptyDecoratorAndManager() +overflowcheck.fold = optimize.use_switch = \ + optimize.unpack_method_calls = lambda arg: _EmptyDecoratorAndManager() -final = internal = type_version_tag = no_gc_clear = no_gc = _empty_decorator +final = internal = type_version_tag = no_gc_clear = no_gc = total_ordering = _empty_decorator binding = lambda _: _empty_decorator @@ -146,27 +147,33 @@ def compile(f): # Special functions def cdiv(a, b): - q = a / b - if q < 0: - q += 1 - return q + if a < 0: + a = -a + b = -b + if b < 0: + return (a + b + 1) // b + return a // b def cmod(a, b): r = a % b - if (a*b) < 0: + if (a * b) < 0 and r: r -= b return r # Emulated language constructs -def cast(type, *args, **kwargs): +def cast(t, *args, **kwargs): kwargs.pop('typecheck', None) assert not kwargs - if hasattr(type, '__call__'): - return type(*args) - else: - return args[0] + + if isinstance(t, typedef): + return t(*args) + elif isinstance(t, type): # Doesn't work with old-style classes of Python 2.x + if len(args) != 1 or not (args[0] is None or isinstance(args[0], t)): + return t(*args) + + return args[0] def sizeof(arg): return 1 @@ -178,14 +185,19 @@ def typeof(arg): def address(arg): return pointer(type(arg))([arg]) -def declare(type=None, value=_Unspecified, **kwds): - if type not in (None, object) and hasattr(type, '__call__'): - if value is not _Unspecified: - return type(value) - else: - return type() +def _is_value_type(t): + if isinstance(t, typedef): + return _is_value_type(t._basetype) + + return isinstance(t, type) and issubclass(t, (StructType, UnionType, ArrayType)) + +def declare(t=None, value=_Unspecified, **kwds): + if value is not _Unspecified: + return cast(t, value) + elif _is_value_type(t): + return t() else: - return value + return None class _nogil(object): """Support for 'with nogil' statement and @nogil decorator. @@ -258,24 +270,45 @@ class PointerType(CythonType): class ArrayType(PointerType): - def __init__(self): - self._items = [None] * self._n + def __init__(self, value=None): + if value is None: + self._items = [None] * self._n + else: + super(ArrayType, self).__init__(value) class StructType(CythonType): - def __init__(self, cast_from=_Unspecified, **data): - if cast_from is not _Unspecified: - # do cast - if len(data) > 0: - raise ValueError('Cannot accept keyword arguments when casting.') - if type(cast_from) is not type(self): - raise ValueError('Cannot cast from %s'%cast_from) - for key, value in cast_from.__dict__.items(): - setattr(self, key, value) + def __init__(self, *posargs, **data): + if not (posargs or data): + return + if posargs and data: + raise ValueError('Cannot accept both positional and keyword arguments.') + + # Allow 'cast_from' as single positional or keyword argument. + if data and len(data) == 1 and 'cast_from' in data: + cast_from = data.pop('cast_from') + elif len(posargs) == 1 and type(posargs[0]) is type(self): + cast_from, posargs = posargs[0], () + elif posargs: + for key, arg in zip(self._members, posargs): + setattr(self, key, arg) + return else: for key, value in data.items(): + if key not in self._members: + raise ValueError("Invalid struct attribute for %s: %s" % ( + self.__class__.__name__, key)) setattr(self, key, value) + return + + # do cast + if data: + raise ValueError('Cannot accept keyword arguments when casting.') + if type(cast_from) is not type(self): + raise ValueError('Cannot cast from %s' % cast_from) + for key, value in cast_from.__dict__.items(): + setattr(self, key, value) def __setattr__(self, key, value): if key in self._members: @@ -296,7 +329,7 @@ class UnionType(CythonType): elif type(cast_from) is type(self): datadict = cast_from.__dict__ else: - raise ValueError('Cannot cast from %s'%cast_from) + raise ValueError('Cannot cast from %s' % cast_from) else: datadict = data if len(datadict) > 1: @@ -393,10 +426,34 @@ py_complex = typedef(complex, "double complex") # Predefined types -int_types = ['char', 'short', 'Py_UNICODE', 'int', 'Py_UCS4', 'long', 'longlong', 'Py_ssize_t', 'size_t'] -float_types = ['longdouble', 'double', 'float'] -complex_types = ['longdoublecomplex', 'doublecomplex', 'floatcomplex', 'complex'] -other_types = ['bint', 'void', 'Py_tss_t'] +int_types = [ + 'char', + 'short', + 'Py_UNICODE', + 'int', + 'Py_UCS4', + 'long', + 'longlong', + 'Py_hash_t', + 'Py_ssize_t', + 'size_t', +] +float_types = [ + 'longdouble', + 'double', + 'float', +] +complex_types = [ + 'longdoublecomplex', + 'doublecomplex', + 'floatcomplex', + 'complex', +] +other_types = [ + 'bint', + 'void', + 'Py_tss_t', +] to_repr = { 'longlong': 'long long', @@ -469,6 +526,53 @@ class CythonDotParallel(object): # def threadsavailable(self): # return 1 -import sys +class CythonDotImportedFromElsewhere(object): + """ + cython.dataclasses just shadows the standard library modules of the same name + """ + def __init__(self, module): + self.__path__ = [] + self.__file__ = None + self.__name__ = module + self.__package__ = module + + def __getattr__(self, attr): + # we typically only expect this to be called once + from importlib import import_module + import sys + try: + mod = import_module(self.__name__) + except ImportError: + # but if they don't exist (Python is not sufficiently up-to-date) then + # you can't use them + raise AttributeError("%s: the standard library module %s is not available" % + (attr, self.__name__)) + sys.modules['cython.%s' % self.__name__] = mod + return getattr(mod, attr) + + +class CythonCImports(object): + """ + Simplistic module mock to make cimports sort-of work in Python code. + """ + def __init__(self, module): + self.__path__ = [] + self.__file__ = None + self.__name__ = module + self.__package__ = module + + def __getattr__(self, item): + if item.startswith('__') and item.endswith('__'): + raise AttributeError(item) + return __import__(item) + + +import math, sys sys.modules['cython.parallel'] = CythonDotParallel() -del sys +sys.modules['cython.cimports'] = CythonCImports('cython.cimports') +sys.modules['cython.cimports.libc'] = CythonCImports('cython.cimports.libc') +sys.modules['cython.cimports.libc.math'] = math +# In pure Python mode @cython.dataclasses.dataclass and dataclass field should just +# shadow the standard library ones (if they are available) +dataclasses = sys.modules['cython.dataclasses'] = CythonDotImportedFromElsewhere('dataclasses') +del math, sys diff --git a/Cython/StringIOTree.pxd b/Cython/StringIOTree.pxd index 20455c9df..49cea78fb 100644 --- a/Cython/StringIOTree.pxd +++ b/Cython/StringIOTree.pxd @@ -1,13 +1,19 @@ cimport cython +cdef object StringIO + +@cython.final cdef class StringIOTree: cdef public list prepended_children cdef public object stream cdef public object write cdef public list markers + cpdef bint empty(self) @cython.locals(x=StringIOTree) cpdef getvalue(self) + @cython.locals(x=StringIOTree) + cdef _collect_in(self, list target_list) @cython.locals(child=StringIOTree) cpdef copyto(self, target) cpdef commit(self) diff --git a/Cython/StringIOTree.py b/Cython/StringIOTree.py index d8239efed..798009758 100644 --- a/Cython/StringIOTree.py +++ b/Cython/StringIOTree.py @@ -23,6 +23,9 @@ EXAMPLE: >>> b.getvalue().split() ['second', 'alpha', 'beta', 'gamma'] +>>> try: from cStringIO import StringIO +... except ImportError: from io import StringIO + >>> i = StringIOTree() >>> d.insert(i) >>> _= i.write('inserted\n') @@ -54,11 +57,23 @@ class StringIOTree(object): self.write = stream.write self.markers = [] + def empty(self): + if self.stream.tell(): + return False + return all([child.empty() for child in self.prepended_children]) if self.prepended_children else True + def getvalue(self): - content = [x.getvalue() for x in self.prepended_children] - content.append(self.stream.getvalue()) + content = [] + self._collect_in(content) return "".join(content) + def _collect_in(self, target_list): + for x in self.prepended_children: + x._collect_in(target_list) + stream_content = self.stream.getvalue() + if stream_content: + target_list.append(stream_content) + def copyto(self, target): """Potentially cheaper than getvalue as no string concatenation needs to happen.""" @@ -78,6 +93,12 @@ class StringIOTree(object): self.stream = StringIO() self.write = self.stream.write + def reset(self): + self.prepended_children = [] + self.markers = [] + self.stream = StringIO() + self.write = self.stream.write + def insert(self, iotree): """ Insert a StringIOTree (and all of its contents) at this location. @@ -106,3 +127,48 @@ class StringIOTree(object): def allmarkers(self): children = self.prepended_children return [m for c in children for m in c.allmarkers()] + self.markers + + """ + # Print the result of allmarkers in a nice human-readable form. Use it only for debugging. + # Prints e.g. + # /path/to/source.pyx: + # cython line 2 maps to 3299-3343 + # cython line 4 maps to 2236-2245 2306 3188-3201 + # /path/to/othersource.pyx: + # cython line 3 maps to 1234-1270 + # ... + # Note: In the example above, 3343 maps to line 2, 3344 does not. + def print_hr_allmarkers(self): + from collections import defaultdict + markers = self.allmarkers() + totmap = defaultdict(lambda: defaultdict(list)) + for c_lineno, (cython_desc, cython_lineno) in enumerate(markers): + if cython_lineno > 0 and cython_desc.filename is not None: + totmap[cython_desc.filename][cython_lineno].append(c_lineno + 1) + reprstr = "" + if totmap == 0: + reprstr += "allmarkers is empty\n" + try: + sorted(totmap.items()) + except: + print(totmap) + print(totmap.items()) + for cython_path, filemap in sorted(totmap.items()): + reprstr += cython_path + ":\n" + for cython_lineno, c_linenos in sorted(filemap.items()): + reprstr += "\tcython line " + str(cython_lineno) + " maps to " + i = 0 + while i < len(c_linenos): + reprstr += str(c_linenos[i]) + flag = False + while i+1 < len(c_linenos) and c_linenos[i+1] == c_linenos[i]+1: + i += 1 + flag = True + if flag: + reprstr += "-" + str(c_linenos[i]) + " " + i += 1 + reprstr += "\n" + + import sys + sys.stdout.write(reprstr) + """ diff --git a/Cython/Tempita/_tempita.py b/Cython/Tempita/_tempita.py index 22a7d233b..148da54d8 100644 --- a/Cython/Tempita/_tempita.py +++ b/Cython/Tempita/_tempita.py @@ -1,3 +1,5 @@ +# cython: language_level=3str + """ A small templating language @@ -33,11 +35,6 @@ from __future__ import absolute_import import re import sys -import cgi -try: - from urllib import quote as url_quote -except ImportError: # Py3 - from urllib.parse import quote as url_quote import os import tokenize from io import StringIO @@ -45,8 +42,7 @@ from io import StringIO from ._looper import looper from .compat3 import bytes, unicode_, basestring_, next, is_unicode, coerce_text -__all__ = ['TemplateError', 'Template', 'sub', 'HTMLTemplate', - 'sub_html', 'html', 'bunch'] +__all__ = ['TemplateError', 'Template', 'sub', 'bunch'] in_re = re.compile(r'\s+in\s+') var_re = re.compile(r'^[a-z_][a-z0-9_]*$', re.I) @@ -144,9 +140,8 @@ class Template(object): def from_filename(cls, filename, namespace=None, encoding=None, default_inherit=None, get_template=get_file_template): - f = open(filename, 'rb') - c = f.read() - f.close() + with open(filename, 'rb') as f: + c = f.read() if encoding: c = c.decode(encoding) return cls(content=c, name=filename, namespace=namespace, @@ -335,7 +330,7 @@ class Template(object): if not isinstance(value, basestring_): value = coerce_text(value) if (is_unicode(value) - and self.default_encoding): + and self.default_encoding): value = value.encode(self.default_encoding) except Exception as e: e.args = (self._add_line_info(e.args[0], pos),) @@ -411,91 +406,6 @@ class bunch(dict): self.__class__.__name__, ' '.join(['%s=%r' % (k, v) for k, v in sorted(self.items())])) -############################################################ -## HTML Templating -############################################################ - - -class html(object): - - def __init__(self, value): - self.value = value - - def __str__(self): - return self.value - - def __html__(self): - return self.value - - def __repr__(self): - return '<%s %r>' % ( - self.__class__.__name__, self.value) - - -def html_quote(value, force=True): - if not force and hasattr(value, '__html__'): - return value.__html__() - if value is None: - return '' - if not isinstance(value, basestring_): - value = coerce_text(value) - if sys.version >= "3" and isinstance(value, bytes): - value = cgi.escape(value.decode('latin1'), 1) - value = value.encode('latin1') - else: - value = cgi.escape(value, 1) - if sys.version < "3": - if is_unicode(value): - value = value.encode('ascii', 'xmlcharrefreplace') - return value - - -def url(v): - v = coerce_text(v) - if is_unicode(v): - v = v.encode('utf8') - return url_quote(v) - - -def attr(**kw): - parts = [] - for name, value in sorted(kw.items()): - if value is None: - continue - if name.endswith('_'): - name = name[:-1] - parts.append('%s="%s"' % (html_quote(name), html_quote(value))) - return html(' '.join(parts)) - - -class HTMLTemplate(Template): - - default_namespace = Template.default_namespace.copy() - default_namespace.update(dict( - html=html, - attr=attr, - url=url, - html_quote=html_quote, - )) - - def _repr(self, value, pos): - if hasattr(value, '__html__'): - value = value.__html__() - quote = False - else: - quote = True - plain = Template._repr(self, value, pos) - if quote: - return html_quote(plain) - else: - return plain - - -def sub_html(content, **kw): - name = kw.get('__name') - tmpl = HTMLTemplate(content, name=name) - return tmpl.substitute(kw) - class TemplateDef(object): def __init__(self, template, func_name, func_signature, @@ -723,7 +633,7 @@ def trim_lex(tokens): else: next_chunk = tokens[i + 1] if (not isinstance(next_chunk, basestring_) - or not isinstance(prev, basestring_)): + or not isinstance(prev, basestring_)): continue prev_ok = not prev or trail_whitespace_re.search(prev) if i == 1 and not prev.strip(): @@ -735,7 +645,7 @@ def trim_lex(tokens): or (i == len(tokens) - 2 and not next_chunk.strip()))): if prev: if ((i == 1 and not prev.strip()) - or prev_ok == 'last'): + or prev_ok == 'last'): tokens[i - 1] = '' else: m = trail_whitespace_re.search(prev) @@ -887,7 +797,7 @@ def parse_cond(tokens, name, context): 'Missing {{endif}}', position=start, name=name) if (isinstance(tokens[0], tuple) - and tokens[0][0] == 'endif'): + and tokens[0][0] == 'endif'): return ('cond', start) + tuple(pieces), tokens[1:] next_chunk, tokens = parse_one_cond(tokens, name, context) pieces.append(next_chunk) @@ -925,7 +835,7 @@ def parse_for(tokens, name, context): tokens = tokens[1:] context = ('for',) + context content = [] - assert first.startswith('for ') + assert first.startswith('for '), first if first.endswith(':'): first = first[:-1] first = first[3:].strip() @@ -949,7 +859,7 @@ def parse_for(tokens, name, context): 'No {{endfor}}', position=pos, name=name) if (isinstance(tokens[0], tuple) - and tokens[0][0] == 'endfor'): + and tokens[0][0] == 'endfor'): return ('for', pos, vars, expr, content), tokens[1:] next_chunk, tokens = parse_expr(tokens, name, context) content.append(next_chunk) @@ -1009,7 +919,7 @@ def parse_def(tokens, name, context): 'Missing {{enddef}}', position=start, name=name) if (isinstance(tokens[0], tuple) - and tokens[0][0] == 'enddef'): + and tokens[0][0] == 'enddef'): return ('def', start, func_name, sig, content), tokens[1:] next_chunk, tokens = parse_expr(tokens, name, context) content.append(next_chunk) @@ -1072,7 +982,7 @@ def parse_signature(sig_text, name, pos): raise TemplateError('Invalid signature: (%s)' % sig_text, position=pos, name=name) if (not nest_count and - (tok_type == tokenize.ENDMARKER or (tok_type == tokenize.OP and tok_string == ','))): + (tok_type == tokenize.ENDMARKER or (tok_type == tokenize.OP and tok_string == ','))): default_expr = isolate_expression(sig_text, start_pos, end_pos) defaults[var_name] = default_expr sig_args.append(var_name) @@ -1131,11 +1041,6 @@ def fill_command(args=None): metavar="FILENAME", help="File to write output to (default stdout)") parser.add_option( - '--html', - dest='use_html', - action='store_true', - help="Use HTML style filling (including automatic HTML quoting)") - parser.add_option( '--env', dest='use_env', action='store_true', @@ -1162,19 +1067,13 @@ def fill_command(args=None): template_content = sys.stdin.read() template_name = '<stdin>' else: - f = open(template_name, 'rb') - template_content = f.read() - f.close() - if options.use_html: - TemplateClass = HTMLTemplate - else: - TemplateClass = Template - template = TemplateClass(template_content, name=template_name) + with open(template_name, 'rb') as f: + template_content = f.read() + template = Template(template_content, name=template_name) result = template.substitute(vars) if options.output: - f = open(options.output, 'wb') - f.write(result) - f.close() + with open(options.output, 'wb') as f: + f.write(result) else: sys.stdout.write(result) diff --git a/Cython/TestUtils.py b/Cython/TestUtils.py index 9d6eb67fc..d3a34e741 100644 --- a/Cython/TestUtils.py +++ b/Cython/TestUtils.py @@ -1,8 +1,14 @@ from __future__ import absolute_import import os +import re import unittest +import shlex +import sys import tempfile +import textwrap +from io import open +from functools import partial from .Compiler import Errors from .CodeWriter import CodeWriter @@ -47,13 +53,10 @@ def treetypes(root): class CythonTest(unittest.TestCase): def setUp(self): - self.listing_file = Errors.listing_file - self.echo_file = Errors.echo_file - Errors.listing_file = Errors.echo_file = None + Errors.init_thread() def tearDown(self): - Errors.listing_file = self.listing_file - Errors.echo_file = self.echo_file + Errors.init_thread() def assertLines(self, expected, result): "Checks that the given strings or lists of strings are equal line by line" @@ -160,11 +163,87 @@ class TransformTest(CythonTest): return tree +# For the test C code validation, we have to take care that the test directives (and thus +# the match strings) do not just appear in (multiline) C code comments containing the original +# Cython source code. Thus, we discard the comments before matching. +# This seems a prime case for re.VERBOSE, but it seems to match some of the whitespace. +_strip_c_comments = partial(re.compile( + re.sub(r'\s+', '', r''' + /[*] ( + (?: [^*\n] | [*][^/] )* + [\n] + (?: [^*] | [*][^/] )* + ) [*]/ + ''') +).sub, '') + +_strip_cython_code_from_html = partial(re.compile( + re.sub(r'\s\s+', '', r''' + (?: + <pre class=["'][^"']*cython\s+line[^"']*["']\s*> + (?:[^<]|<(?!/pre))+ + </pre> + )|(?: + <style[^>]*> + (?:[^<]|<(?!/style))+ + </style> + ) + ''') +).sub, '') + + class TreeAssertVisitor(VisitorTransform): # actually, a TreeVisitor would be enough, but this needs to run # as part of the compiler pipeline - def visit_CompilerDirectivesNode(self, node): + def __init__(self): + super(TreeAssertVisitor, self).__init__() + self._module_pos = None + self._c_patterns = [] + self._c_antipatterns = [] + + def create_c_file_validator(self): + patterns, antipatterns = self._c_patterns, self._c_antipatterns + + def fail(pos, pattern, found, file_path): + Errors.error(pos, "Pattern '%s' %s found in %s" %( + pattern, + 'was' if found else 'was not', + file_path, + )) + + def validate_file_content(file_path, content): + for pattern in patterns: + #print("Searching pattern '%s'" % pattern) + if not re.search(pattern, content): + fail(self._module_pos, pattern, found=False, file_path=file_path) + + for antipattern in antipatterns: + #print("Searching antipattern '%s'" % antipattern) + if re.search(antipattern, content): + fail(self._module_pos, antipattern, found=True, file_path=file_path) + + def validate_c_file(result): + c_file = result.c_file + if not (patterns or antipatterns): + #print("No patterns defined for %s" % c_file) + return result + + with open(c_file, encoding='utf8') as f: + content = f.read() + content = _strip_c_comments(content) + validate_file_content(c_file, content) + + html_file = os.path.splitext(c_file)[0] + ".html" + if os.path.exists(html_file) and os.path.getmtime(c_file) <= os.path.getmtime(html_file): + with open(html_file, encoding='utf8') as f: + content = f.read() + content = _strip_cython_code_from_html(content) + validate_file_content(html_file, content) + + return validate_c_file + + def _check_directives(self, node): directives = node.directives if 'test_assert_path_exists' in directives: for path in directives['test_assert_path_exists']: @@ -174,44 +253,114 @@ class TreeAssertVisitor(VisitorTransform): "Expected path '%s' not found in result tree" % path) if 'test_fail_if_path_exists' in directives: for path in directives['test_fail_if_path_exists']: - if TreePath.find_first(node, path) is not None: + first_node = TreePath.find_first(node, path) + if first_node is not None: Errors.error( - node.pos, - "Unexpected path '%s' found in result tree" % path) + first_node.pos, + "Unexpected path '%s' found in result tree" % path) + if 'test_assert_c_code_has' in directives: + self._c_patterns.extend(directives['test_assert_c_code_has']) + if 'test_fail_if_c_code_has' in directives: + self._c_antipatterns.extend(directives['test_fail_if_c_code_has']) + + def visit_ModuleNode(self, node): + self._module_pos = node.pos + self._check_directives(node) + self.visitchildren(node) + return node + + def visit_CompilerDirectivesNode(self, node): + self._check_directives(node) self.visitchildren(node) return node visit_Node = VisitorTransform.recurse_to_children -def unpack_source_tree(tree_file, dir=None): - if dir is None: - dir = tempfile.mkdtemp() - header = [] - cur_file = None - f = open(tree_file) - try: - lines = f.readlines() - finally: - f.close() - del f +def unpack_source_tree(tree_file, workdir, cython_root): + programs = { + 'PYTHON': [sys.executable], + 'CYTHON': [sys.executable, os.path.join(cython_root, 'cython.py')], + 'CYTHONIZE': [sys.executable, os.path.join(cython_root, 'cythonize.py')] + } + + if workdir is None: + workdir = tempfile.mkdtemp() + header, cur_file = [], None + with open(tree_file, 'rb') as f: + try: + for line in f: + if line[:5] == b'#####': + filename = line.strip().strip(b'#').strip().decode('utf8').replace('/', os.path.sep) + path = os.path.join(workdir, filename) + if not os.path.exists(os.path.dirname(path)): + os.makedirs(os.path.dirname(path)) + if cur_file is not None: + to_close, cur_file = cur_file, None + to_close.close() + cur_file = open(path, 'wb') + elif cur_file is not None: + cur_file.write(line) + elif line.strip() and not line.lstrip().startswith(b'#'): + if line.strip() not in (b'"""', b"'''"): + command = shlex.split(line.decode('utf8')) + if not command: continue + # In Python 3: prog, *args = command + prog, args = command[0], command[1:] + try: + header.append(programs[prog]+args) + except KeyError: + header.append(command) + finally: + if cur_file is not None: + cur_file.close() + return workdir, header + + +def write_file(file_path, content, dedent=False, encoding=None): + r"""Write some content (text or bytes) to the file + at `file_path` without translating `'\n'` into `os.linesep`. + + The default encoding is `'utf-8'`. + """ + if isinstance(content, bytes): + mode = "wb" + + # binary mode doesn't take an encoding and newline arguments + newline = None + default_encoding = None + else: + mode = "w" + + # any "\n" characters written are not translated + # to the system default line separator, os.linesep + newline = "\n" + default_encoding = "utf-8" + + if encoding is None: + encoding = default_encoding + + if dedent: + content = textwrap.dedent(content) + + with open(file_path, mode=mode, encoding=encoding, newline=newline) as f: + f.write(content) + + +def write_newer_file(file_path, newer_than, content, dedent=False, encoding=None): + r""" + Write `content` to the file `file_path` without translating `'\n'` + into `os.linesep` and make sure it is newer than the file `newer_than`. + + The default encoding is `'utf-8'` (same as for `write_file`). + """ + write_file(file_path, content, dedent=dedent, encoding=encoding) + try: - for line in lines: - if line[:5] == '#####': - filename = line.strip().strip('#').strip().replace('/', os.path.sep) - path = os.path.join(dir, filename) - if not os.path.exists(os.path.dirname(path)): - os.makedirs(os.path.dirname(path)) - if cur_file is not None: - f, cur_file = cur_file, None - f.close() - cur_file = open(path, 'w') - elif cur_file is not None: - cur_file.write(line) - elif line.strip() and not line.lstrip().startswith('#'): - if line.strip() not in ('"""', "'''"): - header.append(line) - finally: - if cur_file is not None: - cur_file.close() - return dir, ''.join(header) + other_time = os.path.getmtime(newer_than) + except OSError: + # Support writing a fresh file (which is always newer than a non-existant one) + other_time = None + + while other_time is None or other_time >= os.path.getmtime(file_path): + write_file(file_path, content, dedent=dedent, encoding=encoding) diff --git a/Cython/Tests/TestCodeWriter.py b/Cython/Tests/TestCodeWriter.py index 42e457da2..c3026cb1d 100644 --- a/Cython/Tests/TestCodeWriter.py +++ b/Cython/Tests/TestCodeWriter.py @@ -19,9 +19,9 @@ class TestCodeWriter(CythonTest): def test_print(self): self.t(u""" - print x, y - print x + y ** 2 - print x, y, z, + print(x + y ** 2) + print(x, y, z) + print(x + y, x + y * z, x * (y + z)) """) def test_if(self): @@ -47,6 +47,20 @@ class TestCodeWriter(CythonTest): pass """) + def test_cdef(self): + self.t(u""" + cdef f(x, y, z): + pass + cdef public void (x = 34, y = 54, z): + pass + cdef f(int *x, void *y, Value *z): + pass + cdef f(int **x, void **y, Value **z): + pass + cdef inline f(int &x, Value &z): + pass + """) + def test_longness_and_signedness(self): self.t(u"def f(unsigned long long long long long int y):\n pass") @@ -65,18 +79,50 @@ class TestCodeWriter(CythonTest): def test_for_loop(self): self.t(u""" for x, y, z in f(g(h(34) * 2) + 23): - print x, y, z + print(x, y, z) + else: + print(43) + """) + self.t(u""" + for abc in (1, 2, 3): + print(x, y, z) else: - print 43 + print(43) + """) + + def test_while_loop(self): + self.t(u""" + while True: + while True: + while True: + continue """) def test_inplace_assignment(self): self.t(u"x += 43") + def test_cascaded_assignment(self): + self.t(u"x = y = z = abc = 43") + def test_attribute(self): self.t(u"a.x") + def test_return_none(self): + self.t(u""" + def f(x, y, z): + return + cdef f(x, y, z): + return + def f(x, y, z): + return None + cdef f(x, y, z): + return None + def f(x, y, z): + return 1234 + cdef f(x, y, z): + return 1234 + """) + if __name__ == "__main__": import unittest unittest.main() - diff --git a/Cython/Tests/TestCythonUtils.py b/Cython/Tests/TestCythonUtils.py index 2641900c0..402616101 100644 --- a/Cython/Tests/TestCythonUtils.py +++ b/Cython/Tests/TestCythonUtils.py @@ -1,11 +1,128 @@ import unittest -from ..Utils import build_hex_version +from Cython.Utils import ( + _CACHE_NAME_PATTERN, _build_cache_name, _find_cache_attributes, + build_hex_version, cached_method, clear_method_caches, try_finally_contextmanager) + +METHOD_NAME = "cached_next" +CACHE_NAME = _build_cache_name(METHOD_NAME) +NAMES = CACHE_NAME, METHOD_NAME + +class Cached(object): + @cached_method + def cached_next(self, x): + return next(x) + class TestCythonUtils(unittest.TestCase): def test_build_hex_version(self): self.assertEqual('0x001D00A1', build_hex_version('0.29a1')) - self.assertEqual('0x001D00A1', build_hex_version('0.29a1')) self.assertEqual('0x001D03C4', build_hex_version('0.29.3rc4')) self.assertEqual('0x001D00F0', build_hex_version('0.29')) self.assertEqual('0x040000F0', build_hex_version('4.0')) + + ############################## Cached Methods ############################## + + def test_cache_method_name(self): + method_name = "foo" + cache_name = _build_cache_name(method_name) + match = _CACHE_NAME_PATTERN.match(cache_name) + + self.assertIsNot(match, None) + self.assertEqual(match.group(1), method_name) + + def test_requirements_for_Cached(self): + obj = Cached() + + self.assertFalse(hasattr(obj, CACHE_NAME)) + self.assertTrue(hasattr(obj, METHOD_NAME)) + self.set_of_names_equal(obj, set()) + + def set_of_names_equal(self, obj, value): + self.assertEqual(set(_find_cache_attributes(obj)), value) + + def test_find_cache_attributes(self): + obj = Cached() + method_name = "bar" + cache_name = _build_cache_name(method_name) + + setattr(obj, CACHE_NAME, {}) + setattr(obj, cache_name, {}) + + self.assertFalse(hasattr(obj, method_name)) + self.set_of_names_equal(obj, {NAMES, (cache_name, method_name)}) + + def test_cached_method(self): + obj = Cached() + value = iter(range(3)) # iter for Py2 + cache = {(value,): 0} + + # cache args + self.assertEqual(obj.cached_next(value), 0) + self.set_of_names_equal(obj, {NAMES}) + self.assertEqual(getattr(obj, CACHE_NAME), cache) + + # use cache + self.assertEqual(obj.cached_next(value), 0) + self.set_of_names_equal(obj, {NAMES}) + self.assertEqual(getattr(obj, CACHE_NAME), cache) + + def test_clear_method_caches(self): + obj = Cached() + value = iter(range(3)) # iter for Py2 + cache = {(value,): 1} + + obj.cached_next(value) # cache args + + clear_method_caches(obj) + self.set_of_names_equal(obj, set()) + + self.assertEqual(obj.cached_next(value), 1) + self.set_of_names_equal(obj, {NAMES}) + self.assertEqual(getattr(obj, CACHE_NAME), cache) + + def test_clear_method_caches_with_missing_method(self): + obj = Cached() + method_name = "bar" + cache_name = _build_cache_name(method_name) + names = cache_name, method_name + + setattr(obj, cache_name, object()) + + self.assertFalse(hasattr(obj, method_name)) + self.set_of_names_equal(obj, {names}) + + clear_method_caches(obj) + self.set_of_names_equal(obj, {names}) + + def test_try_finally_contextmanager(self): + states = [] + @try_finally_contextmanager + def gen(*args, **kwargs): + states.append("enter") + yield (args, kwargs) + states.append("exit") + + with gen(1, 2, 3, x=4) as call_args: + assert states == ["enter"] + self.assertEqual(call_args, ((1, 2, 3), {'x': 4})) + assert states == ["enter", "exit"] + + class MyException(RuntimeError): + pass + + del states[:] + with self.assertRaises(MyException): + with gen(1, 2, y=4) as call_args: + assert states == ["enter"] + self.assertEqual(call_args, ((1, 2), {'y': 4})) + raise MyException("FAIL INSIDE") + assert states == ["enter", "exit"] + + del states[:] + with self.assertRaises(StopIteration): + with gen(1, 2, y=4) as call_args: + assert states == ["enter"] + self.assertEqual(call_args, ((1, 2), {'y': 4})) + raise StopIteration("STOP") + assert states == ["enter", "exit"] diff --git a/Cython/Tests/TestJediTyper.py b/Cython/Tests/TestJediTyper.py index 253adef17..ede99b3a8 100644 --- a/Cython/Tests/TestJediTyper.py +++ b/Cython/Tests/TestJediTyper.py @@ -11,7 +11,7 @@ from contextlib import contextmanager from tempfile import NamedTemporaryFile from Cython.Compiler.ParseTreeTransforms import NormalizeTree, InterpretCompilerDirectives -from Cython.Compiler import Main, Symtab, Visitor +from Cython.Compiler import Main, Symtab, Visitor, Options from Cython.TestUtils import TransformTest TOOLS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'Tools')) @@ -210,8 +210,8 @@ class TestTypeInjection(TestJediTyper): """ def setUp(self): super(TestTypeInjection, self).setUp() - compilation_options = Main.CompilationOptions(Main.default_options) - ctx = compilation_options.create_context() + compilation_options = Options.CompilationOptions(Options.default_options) + ctx = Main.Context.from_options(compilation_options) transform = InterpretCompilerDirectives(ctx, ctx.compiler_directives) transform.module_scope = Symtab.ModuleScope('__main__', None, ctx) self.declarations_finder = DeclarationsFinder() diff --git a/Cython/Tests/TestTestUtils.py b/Cython/Tests/TestTestUtils.py new file mode 100644 index 000000000..140cb7c40 --- /dev/null +++ b/Cython/Tests/TestTestUtils.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +import os.path +import unittest +import tempfile +import textwrap +import shutil + +from ..TestUtils import write_file, write_newer_file + + +class TestTestUtils(unittest.TestCase): + def setUp(self): + super(TestTestUtils, self).setUp() + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + if self.temp_dir and os.path.isdir(self.temp_dir): + shutil.rmtree(self.temp_dir) + super(TestTestUtils, self).tearDown() + + def _test_path(self, filename): + return os.path.join(self.temp_dir, filename) + + def _test_write_file(self, content, expected, **kwargs): + file_path = self._test_path("abcfile") + write_file(file_path, content, **kwargs) + assert os.path.isfile(file_path) + + with open(file_path, 'rb') as f: + found = f.read() + assert found == expected, (repr(expected), repr(found)) + + def test_write_file_text(self): + text = u"abcüöä" + self._test_write_file(text, text.encode('utf8')) + + def test_write_file_dedent(self): + text = u""" + A horse is a horse, + of course, of course, + And no one can talk to a horse + of course + """ + self._test_write_file(text, textwrap.dedent(text).encode('utf8'), dedent=True) + + def test_write_file_bytes(self): + self._test_write_file(b"ab\0c", b"ab\0c") + + def test_write_newer_file(self): + file_path_1 = self._test_path("abcfile1.txt") + file_path_2 = self._test_path("abcfile2.txt") + write_file(file_path_1, "abc") + assert os.path.isfile(file_path_1) + write_newer_file(file_path_2, file_path_1, "xyz") + assert os.path.isfile(file_path_2) + assert os.path.getmtime(file_path_2) > os.path.getmtime(file_path_1) + + def test_write_newer_file_same(self): + file_path = self._test_path("abcfile.txt") + write_file(file_path, "abc") + mtime = os.path.getmtime(file_path) + write_newer_file(file_path, file_path, "xyz") + assert os.path.getmtime(file_path) > mtime + + def test_write_newer_file_fresh(self): + file_path = self._test_path("abcfile.txt") + assert not os.path.exists(file_path) + write_newer_file(file_path, file_path, "xyz") + assert os.path.isfile(file_path) diff --git a/Cython/Tests/xmlrunner.py b/Cython/Tests/xmlrunner.py index d6838aa22..eeeb49394 100644 --- a/Cython/Tests/xmlrunner.py +++ b/Cython/Tests/xmlrunner.py @@ -109,8 +109,7 @@ class _XMLTestResult(TextTestResult): self.elapsed_times = elapsed_times self.output_patched = False - def _prepare_callback(self, test_info, target_list, verbose_str, - short_str): + def _prepare_callback(self, test_info, target_list, verbose_str, short_str): """Append a _TestInfo to the given target list and sets a callback method to be called by stopTest method. """ @@ -125,7 +124,7 @@ class _XMLTestResult(TextTestResult): self.start_time = self.stop_time = 0 if self.showAll: - self.stream.writeln('(%.3fs) %s' % \ + self.stream.writeln('(%.3fs) %s' % (test_info.get_elapsed_time(), verbose_str)) elif self.dots: self.stream.write(short_str) @@ -300,8 +299,7 @@ class _XMLTestResult(TextTestResult): "Generates the XML reports to a given XMLTestRunner object." all_results = self._get_info_by_testcase() - if type(test_runner.output) == str and not \ - os.path.exists(test_runner.output): + if isinstance(test_runner.output, str) and not os.path.exists(test_runner.output): os.makedirs(test_runner.output) for suite, tests in all_results.items(): @@ -321,7 +319,7 @@ class _XMLTestResult(TextTestResult): xml_content = doc.toprettyxml(indent='\t') if type(test_runner.output) is str: - report_file = open('%s%sTEST-%s.xml' % \ + report_file = open('%s%sTEST-%s.xml' % (test_runner.output, os.sep, suite), 'w') try: report_file.write(xml_content) @@ -348,7 +346,7 @@ class XMLTestRunner(TextTestRunner): """Create the TestResult object which will be used to store information about the executed tests. """ - return _XMLTestResult(self.stream, self.descriptions, \ + return _XMLTestResult(self.stream, self.descriptions, self.verbosity, self.elapsed_times) def run(self, test): diff --git a/Cython/Utility/AsyncGen.c b/Cython/Utility/AsyncGen.c index dd4bf3728..a3d068e97 100644 --- a/Cython/Utility/AsyncGen.c +++ b/Cython/Utility/AsyncGen.c @@ -11,6 +11,7 @@ typedef struct { PyObject *ag_finalizer; int ag_hooks_inited; int ag_closed; + int ag_running_async; } __pyx_PyAsyncGenObject; static PyTypeObject *__pyx__PyAsyncGenWrappedValueType = 0; @@ -18,11 +19,11 @@ static PyTypeObject *__pyx__PyAsyncGenASendType = 0; static PyTypeObject *__pyx__PyAsyncGenAThrowType = 0; static PyTypeObject *__pyx_AsyncGenType = 0; -#define __Pyx_AsyncGen_CheckExact(obj) (Py_TYPE(obj) == __pyx_AsyncGenType) +#define __Pyx_AsyncGen_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_AsyncGenType) #define __pyx_PyAsyncGenASend_CheckExact(o) \ - (Py_TYPE(o) == __pyx__PyAsyncGenASendType) + __Pyx_IS_TYPE(o, __pyx__PyAsyncGenASendType) #define __pyx_PyAsyncGenAThrow_CheckExact(o) \ - (Py_TYPE(o) == __pyx__PyAsyncGenAThrowType) + __Pyx_IS_TYPE(o, __pyx__PyAsyncGenAThrowType) static PyObject *__Pyx_async_gen_anext(PyObject *o); static CYTHON_INLINE PyObject *__Pyx_async_gen_asend_iternext(PyObject *o); @@ -42,10 +43,11 @@ static __pyx_CoroutineObject *__Pyx_AsyncGen_New( gen->ag_finalizer = NULL; gen->ag_closed = 0; gen->ag_hooks_inited = 0; + gen->ag_running_async = 0; return __Pyx__Coroutine_NewInit((__pyx_CoroutineObject*)gen, body, code, closure, name, qualname, module_name); } -static int __pyx_AsyncGen_init(void); +static int __pyx_AsyncGen_init(PyObject *module); static void __Pyx_PyAsyncGen_Fini(void); //////////////////// AsyncGenerator.cleanup //////////////////// @@ -127,6 +129,8 @@ static PyObject *__Pyx_async_gen_athrow_new(__pyx_PyAsyncGenObject *, PyObject * static const char *__Pyx_NON_INIT_CORO_MSG = "can't send non-None value to a just-started coroutine"; static const char *__Pyx_ASYNC_GEN_IGNORED_EXIT_MSG = "async generator ignored GeneratorExit"; +static const char *__Pyx_ASYNC_GEN_CANNOT_REUSE_SEND_MSG = "cannot reuse already awaited __anext__()/asend()"; +static const char *__Pyx_ASYNC_GEN_CANNOT_REUSE_CLOSE_MSG = "cannot reuse already awaited aclose()/athrow()"; typedef enum { __PYX_AWAITABLE_STATE_INIT, /* new awaitable, has not yet been iterated */ @@ -178,7 +182,7 @@ static __pyx_PyAsyncGenASend *__Pyx_ag_asend_freelist[_PyAsyncGen_MAXFREELIST]; static int __Pyx_ag_asend_freelist_free = 0; #define __pyx__PyAsyncGenWrappedValue_CheckExact(o) \ - (Py_TYPE(o) == __pyx__PyAsyncGenWrappedValueType) + __Pyx_IS_TYPE(o, __pyx__PyAsyncGenWrappedValueType) static int @@ -253,14 +257,15 @@ static PyObject * __Pyx_async_gen_anext(PyObject *g) { __pyx_PyAsyncGenObject *o = (__pyx_PyAsyncGenObject*) g; - if (__Pyx_async_gen_init_hooks(o)) { + if (unlikely(__Pyx_async_gen_init_hooks(o))) { return NULL; } return __Pyx_async_gen_asend_new(o, NULL); } static PyObject * -__Pyx_async_gen_anext_method(PyObject *g, CYTHON_UNUSED PyObject *arg) { +__Pyx_async_gen_anext_method(PyObject *g, PyObject *arg) { + CYTHON_UNUSED_VAR(arg); return __Pyx_async_gen_anext(g); } @@ -268,7 +273,7 @@ __Pyx_async_gen_anext_method(PyObject *g, CYTHON_UNUSED PyObject *arg) { static PyObject * __Pyx_async_gen_asend(__pyx_PyAsyncGenObject *o, PyObject *arg) { - if (__Pyx_async_gen_init_hooks(o)) { + if (unlikely(__Pyx_async_gen_init_hooks(o))) { return NULL; } return __Pyx_async_gen_asend_new(o, arg); @@ -276,9 +281,10 @@ __Pyx_async_gen_asend(__pyx_PyAsyncGenObject *o, PyObject *arg) static PyObject * -__Pyx_async_gen_aclose(__pyx_PyAsyncGenObject *o, CYTHON_UNUSED PyObject *arg) +__Pyx_async_gen_aclose(__pyx_PyAsyncGenObject *o, PyObject *arg) { - if (__Pyx_async_gen_init_hooks(o)) { + CYTHON_UNUSED_VAR(arg); + if (unlikely(__Pyx_async_gen_init_hooks(o))) { return NULL; } return __Pyx_async_gen_athrow_new(o, NULL); @@ -288,7 +294,7 @@ __Pyx_async_gen_aclose(__pyx_PyAsyncGenObject *o, CYTHON_UNUSED PyObject *arg) static PyObject * __Pyx_async_gen_athrow(__pyx_PyAsyncGenObject *o, PyObject *args) { - if (__Pyx_async_gen_init_hooks(o)) { + if (unlikely(__Pyx_async_gen_init_hooks(o))) { return NULL; } return __Pyx_async_gen_athrow_new(o, args); @@ -296,7 +302,8 @@ __Pyx_async_gen_athrow(__pyx_PyAsyncGenObject *o, PyObject *args) static PyObject * -__Pyx_async_gen_self_method(PyObject *g, CYTHON_UNUSED PyObject *arg) { +__Pyx_async_gen_self_method(PyObject *g, PyObject *arg) { + CYTHON_UNUSED_VAR(arg); return __Pyx_NewRef(g); } @@ -313,11 +320,15 @@ static PyGetSetDef __Pyx_async_gen_getsetlist[] = { static PyMemberDef __Pyx_async_gen_memberlist[] = { //REMOVED: {(char*) "ag_frame", T_OBJECT, offsetof(__pyx_PyAsyncGenObject, ag_frame), READONLY}, - {(char*) "ag_running", T_BOOL, offsetof(__pyx_CoroutineObject, is_running), READONLY, NULL}, + {(char*) "ag_running", T_BOOL, offsetof(__pyx_PyAsyncGenObject, ag_running_async), READONLY, NULL}, //REMOVED: {(char*) "ag_code", T_OBJECT, offsetof(__pyx_PyAsyncGenObject, ag_code), READONLY}, //ADDED: "ag_await" {(char*) "ag_await", T_OBJECT, offsetof(__pyx_CoroutineObject, yieldfrom), READONLY, (char*) PyDoc_STR("object being awaited on, or None")}, + {(char *) "__module__", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_modulename), 0, 0}, +#if CYTHON_USE_TYPE_SPECS + {(char *) "__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CoroutineObject, gi_weakreflist), READONLY, 0}, +#endif {0, 0, 0, 0, 0} /* Sentinel */ }; @@ -346,6 +357,31 @@ static PyMethodDef __Pyx_async_gen_methods[] = { }; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_AsyncGenType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_Coroutine_dealloc}, + {Py_am_aiter, (void *)PyObject_SelfIter}, + {Py_am_anext, (void *)__Pyx_async_gen_anext}, + {Py_tp_repr, (void *)__Pyx_async_gen_repr}, + {Py_tp_traverse, (void *)__Pyx_async_gen_traverse}, + {Py_tp_methods, (void *)__Pyx_async_gen_methods}, + {Py_tp_members, (void *)__Pyx_async_gen_memberlist}, + {Py_tp_getset, (void *)__Pyx_async_gen_getsetlist}, +#if CYTHON_USE_TP_FINALIZE + {Py_tp_finalize, (void *)__Pyx_Coroutine_del}, +#endif + {0, 0}, +}; + +static PyType_Spec __pyx_AsyncGenType_spec = { + __PYX_TYPE_MODULE_PREFIX "async_generator", + sizeof(__pyx_PyAsyncGenObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/ + __pyx_AsyncGenType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + #if CYTHON_USE_ASYNC_SLOTS static __Pyx_PyAsyncMethodsStruct __Pyx_async_gen_as_async = { 0, /* am_await */ @@ -363,7 +399,7 @@ static PyTypeObject __pyx_AsyncGenType_type = { sizeof(__pyx_PyAsyncGenObject), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)__Pyx_Coroutine_dealloc, /* tp_dealloc */ - 0, /* tp_print */ + 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ #if CYTHON_USE_ASYNC_SLOTS @@ -427,7 +463,7 @@ static PyTypeObject __pyx_AsyncGenType_type = { #if PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800) 0, /*tp_vectorcall*/ #endif -#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 +#if __PYX_NEED_TP_PRINT_SLOT 0, /*tp_print*/ #endif #if PY_VERSION_HEX >= 0x030C0000 @@ -437,6 +473,7 @@ static PyTypeObject __pyx_AsyncGenType_type = { 0, /*tp_pypy_flags*/ #endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ static int @@ -448,14 +485,14 @@ __Pyx_PyAsyncGen_ClearFreeLists(void) __pyx__PyAsyncGenWrappedValue *o; o = __Pyx_ag_value_freelist[--__Pyx_ag_value_freelist_free]; assert(__pyx__PyAsyncGenWrappedValue_CheckExact(o)); - PyObject_GC_Del(o); + __Pyx_PyHeapTypeObject_GC_Del(o); } while (__Pyx_ag_asend_freelist_free) { __pyx_PyAsyncGenASend *o; o = __Pyx_ag_asend_freelist[--__Pyx_ag_asend_freelist_free]; - assert(Py_TYPE(o) == __pyx__PyAsyncGenASendType); - PyObject_GC_Del(o); + assert(__Pyx_IS_TYPE(o, __pyx__PyAsyncGenASendType)); + __Pyx_PyHeapTypeObject_GC_Del(o); } return ret; @@ -480,6 +517,7 @@ __Pyx_async_gen_unwrap_value(__pyx_PyAsyncGenObject *gen, PyObject *result) gen->ag_closed = 1; } + gen->ag_running_async = 0; return NULL; } @@ -487,6 +525,7 @@ __Pyx_async_gen_unwrap_value(__pyx_PyAsyncGenObject *gen, PyObject *result) /* async yield */ __Pyx_ReturnWithStopIteration(((__pyx__PyAsyncGenWrappedValue*)result)->agw_val); Py_DECREF(result); + gen->ag_running_async = 0; return NULL; } @@ -503,11 +542,11 @@ __Pyx_async_gen_asend_dealloc(__pyx_PyAsyncGenASend *o) PyObject_GC_UnTrack((PyObject *)o); Py_CLEAR(o->ags_gen); Py_CLEAR(o->ags_sendval); - if (__Pyx_ag_asend_freelist_free < _PyAsyncGen_MAXFREELIST) { + if (likely(__Pyx_ag_asend_freelist_free < _PyAsyncGen_MAXFREELIST)) { assert(__pyx_PyAsyncGenASend_CheckExact(o)); __Pyx_ag_asend_freelist[__Pyx_ag_asend_freelist_free++] = o; } else { - PyObject_GC_Del(o); + __Pyx_PyHeapTypeObject_GC_Del(o); } } @@ -527,17 +566,25 @@ __Pyx_async_gen_asend_send(PyObject *g, PyObject *arg) PyObject *result; if (unlikely(o->ags_state == __PYX_AWAITABLE_STATE_CLOSED)) { - PyErr_SetNone(PyExc_StopIteration); + PyErr_SetString(PyExc_RuntimeError, __Pyx_ASYNC_GEN_CANNOT_REUSE_SEND_MSG); return NULL; } if (o->ags_state == __PYX_AWAITABLE_STATE_INIT) { + if (unlikely(o->ags_gen->ag_running_async)) { + PyErr_SetString( + PyExc_RuntimeError, + "anext(): asynchronous generator is already running"); + return NULL; + } + if (arg == NULL || arg == Py_None) { arg = o->ags_sendval ? o->ags_sendval : Py_None; } o->ags_state = __PYX_AWAITABLE_STATE_ITER; } + o->ags_gen->ag_running_async = 1; result = __Pyx_Coroutine_Send((PyObject*)o->ags_gen, arg); result = __Pyx_async_gen_unwrap_value(o->ags_gen, result); @@ -562,7 +609,7 @@ __Pyx_async_gen_asend_throw(__pyx_PyAsyncGenASend *o, PyObject *args) PyObject *result; if (unlikely(o->ags_state == __PYX_AWAITABLE_STATE_CLOSED)) { - PyErr_SetNone(PyExc_StopIteration); + PyErr_SetString(PyExc_RuntimeError, __Pyx_ASYNC_GEN_CANNOT_REUSE_SEND_MSG); return NULL; } @@ -578,9 +625,10 @@ __Pyx_async_gen_asend_throw(__pyx_PyAsyncGenASend *o, PyObject *args) static PyObject * -__Pyx_async_gen_asend_close(PyObject *g, CYTHON_UNUSED PyObject *args) +__Pyx_async_gen_asend_close(PyObject *g, PyObject *args) { __pyx_PyAsyncGenASend *o = (__pyx_PyAsyncGenASend*) g; + CYTHON_UNUSED_VAR(args); o->ags_state = __PYX_AWAITABLE_STATE_CLOSED; Py_RETURN_NONE; } @@ -595,6 +643,26 @@ static PyMethodDef __Pyx_async_gen_asend_methods[] = { }; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx__PyAsyncGenASendType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_async_gen_asend_dealloc}, + {Py_am_await, (void *)PyObject_SelfIter}, + {Py_tp_traverse, (void *)__Pyx_async_gen_asend_traverse}, + {Py_tp_methods, (void *)__Pyx_async_gen_asend_methods}, + {Py_tp_iter, (void *)PyObject_SelfIter}, + {Py_tp_iternext, (void *)__Pyx_async_gen_asend_iternext}, + {0, 0}, +}; + +static PyType_Spec __pyx__PyAsyncGenASendType_spec = { + __PYX_TYPE_MODULE_PREFIX "async_generator_asend", + sizeof(__pyx_PyAsyncGenASend), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + __pyx__PyAsyncGenASendType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + #if CYTHON_USE_ASYNC_SLOTS static __Pyx_PyAsyncMethodsStruct __Pyx_async_gen_asend_as_async = { PyObject_SelfIter, /* am_await */ @@ -606,7 +674,6 @@ static __Pyx_PyAsyncMethodsStruct __Pyx_async_gen_asend_as_async = { }; #endif - static PyTypeObject __pyx__PyAsyncGenASendType_type = { PyVarObject_HEAD_INIT(0, 0) "async_generator_asend", /* tp_name */ @@ -614,7 +681,7 @@ static PyTypeObject __pyx__PyAsyncGenASendType_type = { 0, /* tp_itemsize */ /* methods */ (destructor)__Pyx_async_gen_asend_dealloc, /* tp_dealloc */ - 0, /* tp_print */ + 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ #if CYTHON_USE_ASYNC_SLOTS @@ -671,7 +738,7 @@ static PyTypeObject __pyx__PyAsyncGenASendType_type = { #if PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800) 0, /*tp_vectorcall*/ #endif -#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 +#if __PYX_NEED_TP_PRINT_SLOT 0, /*tp_print*/ #endif #if PY_VERSION_HEX >= 0x030C0000 @@ -681,19 +748,20 @@ static PyTypeObject __pyx__PyAsyncGenASendType_type = { 0, /*tp_pypy_flags*/ #endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ static PyObject * __Pyx_async_gen_asend_new(__pyx_PyAsyncGenObject *gen, PyObject *sendval) { __pyx_PyAsyncGenASend *o; - if (__Pyx_ag_asend_freelist_free) { + if (likely(__Pyx_ag_asend_freelist_free)) { __Pyx_ag_asend_freelist_free--; o = __Pyx_ag_asend_freelist[__Pyx_ag_asend_freelist_free]; _Py_NewReference((PyObject *)o); } else { o = PyObject_GC_New(__pyx_PyAsyncGenASend, __pyx__PyAsyncGenASendType); - if (o == NULL) { + if (unlikely(o == NULL)) { return NULL; } } @@ -719,11 +787,11 @@ __Pyx_async_gen_wrapped_val_dealloc(__pyx__PyAsyncGenWrappedValue *o) { PyObject_GC_UnTrack((PyObject *)o); Py_CLEAR(o->agw_val); - if (__Pyx_ag_value_freelist_free < _PyAsyncGen_MAXFREELIST) { + if (likely(__Pyx_ag_value_freelist_free < _PyAsyncGen_MAXFREELIST)) { assert(__pyx__PyAsyncGenWrappedValue_CheckExact(o)); __Pyx_ag_value_freelist[__Pyx_ag_value_freelist_free++] = o; } else { - PyObject_GC_Del(o); + __Pyx_PyHeapTypeObject_GC_Del(o); } } @@ -737,6 +805,22 @@ __Pyx_async_gen_wrapped_val_traverse(__pyx__PyAsyncGenWrappedValue *o, } +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx__PyAsyncGenWrappedValueType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_async_gen_wrapped_val_dealloc}, + {Py_tp_traverse, (void *)__Pyx_async_gen_wrapped_val_traverse}, + {0, 0}, +}; + +static PyType_Spec __pyx__PyAsyncGenWrappedValueType_spec = { + __PYX_TYPE_MODULE_PREFIX "async_generator_wrapped_value", + sizeof(__pyx__PyAsyncGenWrappedValue), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + __pyx__PyAsyncGenWrappedValueType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + static PyTypeObject __pyx__PyAsyncGenWrappedValueType_type = { PyVarObject_HEAD_INIT(0, 0) "async_generator_wrapped_value", /* tp_name */ @@ -744,7 +828,7 @@ static PyTypeObject __pyx__PyAsyncGenWrappedValueType_type = { 0, /* tp_itemsize */ /* methods */ (destructor)__Pyx_async_gen_wrapped_val_dealloc, /* tp_dealloc */ - 0, /* tp_print */ + 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_as_async */ @@ -792,7 +876,7 @@ static PyTypeObject __pyx__PyAsyncGenWrappedValueType_type = { #if PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800) 0, /*tp_vectorcall*/ #endif -#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 +#if __PYX_NEED_TP_PRINT_SLOT 0, /*tp_print*/ #endif #if PY_VERSION_HEX >= 0x030C0000 @@ -802,6 +886,7 @@ static PyTypeObject __pyx__PyAsyncGenWrappedValueType_type = { 0, /*tp_pypy_flags*/ #endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ static PyObject * @@ -811,7 +896,7 @@ __Pyx__PyAsyncGenValueWrapperNew(PyObject *val) __pyx__PyAsyncGenWrappedValue *o; assert(val); - if (__Pyx_ag_value_freelist_free) { + if (likely(__Pyx_ag_value_freelist_free)) { __Pyx_ag_value_freelist_free--; o = __Pyx_ag_value_freelist[__Pyx_ag_value_freelist_free]; assert(__pyx__PyAsyncGenWrappedValue_CheckExact(o)); @@ -839,7 +924,7 @@ __Pyx_async_gen_athrow_dealloc(__pyx_PyAsyncGenAThrow *o) PyObject_GC_UnTrack((PyObject *)o); Py_CLEAR(o->agt_gen); Py_CLEAR(o->agt_args); - PyObject_GC_Del(o); + __Pyx_PyHeapTypeObject_GC_Del(o); } @@ -856,34 +941,56 @@ static PyObject * __Pyx_async_gen_athrow_send(__pyx_PyAsyncGenAThrow *o, PyObject *arg) { __pyx_CoroutineObject *gen = (__pyx_CoroutineObject*)o->agt_gen; - PyObject *retval; + PyObject *retval, *exc_type; - if (o->agt_state == __PYX_AWAITABLE_STATE_CLOSED) { + if (unlikely(o->agt_state == __PYX_AWAITABLE_STATE_CLOSED)) { + PyErr_SetString(PyExc_RuntimeError, __Pyx_ASYNC_GEN_CANNOT_REUSE_CLOSE_MSG); + return NULL; + } + + if (unlikely(gen->resume_label == -1)) { + // already run past the end + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; PyErr_SetNone(PyExc_StopIteration); return NULL; } if (o->agt_state == __PYX_AWAITABLE_STATE_INIT) { - if (o->agt_gen->ag_closed) { - PyErr_SetNone(PyExc_StopIteration); + if (unlikely(o->agt_gen->ag_running_async)) { + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; + if (o->agt_args == NULL) { + PyErr_SetString( + PyExc_RuntimeError, + "aclose(): asynchronous generator is already running"); + } else { + PyErr_SetString( + PyExc_RuntimeError, + "athrow(): asynchronous generator is already running"); + } + return NULL; + } + + if (unlikely(o->agt_gen->ag_closed)) { + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; + PyErr_SetNone(__Pyx_PyExc_StopAsyncIteration); return NULL; } - if (arg != Py_None) { + if (unlikely(arg != Py_None)) { PyErr_SetString(PyExc_RuntimeError, __Pyx_NON_INIT_CORO_MSG); return NULL; } o->agt_state = __PYX_AWAITABLE_STATE_ITER; + o->agt_gen->ag_running_async = 1; if (o->agt_args == NULL) { /* aclose() mode */ o->agt_gen->ag_closed = 1; retval = __Pyx__Coroutine_Throw((PyObject*)gen, - /* Do not close generator when - PyExc_GeneratorExit is passed */ - PyExc_GeneratorExit, NULL, NULL, NULL, 0); + /* Do not close generator when PyExc_GeneratorExit is passed */ + PyExc_GeneratorExit, NULL, NULL, NULL, 0); if (retval && __pyx__PyAsyncGenWrappedValue_CheckExact(retval)) { Py_DECREF(retval); @@ -894,14 +1001,13 @@ __Pyx_async_gen_athrow_send(__pyx_PyAsyncGenAThrow *o, PyObject *arg) PyObject *tb = NULL; PyObject *val = NULL; - if (!PyArg_UnpackTuple(o->agt_args, "athrow", 1, 3, - &typ, &val, &tb)) { + if (unlikely(!PyArg_UnpackTuple(o->agt_args, "athrow", 1, 3, &typ, &val, &tb))) { return NULL; } retval = __Pyx__Coroutine_Throw((PyObject*)gen, - /* Do not close generator when PyExc_GeneratorExit is passed */ - typ, val, tb, o->agt_args, 0); + /* Do not close generator when PyExc_GeneratorExit is passed */ + typ, val, tb, o->agt_args, 0); retval = __Pyx_async_gen_unwrap_value(o->agt_gen, retval); } if (retval == NULL) { @@ -918,7 +1024,7 @@ __Pyx_async_gen_athrow_send(__pyx_PyAsyncGenAThrow *o, PyObject *arg) } else { /* aclose() mode */ if (retval) { - if (__pyx__PyAsyncGenWrappedValue_CheckExact(retval)) { + if (unlikely(__pyx__PyAsyncGenWrappedValue_CheckExact(retval))) { Py_DECREF(retval); goto yield_close; } @@ -932,26 +1038,26 @@ __Pyx_async_gen_athrow_send(__pyx_PyAsyncGenAThrow *o, PyObject *arg) } yield_close: + o->agt_gen->ag_running_async = 0; + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; PyErr_SetString( PyExc_RuntimeError, __Pyx_ASYNC_GEN_IGNORED_EXIT_MSG); return NULL; check_error: - if (PyErr_ExceptionMatches(__Pyx_PyExc_StopAsyncIteration)) { - o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; + o->agt_gen->ag_running_async = 0; + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; + exc_type = PyErr_Occurred(); + if (__Pyx_PyErr_GivenExceptionMatches2(exc_type, __Pyx_PyExc_StopAsyncIteration, PyExc_GeneratorExit)) { if (o->agt_args == NULL) { // when aclose() is called we don't want to propagate - // StopAsyncIteration; just raise StopIteration, signalling - // that 'aclose()' is done. + // StopAsyncIteration or GeneratorExit; just raise + // StopIteration, signalling that this 'aclose()' await + // is done. PyErr_Clear(); PyErr_SetNone(PyExc_StopIteration); } } - else if (PyErr_ExceptionMatches(PyExc_GeneratorExit)) { - o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; - PyErr_Clear(); /* ignore these errors */ - PyErr_SetNone(PyExc_StopIteration); - } return NULL; } @@ -961,13 +1067,8 @@ __Pyx_async_gen_athrow_throw(__pyx_PyAsyncGenAThrow *o, PyObject *args) { PyObject *retval; - if (o->agt_state == __PYX_AWAITABLE_STATE_INIT) { - PyErr_SetString(PyExc_RuntimeError, __Pyx_NON_INIT_CORO_MSG); - return NULL; - } - - if (o->agt_state == __PYX_AWAITABLE_STATE_CLOSED) { - PyErr_SetNone(PyExc_StopIteration); + if (unlikely(o->agt_state == __PYX_AWAITABLE_STATE_CLOSED)) { + PyErr_SetString(PyExc_RuntimeError, __Pyx_ASYNC_GEN_CANNOT_REUSE_CLOSE_MSG); return NULL; } @@ -975,12 +1076,24 @@ __Pyx_async_gen_athrow_throw(__pyx_PyAsyncGenAThrow *o, PyObject *args) if (o->agt_args) { return __Pyx_async_gen_unwrap_value(o->agt_gen, retval); } else { - /* aclose() mode */ - if (retval && __pyx__PyAsyncGenWrappedValue_CheckExact(retval)) { + // aclose() mode + PyObject *exc_type; + if (unlikely(retval && __pyx__PyAsyncGenWrappedValue_CheckExact(retval))) { + o->agt_gen->ag_running_async = 0; + o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; Py_DECREF(retval); PyErr_SetString(PyExc_RuntimeError, __Pyx_ASYNC_GEN_IGNORED_EXIT_MSG); return NULL; } + exc_type = PyErr_Occurred(); + if (__Pyx_PyErr_GivenExceptionMatches2(exc_type, __Pyx_PyExc_StopAsyncIteration, PyExc_GeneratorExit)) { + // when aclose() is called we don't want to propagate + // StopAsyncIteration or GeneratorExit; just raise + // StopIteration, signalling that this 'aclose()' await + // is done. + PyErr_Clear(); + PyErr_SetNone(PyExc_StopIteration); + } return retval; } } @@ -994,9 +1107,10 @@ __Pyx_async_gen_athrow_iternext(__pyx_PyAsyncGenAThrow *o) static PyObject * -__Pyx_async_gen_athrow_close(PyObject *g, CYTHON_UNUSED PyObject *args) +__Pyx_async_gen_athrow_close(PyObject *g, PyObject *args) { __pyx_PyAsyncGenAThrow *o = (__pyx_PyAsyncGenAThrow*) g; + CYTHON_UNUSED_VAR(args); o->agt_state = __PYX_AWAITABLE_STATE_CLOSED; Py_RETURN_NONE; } @@ -1011,6 +1125,27 @@ static PyMethodDef __Pyx_async_gen_athrow_methods[] = { }; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx__PyAsyncGenAThrowType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_async_gen_athrow_dealloc}, + {Py_am_await, (void *)PyObject_SelfIter}, + {Py_tp_traverse, (void *)__Pyx_async_gen_athrow_traverse}, + {Py_tp_iter, (void *)PyObject_SelfIter}, + {Py_tp_iternext, (void *)__Pyx_async_gen_athrow_iternext}, + {Py_tp_methods, (void *)__Pyx_async_gen_athrow_methods}, + {Py_tp_getattro, (void *)__Pyx_PyObject_GenericGetAttrNoDict}, + {0, 0}, +}; + +static PyType_Spec __pyx__PyAsyncGenAThrowType_spec = { + __PYX_TYPE_MODULE_PREFIX "async_generator_athrow", + sizeof(__pyx_PyAsyncGenAThrow), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + __pyx__PyAsyncGenAThrowType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + #if CYTHON_USE_ASYNC_SLOTS static __Pyx_PyAsyncMethodsStruct __Pyx_async_gen_athrow_as_async = { PyObject_SelfIter, /* am_await */ @@ -1022,14 +1157,13 @@ static __Pyx_PyAsyncMethodsStruct __Pyx_async_gen_athrow_as_async = { }; #endif - static PyTypeObject __pyx__PyAsyncGenAThrowType_type = { PyVarObject_HEAD_INIT(0, 0) "async_generator_athrow", /* tp_name */ sizeof(__pyx_PyAsyncGenAThrow), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)__Pyx_async_gen_athrow_dealloc, /* tp_dealloc */ - 0, /* tp_print */ + 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ #if CYTHON_USE_ASYNC_SLOTS @@ -1086,7 +1220,7 @@ static PyTypeObject __pyx__PyAsyncGenAThrowType_type = { #if PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800) 0, /*tp_vectorcall*/ #endif -#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 +#if __PYX_NEED_TP_PRINT_SLOT 0, /*tp_print*/ #endif #if PY_VERSION_HEX >= 0x030C0000 @@ -1096,6 +1230,7 @@ static PyTypeObject __pyx__PyAsyncGenAThrowType_type = { 0, /*tp_pypy_flags*/ #endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ static PyObject * @@ -1103,7 +1238,7 @@ __Pyx_async_gen_athrow_new(__pyx_PyAsyncGenObject *gen, PyObject *args) { __pyx_PyAsyncGenAThrow *o; o = PyObject_GC_New(__pyx_PyAsyncGenAThrow, __pyx__PyAsyncGenAThrowType); - if (o == NULL) { + if (unlikely(o == NULL)) { return NULL; } o->agt_gen = gen; @@ -1118,26 +1253,42 @@ __Pyx_async_gen_athrow_new(__pyx_PyAsyncGenObject *gen, PyObject *args) /* ---------- global type sharing ------------ */ -static int __pyx_AsyncGen_init(void) { +static int __pyx_AsyncGen_init(PyObject *module) { +#if CYTHON_USE_TYPE_SPECS + __pyx_AsyncGenType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx_AsyncGenType_spec, NULL); +#else + CYTHON_MAYBE_UNUSED_VAR(module); // on Windows, C-API functions can't be used in slots statically __pyx_AsyncGenType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; - __pyx__PyAsyncGenWrappedValueType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; - __pyx__PyAsyncGenAThrowType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; - __pyx__PyAsyncGenASendType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; - __pyx_AsyncGenType = __Pyx_FetchCommonType(&__pyx_AsyncGenType_type); +#endif if (unlikely(!__pyx_AsyncGenType)) return -1; +#if CYTHON_USE_TYPE_SPECS + __pyx__PyAsyncGenAThrowType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx__PyAsyncGenAThrowType_spec, NULL); +#else + __pyx__PyAsyncGenAThrowType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; __pyx__PyAsyncGenAThrowType = __Pyx_FetchCommonType(&__pyx__PyAsyncGenAThrowType_type); +#endif if (unlikely(!__pyx__PyAsyncGenAThrowType)) return -1; +#if CYTHON_USE_TYPE_SPECS + __pyx__PyAsyncGenWrappedValueType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx__PyAsyncGenWrappedValueType_spec, NULL); +#else + __pyx__PyAsyncGenWrappedValueType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; __pyx__PyAsyncGenWrappedValueType = __Pyx_FetchCommonType(&__pyx__PyAsyncGenWrappedValueType_type); +#endif if (unlikely(!__pyx__PyAsyncGenWrappedValueType)) return -1; +#if CYTHON_USE_TYPE_SPECS + __pyx__PyAsyncGenASendType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx__PyAsyncGenASendType_spec, NULL); +#else + __pyx__PyAsyncGenASendType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; __pyx__PyAsyncGenASendType = __Pyx_FetchCommonType(&__pyx__PyAsyncGenASendType_type); +#endif if (unlikely(!__pyx__PyAsyncGenASendType)) return -1; diff --git a/Cython/Utility/Buffer.c b/Cython/Utility/Buffer.c index 3c7105fa3..8958b9e50 100644 --- a/Cython/Utility/Buffer.c +++ b/Cython/Utility/Buffer.c @@ -54,8 +54,6 @@ static void __Pyx_RaiseBufferFallbackError(void) { /////////////// BufferFormatStructs.proto /////////////// //@proto_block: utility_code_proto_before_types -#define IS_UNSIGNED(type) (((type) -1) > 0) - /* Run-time type information about structs used with buffers */ struct __Pyx_StructField_; @@ -111,6 +109,7 @@ typedef struct { #if PY_MAJOR_VERSION < 3 static int __Pyx_GetBuffer(PyObject *obj, Py_buffer *view, int flags) { + __Pyx_TypeName obj_type_name; if (PyObject_CheckBuffer(obj)) return PyObject_GetBuffer(obj, view, flags); {{for type_ptr, getbuffer, releasebuffer in types}} @@ -119,7 +118,11 @@ static int __Pyx_GetBuffer(PyObject *obj, Py_buffer *view, int flags) { {{endif}} {{endfor}} - PyErr_Format(PyExc_TypeError, "'%.200s' does not have the buffer interface", Py_TYPE(obj)->tp_name); + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'" __Pyx_FMT_TYPENAME "' does not have the buffer interface", + obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); return -1; } @@ -224,8 +227,8 @@ fail:; // the format string; the access mode/flags is checked by the // exporter. See: // -// http://docs.python.org/3/library/struct.html -// http://legacy.python.org/dev/peps/pep-3118/#additions-to-the-struct-string-syntax +// https://docs.python.org/3/library/struct.html +// https://www.python.org/dev/peps/pep-3118/#additions-to-the-struct-string-syntax // // The alignment code is copied from _struct.c in Python. @@ -372,7 +375,8 @@ typedef struct { char c; void *x; } __Pyx_st_void_p; typedef struct { char c; PY_LONG_LONG x; } __Pyx_st_longlong; #endif -static size_t __Pyx_BufFmt_TypeCharToAlignment(char ch, CYTHON_UNUSED int is_complex) { +static size_t __Pyx_BufFmt_TypeCharToAlignment(char ch, int is_complex) { + CYTHON_UNUSED_VAR(is_complex); switch (ch) { case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; case 'h': case 'H': return sizeof(__Pyx_st_short) - sizeof(short); @@ -406,7 +410,8 @@ typedef struct { void *x; char c; } __Pyx_pad_void_p; typedef struct { PY_LONG_LONG x; char c; } __Pyx_pad_longlong; #endif -static size_t __Pyx_BufFmt_TypeCharToPadding(char ch, CYTHON_UNUSED int is_complex) { +static size_t __Pyx_BufFmt_TypeCharToPadding(char ch, int is_complex) { + CYTHON_UNUSED_VAR(is_complex); switch (ch) { case '?': case 'c': case 'b': case 'B': case 's': case 'p': return 1; case 'h': case 'H': return sizeof(__Pyx_pad_short) - sizeof(short); diff --git a/Cython/Utility/Builtins.c b/Cython/Utility/Builtins.c index 32aeff8f2..b77f95105 100644 --- a/Cython/Utility/Builtins.c +++ b/Cython/Utility/Builtins.c @@ -20,47 +20,7 @@ static PyObject* __Pyx_Globals(void); /*proto*/ // access requires a rewrite as a dedicated class. static PyObject* __Pyx_Globals(void) { - Py_ssize_t i; - PyObject *names; - PyObject *globals = $moddict_cname; - Py_INCREF(globals); - names = PyObject_Dir($module_cname); - if (!names) - goto bad; - for (i = PyList_GET_SIZE(names)-1; i >= 0; i--) { -#if CYTHON_COMPILING_IN_PYPY - PyObject* name = PySequence_ITEM(names, i); - if (!name) - goto bad; -#else - PyObject* name = PyList_GET_ITEM(names, i); -#endif - if (!PyDict_Contains(globals, name)) { - PyObject* value = __Pyx_GetAttr($module_cname, name); - if (!value) { -#if CYTHON_COMPILING_IN_PYPY - Py_DECREF(name); -#endif - goto bad; - } - if (PyDict_SetItem(globals, name, value) < 0) { -#if CYTHON_COMPILING_IN_PYPY - Py_DECREF(name); -#endif - Py_DECREF(value); - goto bad; - } - } -#if CYTHON_COMPILING_IN_PYPY - Py_DECREF(name); -#endif - } - Py_DECREF(names); - return globals; -bad: - Py_XDECREF(names); - Py_XDECREF(globals); - return NULL; + return __Pyx_NewRef($moddict_cname); } //////////////////// PyExecGlobals.proto //////////////////// @@ -68,17 +28,11 @@ bad: static PyObject* __Pyx_PyExecGlobals(PyObject*); //////////////////// PyExecGlobals //////////////////// -//@requires: Globals +//@substitute: naming //@requires: PyExec static PyObject* __Pyx_PyExecGlobals(PyObject* code) { - PyObject* result; - PyObject* globals = __Pyx_Globals(); - if (unlikely(!globals)) - return NULL; - result = __Pyx_PyExec2(code, globals); - Py_DECREF(globals); - return result; + return __Pyx_PyExec2(code, $moddict_cname); } //////////////////// PyExec.proto //////////////////// @@ -100,9 +54,13 @@ static PyObject* __Pyx_PyExec3(PyObject* o, PyObject* globals, PyObject* locals) if (!globals || globals == Py_None) { globals = $moddict_cname; - } else if (!PyDict_Check(globals)) { - PyErr_Format(PyExc_TypeError, "exec() arg 2 must be a dict, not %.200s", - Py_TYPE(globals)->tp_name); + } else if (unlikely(!PyDict_Check(globals))) { + __Pyx_TypeName globals_type_name = + __Pyx_PyType_GetName(Py_TYPE(globals)); + PyErr_Format(PyExc_TypeError, + "exec() arg 2 must be a dict, not " __Pyx_FMT_TYPENAME, + globals_type_name); + __Pyx_DECREF_TypeName(globals_type_name); goto bad; } if (!locals || locals == Py_None) { @@ -110,12 +68,12 @@ static PyObject* __Pyx_PyExec3(PyObject* o, PyObject* globals, PyObject* locals) } if (__Pyx_PyDict_GetItemStr(globals, PYIDENT("__builtins__")) == NULL) { - if (PyDict_SetItem(globals, PYIDENT("__builtins__"), PyEval_GetBuiltins()) < 0) + if (unlikely(PyDict_SetItem(globals, PYIDENT("__builtins__"), PyEval_GetBuiltins()) < 0)) goto bad; } if (PyCode_Check(o)) { - if (__Pyx_PyCode_HasFreeVars((PyCodeObject *)o)) { + if (unlikely(__Pyx_PyCode_HasFreeVars((PyCodeObject *)o))) { PyErr_SetString(PyExc_TypeError, "code object passed to exec() may not contain free variables"); goto bad; @@ -134,16 +92,18 @@ static PyObject* __Pyx_PyExec3(PyObject* o, PyObject* globals, PyObject* locals) if (PyUnicode_Check(o)) { cf.cf_flags = PyCF_SOURCE_IS_UTF8; s = PyUnicode_AsUTF8String(o); - if (!s) goto bad; + if (unlikely(!s)) goto bad; o = s; #if PY_MAJOR_VERSION >= 3 - } else if (!PyBytes_Check(o)) { + } else if (unlikely(!PyBytes_Check(o))) { #else - } else if (!PyString_Check(o)) { + } else if (unlikely(!PyString_Check(o))) { #endif + __Pyx_TypeName o_type_name = __Pyx_PyType_GetName(Py_TYPE(o)); PyErr_Format(PyExc_TypeError, - "exec: arg 1 must be string, bytes or code object, got %.200s", - Py_TYPE(o)->tp_name); + "exec: arg 1 must be string, bytes or code object, got " __Pyx_FMT_TYPENAME, + o_type_name); + __Pyx_DECREF_TypeName(o_type_name); goto bad; } #if PY_MAJOR_VERSION >= 3 @@ -170,7 +130,7 @@ bad: static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *, PyObject *, PyObject *); /*proto*/ //////////////////// GetAttr3 //////////////////// -//@requires: ObjectHandling.c::GetAttr +//@requires: ObjectHandling.c::PyObjectGetAttrStr //@requires: Exceptions.c::PyThreadStateGet //@requires: Exceptions.c::PyErrFetchRestore //@requires: Exceptions.c::PyErrExceptionMatches @@ -186,7 +146,17 @@ static PyObject *__Pyx_GetAttr3Default(PyObject *d) { } static CYTHON_INLINE PyObject *__Pyx_GetAttr3(PyObject *o, PyObject *n, PyObject *d) { - PyObject *r = __Pyx_GetAttr(o, n); + PyObject *r; +#if CYTHON_USE_TYPE_SLOTS + if (likely(PyString_Check(n))) { + r = __Pyx_PyObject_GetAttrStrNoError(o, n); + if (unlikely(!r) && likely(!PyErr_Occurred())) { + r = __Pyx_NewRef(d); + } + return r; + } +#endif + r = PyObject_GetAttr(o, n); return (likely(r)) ? r : __Pyx_GetAttr3Default(d); } @@ -205,7 +175,7 @@ static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) { return -1; } r = __Pyx_GetAttr(o, n); - if (unlikely(!r)) { + if (!r) { PyErr_Clear(); return 0; } else { @@ -219,11 +189,12 @@ static CYTHON_INLINE int __Pyx_HasAttr(PyObject *o, PyObject *n) { static PyObject* __Pyx_Intern(PyObject* s); /* proto */ //////////////////// Intern //////////////////// +//@requires: ObjectHandling.c::RaiseUnexpectedTypeError static PyObject* __Pyx_Intern(PyObject* s) { - if (!(likely(PyString_CheckExact(s)))) { - PyErr_Format(PyExc_TypeError, "Expected %.16s, got %.200s", "str", Py_TYPE(s)->tp_name); - return 0; + if (unlikely(!PyString_CheckExact(s))) { + __Pyx_RaiseUnexpectedTypeError("str", s); + return NULL; } Py_INCREF(s); #if PY_MAJOR_VERSION >= 3 @@ -263,7 +234,7 @@ static PyObject *__Pyx_PyLong_AbsNeg(PyObject *num);/*proto*/ #define __Pyx_PyNumber_Absolute(x) \ ((likely(PyLong_CheckExact(x))) ? \ - (likely(Py_SIZE(x) >= 0) ? (Py_INCREF(x), (x)) : __Pyx_PyLong_AbsNeg(x)) : \ + (likely(__Pyx_PyLong_IsNonNeg(x)) ? (Py_INCREF(x), (x)) : __Pyx_PyLong_AbsNeg(x)) : \ PyNumber_Absolute(x)) #else @@ -274,16 +245,27 @@ static PyObject *__Pyx_PyLong_AbsNeg(PyObject *num);/*proto*/ #if CYTHON_USE_PYLONG_INTERNALS static PyObject *__Pyx_PyLong_AbsNeg(PyObject *n) { +#if PY_VERSION_HEX >= 0x030C00A7 + if (likely(__Pyx_PyLong_IsCompact(n))) { + return PyLong_FromSize_t(__Pyx_PyLong_CompactValueUnsigned(n)); + } +#else if (likely(Py_SIZE(n) == -1)) { // digits are unsigned - return PyLong_FromLong(((PyLongObject*)n)->ob_digit[0]); + return PyLong_FromUnsignedLong(__Pyx_PyLong_Digits(n)[0]); } +#endif #if CYTHON_COMPILING_IN_CPYTHON { PyObject *copy = _PyLong_Copy((PyLongObject*)n); if (likely(copy)) { + #if PY_VERSION_HEX >= 0x030C00A7 + // clear the sign bits to set the sign from SIGN_NEGATIVE (2) to positive (0) + ((PyLongObject*)copy)->long_value.lv_tag = ((PyLongObject*)copy)->long_value.lv_tag & ~3; + #else // negate the size to swap the sign __Pyx_SET_SIZE(copy, -Py_SIZE(copy)); + #endif } return copy; } @@ -299,6 +281,42 @@ static PyObject *__Pyx_PyLong_AbsNeg(PyObject *n) { #define __Pyx_PyNumber_Power2(a, b) PyNumber_Power(a, b, Py_None) +//////////////////// int_pyucs4.proto //////////////////// + +static CYTHON_INLINE int __Pyx_int_from_UCS4(Py_UCS4 uchar); + +//////////////////// int_pyucs4 //////////////////// + +static int __Pyx_int_from_UCS4(Py_UCS4 uchar) { + int digit = Py_UNICODE_TODIGIT(uchar); + if (unlikely(digit < 0)) { + PyErr_Format(PyExc_ValueError, + "invalid literal for int() with base 10: '%c'", + (int) uchar); + return -1; + } + return digit; +} + + +//////////////////// float_pyucs4.proto //////////////////// + +static CYTHON_INLINE double __Pyx_double_from_UCS4(Py_UCS4 uchar); + +//////////////////// float_pyucs4 //////////////////// + +static double __Pyx_double_from_UCS4(Py_UCS4 uchar) { + double digit = Py_UNICODE_TONUMERIC(uchar); + if (unlikely(digit < 0.0)) { + PyErr_Format(PyExc_ValueError, + "could not convert string to float: '%c'", + (int) uchar); + return -1.0; + } + return digit; +} + + //////////////////// object_ord.proto //////////////////// //@requires: TypeConversion.c::UnicodeAsUCS4 @@ -332,8 +350,11 @@ static long __Pyx__PyObject_Ord(PyObject* c) { #endif } else { // FIXME: support character buffers - but CPython doesn't support them either + __Pyx_TypeName c_type_name = __Pyx_PyType_GetName(Py_TYPE(c)); PyErr_Format(PyExc_TypeError, - "ord() expected string of length 1, but %.200s found", Py_TYPE(c)->tp_name); + "ord() expected string of length 1, but " __Pyx_FMT_TYPENAME " found", + c_type_name); + __Pyx_DECREF_TypeName(c_type_name); return (long)(Py_UCS4)-1; } PyErr_Format(PyExc_TypeError, @@ -422,9 +443,6 @@ static CYTHON_INLINE PyObject* __Pyx_PyDict_IterItems(PyObject* d) { //////////////////// py_dict_viewkeys.proto //////////////////// -#if PY_VERSION_HEX < 0x02070000 -#error This module uses dict views, which require Python 2.7 or later -#endif static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewKeys(PyObject* d); /*proto*/ //////////////////// py_dict_viewkeys //////////////////// @@ -438,9 +456,6 @@ static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewKeys(PyObject* d) { //////////////////// py_dict_viewvalues.proto //////////////////// -#if PY_VERSION_HEX < 0x02070000 -#error This module uses dict views, which require Python 2.7 or later -#endif static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewValues(PyObject* d); /*proto*/ //////////////////// py_dict_viewvalues //////////////////// @@ -454,9 +469,6 @@ static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewValues(PyObject* d) { //////////////////// py_dict_viewitems.proto //////////////////// -#if PY_VERSION_HEX < 0x02070000 -#error This module uses dict views, which require Python 2.7 or later -#endif static CYTHON_INLINE PyObject* __Pyx_PyDict_ViewItems(PyObject* d); /*proto*/ //////////////////// py_dict_viewitems //////////////////// @@ -540,3 +552,50 @@ static CYTHON_INLINE int __Pyx_PySet_Update(PyObject* set, PyObject* it) { Py_DECREF(retval); return 0; } + +///////////////// memoryview_get_from_buffer.proto //////////////////// + +// buffer is in limited api from Py3.11 +#if !CYTHON_COMPILING_IN_LIMITED_API || CYTHON_LIMITED_API >= 0x030b0000 +#define __Pyx_PyMemoryView_Get_{{name}}(o) PyMemoryView_GET_BUFFER(o)->{{name}} +#else +{{py: +out_types = dict( + ndim='int', readonly='int', + len='Py_ssize_t', itemsize='Py_ssize_t') +}} // can't get format like this unfortunately. It's unicode via getattr +{{py: out_type = out_types[name]}} +static {{out_type}} __Pyx_PyMemoryView_Get_{{name}}(PyObject *obj); /* proto */ +#endif + +////////////// memoryview_get_from_buffer ///////////////////////// + +#if !CYTHON_COMPILING_IN_LIMITED_API || CYTHON_LIMITED_API >= 0x030b0000 +#else +{{py: +out_types = dict( + ndim='int', readonly='int', + len='Py_ssize_t', itemsize='Py_ssize_t') +}} +{{py: out_type = out_types[name]}} +static {{out_type}} __Pyx_PyMemoryView_Get_{{name}}(PyObject *obj) { + {{out_type}} result; + PyObject *attr = PyObject_GetAttr(obj, PYIDENT("{{name}}")); + if (!attr) { + goto bad; + } +{{if out_type == 'int'}} + // I'm not worrying about overflow here because + // ultimately it comes from a C struct that's an int + result = PyLong_AsLong(attr); +{{elif out_type == 'Py_ssize_t'}} + result = PyLong_AsSsize_t(attr); +{{endif}} + Py_DECREF(attr); + return result; + + bad: + Py_XDECREF(attr); + return -1; +} +#endif diff --git a/Cython/Utility/CConvert.pyx b/Cython/Utility/CConvert.pyx index 5969f6a58..4ae66162e 100644 --- a/Cython/Utility/CConvert.pyx +++ b/Cython/Utility/CConvert.pyx @@ -6,19 +6,20 @@ cdef extern from *: PyTypeObject *Py_TYPE(obj) bint PyMapping_Check(obj) object PyErr_Format(exc, const char *format, ...) + int __Pyx_RaiseUnexpectedTypeError(const char *expected, object obj) except 0 @cname("{{funcname}}") cdef {{struct_type}} {{funcname}}(obj) except *: cdef {{struct_type}} result if not PyMapping_Check(obj): - PyErr_Format(TypeError, b"Expected %.16s, got %.200s", b"a mapping", Py_TYPE(obj).tp_name) + __Pyx_RaiseUnexpectedTypeError(b"a mapping", obj) {{for member in var_entries:}} try: value = obj['{{member.name}}'] except KeyError: raise ValueError("No value specified for struct attribute '{{member.name}}'") - result.{{member.cname}} = value + result.{{member.name}} = value {{endfor}} return result @@ -31,13 +32,14 @@ cdef extern from *: PyTypeObject *Py_TYPE(obj) bint PyMapping_Check(obj) object PyErr_Format(exc, const char *format, ...) + int __Pyx_RaiseUnexpectedTypeError(const char *expected, object obj) except 0 @cname("{{funcname}}") cdef {{struct_type}} {{funcname}}(obj) except *: cdef {{struct_type}} result cdef Py_ssize_t length if not PyMapping_Check(obj): - PyErr_Format(TypeError, b"Expected %.16s, got %.200s", b"a mapping", Py_TYPE(obj).tp_name) + __Pyx_RaiseUnexpectedTypeError(b"a mapping", obj) last_found = None length = len(obj) diff --git a/Cython/Utility/Capsule.c b/Cython/Utility/Capsule.c deleted file mode 100644 index cc4fe0d88..000000000 --- a/Cython/Utility/Capsule.c +++ /dev/null @@ -1,20 +0,0 @@ -//////////////// Capsule.proto //////////////// - -/* Todo: wrap the rest of the functionality in similar functions */ -static CYTHON_INLINE PyObject *__pyx_capsule_create(void *p, const char *sig); - -//////////////// Capsule //////////////// - -static CYTHON_INLINE PyObject * -__pyx_capsule_create(void *p, CYTHON_UNUSED const char *sig) -{ - PyObject *cobj; - -#if PY_VERSION_HEX >= 0x02070000 - cobj = PyCapsule_New(p, sig, NULL); -#else - cobj = PyCObject_FromVoidPtr(p, NULL); -#endif - - return cobj; -} diff --git a/Cython/Utility/CommonStructures.c b/Cython/Utility/CommonStructures.c index c7945feb4..f39f3d70d 100644 --- a/Cython/Utility/CommonStructures.c +++ b/Cython/Utility/CommonStructures.c @@ -1,43 +1,80 @@ +/////////////// FetchSharedCythonModule.proto /////// + +static PyObject *__Pyx_FetchSharedCythonABIModule(void); + +/////////////// FetchSharedCythonModule //////////// + +static PyObject *__Pyx_FetchSharedCythonABIModule(void) { + PyObject *abi_module = PyImport_AddModule((char*) __PYX_ABI_MODULE_NAME); + if (unlikely(!abi_module)) return NULL; + Py_INCREF(abi_module); + return abi_module; +} + /////////////// FetchCommonType.proto /////////////// +#if !CYTHON_USE_TYPE_SPECS static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); +#else +static PyTypeObject* __Pyx_FetchCommonTypeFromSpec(PyObject *module, PyType_Spec *spec, PyObject *bases); +#endif /////////////// FetchCommonType /////////////// +//@requires:ExtensionTypes.c::FixUpExtensionType +//@requires: FetchSharedCythonModule +//@requires:StringTools.c::IncludeStringH + +static int __Pyx_VerifyCachedType(PyObject *cached_type, + const char *name, + Py_ssize_t basicsize, + Py_ssize_t expected_basicsize) { + if (!PyType_Check(cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", name); + return -1; + } + if (basicsize != expected_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + name); + return -1; + } + return 0; +} +#if !CYTHON_USE_TYPE_SPECS static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { - PyObject* fake_module; - PyTypeObject* cached_type = NULL; - - fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); - if (!fake_module) return NULL; - Py_INCREF(fake_module); - - cached_type = (PyTypeObject*) PyObject_GetAttrString(fake_module, type->tp_name); + PyObject* abi_module; + const char* object_name; + PyTypeObject *cached_type = NULL; + + abi_module = __Pyx_FetchSharedCythonABIModule(); + if (!abi_module) return NULL; + // get the final part of the object name (after the last dot) + object_name = strrchr(type->tp_name, '.'); + object_name = object_name ? object_name+1 : type->tp_name; + cached_type = (PyTypeObject*) PyObject_GetAttrString(abi_module, object_name); if (cached_type) { - if (!PyType_Check((PyObject*)cached_type)) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s is not a type object", - type->tp_name); + if (__Pyx_VerifyCachedType( + (PyObject *)cached_type, + object_name, + cached_type->tp_basicsize, + type->tp_basicsize) < 0) { goto bad; } - if (cached_type->tp_basicsize != type->tp_basicsize) { - PyErr_Format(PyExc_TypeError, - "Shared Cython type %.200s has the wrong size, try recompiling", - type->tp_name); - goto bad; - } - } else { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; - PyErr_Clear(); - if (PyType_Ready(type) < 0) goto bad; - if (PyObject_SetAttrString(fake_module, type->tp_name, (PyObject*) type) < 0) - goto bad; - Py_INCREF(type); - cached_type = type; + goto done; } + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; + PyErr_Clear(); + if (PyType_Ready(type) < 0) goto bad; + if (PyObject_SetAttrString(abi_module, object_name, (PyObject *)type) < 0) + goto bad; + Py_INCREF(type); + cached_type = type; + done: - Py_DECREF(fake_module); + Py_DECREF(abi_module); // NOTE: always returns owned reference, or NULL on error return cached_type; @@ -46,41 +83,60 @@ bad: cached_type = NULL; goto done; } +#else +static PyTypeObject *__Pyx_FetchCommonTypeFromSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *abi_module, *cached_type = NULL; + // get the final part of the object name (after the last dot) + const char* object_name = strrchr(spec->name, '.'); + object_name = object_name ? object_name+1 : spec->name; -/////////////// FetchCommonPointer.proto /////////////// - -static void* __Pyx_FetchCommonPointer(void* pointer, const char* name); - -/////////////// FetchCommonPointer /////////////// - - -static void* __Pyx_FetchCommonPointer(void* pointer, const char* name) { -#if PY_VERSION_HEX >= 0x02070000 - PyObject* fake_module = NULL; - PyObject* capsule = NULL; - void* value = NULL; - - fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); - if (!fake_module) return NULL; - Py_INCREF(fake_module); + abi_module = __Pyx_FetchSharedCythonABIModule(); + if (!abi_module) return NULL; - capsule = PyObject_GetAttrString(fake_module, name); - if (!capsule) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; - PyErr_Clear(); - capsule = PyCapsule_New(pointer, name, NULL); - if (!capsule) goto bad; - if (PyObject_SetAttrString(fake_module, name, capsule) < 0) + cached_type = PyObject_GetAttrString(abi_module, object_name); + if (cached_type) { + Py_ssize_t basicsize; +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_basicsize; + py_basicsize = PyObject_GetAttrString(cached_type, "__basicsize__"); + if (unlikely(!py_basicsize)) goto bad; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = 0; + if (unlikely(basicsize == (Py_ssize_t)-1) && PyErr_Occurred()) goto bad; +#else + basicsize = likely(PyType_Check(cached_type)) ? ((PyTypeObject*) cached_type)->tp_basicsize : -1; +#endif + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + basicsize, + spec->basicsize) < 0) { goto bad; + } + goto done; } - value = PyCapsule_GetPointer(capsule, name); + + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; + PyErr_Clear(); + // We pass the ABI module reference to avoid keeping the user module alive by foreign type usages. + CYTHON_UNUSED_VAR(module); + cached_type = __Pyx_PyType_FromModuleAndSpec(abi_module, spec, bases); + if (unlikely(!cached_type)) goto bad; + if (unlikely(__Pyx_fix_up_extension_type_from_spec(spec, (PyTypeObject *) cached_type) < 0)) goto bad; + if (PyObject_SetAttrString(abi_module, object_name, cached_type) < 0) goto bad; + +done: + Py_DECREF(abi_module); + // NOTE: always returns owned reference, or NULL on error + assert(cached_type == NULL || PyType_Check(cached_type)); + return (PyTypeObject *) cached_type; bad: - Py_XDECREF(capsule); - Py_DECREF(fake_module); - return value; -#else - return pointer; -#endif + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; } +#endif + diff --git a/Cython/Utility/Complex.c b/Cython/Utility/Complex.c index 15d5f544d..fd2cd081a 100644 --- a/Cython/Utility/Complex.c +++ b/Cython/Utility/Complex.c @@ -4,7 +4,8 @@ #if !defined(CYTHON_CCOMPLEX) #if defined(__cplusplus) #define CYTHON_CCOMPLEX 1 - #elif defined(_Complex_I) + #elif defined(_Complex_I) || (__STDC_VERSION__ >= 201112L && !defined(__STDC_NO_COMPLEX__)) + // <complex.h> should exist since C99, but only C11 defines a test to detect it #define CYTHON_CCOMPLEX 1 #else #define CYTHON_CCOMPLEX 0 @@ -24,7 +25,6 @@ #define _Complex_I 1.0fj #endif - /////////////// RealImag.proto /////////////// #if CYTHON_CCOMPLEX @@ -49,11 +49,42 @@ #define __Pyx_SET_CIMAG(z,y) __Pyx_CIMAG(z) = (y) #endif +/////////////// RealImag_Cy.proto /////////////// + +// alternative version of RealImag.proto for the case where +// we definitely want to force it to use the Cython utility +// code version of complex. +// Because integer complex types simply aren't covered by +// the C or C++ standards +// (although practically will probably work in C++). + +#define __Pyx_CREAL_Cy(z) ((z).real) +#define __Pyx_CIMAG_Cy(z) ((z).imag) +#define __Pyx_SET_CREAL_Cy(z,x) __Pyx_CREAL_Cy(z) = (x) +#define __Pyx_SET_CIMAG_Cy(z,y) __Pyx_CIMAG_cy(z) = (y) + +/////////////// RealImag_CyTypedef.proto ////////// +//@requires: RealImag +//@requires: RealImag_Cy + +#if __cplusplus +// C++ is fine with complexes based on typedefs because the template sees through them +#define __Pyx_CREAL_CyTypedef(z) __Pyx_CREAL(z) +#define __Pyx_CIMAG_CyTypedef(z) __Pyx_CIMAG(z) +#define __Pyx_SET_CREAL_CyTypedef(z,x) __Pyx_SET_CREAL(z) +#define __Pyx_SET_CIMAG_CyTypedef(z,x) __Pyx_SET_CIMAG(z) +#else +// C isn't +#define __Pyx_CREAL_CyTypedef(z) __Pyx_CREAL_Cy(z) +#define __Pyx_CIMAG_CyTypedef(z) __Pyx_CIMAG_Cy(z) +#define __Pyx_SET_CREAL_CyTypedef(z,x) __Pyx_SET_CREAL_Cy(z) +#define __Pyx_SET_CIMAG_CyTypedef(z,x) __Pyx_SET_CIMAG_Cy(z) +#endif /////////////// Declarations.proto /////////////// //@proto_block: complex_type_declarations -#if CYTHON_CCOMPLEX +#if CYTHON_CCOMPLEX && ({{is_float}}) && (!{{is_extern_float_typedef}} || __cplusplus) #ifdef __cplusplus typedef ::std::complex< {{real_type}} > {{type_name}}; #else @@ -67,7 +98,7 @@ static CYTHON_INLINE {{type}} {{type_name}}_from_parts({{real_type}}, {{real_typ /////////////// Declarations /////////////// -#if CYTHON_CCOMPLEX +#if CYTHON_CCOMPLEX && ({{is_float}}) && (!{{is_extern_float_typedef}} || __cplusplus) #ifdef __cplusplus static CYTHON_INLINE {{type}} {{type_name}}_from_parts({{real_type}} x, {{real_type}} y) { return ::std::complex< {{real_type}} >(x, y); @@ -89,10 +120,10 @@ static CYTHON_INLINE {{type}} {{type_name}}_from_parts({{real_type}}, {{real_typ /////////////// ToPy.proto /////////////// -#define __pyx_PyComplex_FromComplex(z) \ - PyComplex_FromDoubles((double)__Pyx_CREAL(z), \ - (double)__Pyx_CIMAG(z)) - +{{py: func_suffix = "_CyTypedef" if is_extern_float_typedef else ("" if is_float else "_Cy")}} +#define __pyx_PyComplex_FromComplex{{func_suffix}}(z) \ + PyComplex_FromDoubles((double)__Pyx_CREAL{{func_suffix}}(z), \ + (double)__Pyx_CIMAG{{func_suffix}}(z)) /////////////// FromPy.proto /////////////// @@ -116,7 +147,7 @@ static {{type}} __Pyx_PyComplex_As_{{type_name}}(PyObject* o) { /////////////// Arithmetic.proto /////////////// -#if CYTHON_CCOMPLEX +#if CYTHON_CCOMPLEX && ({{is_float}}) && (!{{is_extern_float_typedef}} || __cplusplus) #define __Pyx_c_eq{{func_suffix}}(a, b) ((a)==(b)) #define __Pyx_c_sum{{func_suffix}}(a, b) ((a)+(b)) #define __Pyx_c_diff{{func_suffix}}(a, b) ((a)-(b)) @@ -155,7 +186,7 @@ static {{type}} __Pyx_PyComplex_As_{{type_name}}(PyObject* o) { /////////////// Arithmetic /////////////// -#if CYTHON_CCOMPLEX +#if CYTHON_CCOMPLEX && ({{is_float}}) && (!{{is_extern_float_typedef}} || __cplusplus) #else static CYTHON_INLINE int __Pyx_c_eq{{func_suffix}}({{type}} a, {{type}} b) { return (a.real == b.real) && (a.imag == b.imag); @@ -289,3 +320,46 @@ static {{type}} __Pyx_PyComplex_As_{{type_name}}(PyObject* o) { } #endif #endif + +/////////////// SoftComplexToDouble.proto ////////////////// + +static double __Pyx_SoftComplexToDouble(__pyx_t_double_complex value, int have_gil); /* proto */ + +/////////////// SoftComplexToDouble ////////////////// +//@requires: RealImag + +static double __Pyx_SoftComplexToDouble(__pyx_t_double_complex value, int have_gil) { + // This isn't an absolutely perfect match for the Python behaviour: + // In Python the type would be determined right after the number is + // created (usually '**'), while here it's determined when coerced + // to a PyObject, which may be a few operations later. + if (unlikely(__Pyx_CIMAG(value))) { + PyGILState_STATE gilstate; + if (!have_gil) + gilstate = PyGILState_Ensure(); + PyErr_SetString(PyExc_TypeError, + "Cannot convert 'complex' with non-zero imaginary component to 'double' " + "(this most likely comes from the '**' operator; " + "use 'cython.cpow(True)' to return 'nan' instead of a " + "complex number)."); + if (!have_gil) + PyGILState_Release(gilstate); + return -1.; + } + return __Pyx_CREAL(value); +} + +///////// SoftComplexToPy.proto /////////////////////// + +static PyObject *__pyx_Py_FromSoftComplex(__pyx_t_double_complex value); /* proto */ + +//////// SoftComplexToPy //////////////// +//@requires: RealImag + +static PyObject *__pyx_Py_FromSoftComplex(__pyx_t_double_complex value) { + if (__Pyx_CIMAG(value)) { + return PyComplex_FromDoubles(__Pyx_CREAL(value), __Pyx_CIMAG(value)); + } else { + return PyFloat_FromDouble(__Pyx_CREAL(value)); + } +} diff --git a/Cython/Utility/Coroutine.c b/Cython/Utility/Coroutine.c index aaa8a8e26..8d3c64a4f 100644 --- a/Cython/Utility/Coroutine.c +++ b/Cython/Utility/Coroutine.c @@ -5,12 +5,15 @@ static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject //////////////////// GeneratorYieldFrom //////////////////// //@requires: Generator -static void __PyxPyIter_CheckErrorAndDecref(PyObject *source) { +#if CYTHON_USE_TYPE_SLOTS +static void __Pyx_PyIter_CheckErrorAndDecref(PyObject *source) { + __Pyx_TypeName source_type_name = __Pyx_PyType_GetName(Py_TYPE(source)); PyErr_Format(PyExc_TypeError, - "iter() returned non-iterator of type '%.100s'", - Py_TYPE(source)->tp_name); + "iter() returned non-iterator of type '" __Pyx_FMT_TYPENAME "'", source_type_name); + __Pyx_DECREF_TypeName(source_type_name); Py_DECREF(source); } +#endif static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject *gen, PyObject *source) { PyObject *source_gen, *retval; @@ -29,7 +32,7 @@ static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject if (unlikely(!source_gen)) return NULL; if (unlikely(!PyIter_Check(source_gen))) { - __PyxPyIter_CheckErrorAndDecref(source_gen); + __Pyx_PyIter_CheckErrorAndDecref(source_gen); return NULL; } } else @@ -41,11 +44,7 @@ static CYTHON_INLINE PyObject* __Pyx_Generator_Yield_From(__pyx_CoroutineObject return NULL; } // source_gen is now the iterator, make the first next() call -#if CYTHON_USE_TYPE_SLOTS - retval = Py_TYPE(source_gen)->tp_iternext(source_gen); -#else - retval = PyIter_Next(source_gen); -#endif + retval = __Pyx_PyObject_GetIterNextFunc(source_gen)(source_gen); } if (likely(retval)) { gen->yieldfrom = source_gen; @@ -74,11 +73,7 @@ static PyObject* __Pyx__Coroutine_Yield_From_Generic(__pyx_CoroutineObject *gen, if (__Pyx_Coroutine_Check(source_gen)) { retval = __Pyx_Generator_Next(source_gen); } else { -#if CYTHON_USE_TYPE_SLOTS - retval = Py_TYPE(source_gen)->tp_iternext(source_gen); -#else - retval = PyIter_Next(source_gen); -#endif + retval = __Pyx_PyObject_GetIterNextFunc(source_gen)(source_gen); } if (retval) { gen->yieldfrom = source_gen; @@ -136,13 +131,13 @@ static CYTHON_INLINE PyObject *__Pyx_Coroutine_GetAwaitableIter(PyObject *o) { static void __Pyx_Coroutine_AwaitableIterError(PyObject *source) { #if PY_VERSION_HEX >= 0x030600B3 || defined(_PyErr_FormatFromCause) - _PyErr_FormatFromCause( - PyExc_TypeError, - "'async for' received an invalid object " - "from __anext__: %.100s", - Py_TYPE(source)->tp_name); + __Pyx_TypeName source_type_name = __Pyx_PyType_GetName(Py_TYPE(source)); + _PyErr_FormatFromCause(PyExc_TypeError, + "'async for' received an invalid object from __anext__: " __Pyx_FMT_TYPENAME, source_type_name); + __Pyx_DECREF_TypeName(source_type_name); #elif PY_MAJOR_VERSION >= 3 PyObject *exc, *val, *val2, *tb; + __Pyx_TypeName source_type_name = __Pyx_PyType_GetName(Py_TYPE(source)); assert(PyErr_Occurred()); PyErr_Fetch(&exc, &val, &tb); PyErr_NormalizeException(&exc, &val, &tb); @@ -152,11 +147,9 @@ static void __Pyx_Coroutine_AwaitableIterError(PyObject *source) { } Py_DECREF(exc); assert(!PyErr_Occurred()); - PyErr_Format( - PyExc_TypeError, - "'async for' received an invalid object " - "from __anext__: %.100s", - Py_TYPE(source)->tp_name); + PyErr_Format(PyExc_TypeError, + "'async for' received an invalid object from __anext__: " __Pyx_FMT_TYPENAME, source_type_name); + __Pyx_DECREF_TypeName(source_type_name); PyErr_Fetch(&exc, &val2, &tb); PyErr_NormalizeException(&exc, &val2, &tb); @@ -211,9 +204,10 @@ static PyObject *__Pyx__Coroutine_GetAwaitableIter(PyObject *obj) { goto bad; } if (unlikely(!PyIter_Check(res))) { + __Pyx_TypeName res_type_name = __Pyx_PyType_GetName(Py_TYPE(res)); PyErr_Format(PyExc_TypeError, - "__await__() returned non-iterator of type '%.100s'", - Py_TYPE(res)->tp_name); + "__await__() returned non-iterator of type '" __Pyx_FMT_TYPENAME "'", res_type_name); + __Pyx_DECREF_TypeName(res_type_name); Py_CLEAR(res); } else { int is_coroutine = 0; @@ -233,9 +227,12 @@ static PyObject *__Pyx__Coroutine_GetAwaitableIter(PyObject *obj) { } return res; slot_error: - PyErr_Format(PyExc_TypeError, - "object %.100s can't be used in 'await' expression", - Py_TYPE(obj)->tp_name); + { + __Pyx_TypeName obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "object " __Pyx_FMT_TYPENAME " can't be used in 'await' expression", obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + } bad: return NULL; } @@ -251,6 +248,7 @@ static CYTHON_INLINE PyObject *__Pyx_Coroutine_AsyncIterNext(PyObject *o); /*pro //@requires: ObjectHandling.c::PyObjectCallMethod0 static PyObject *__Pyx_Coroutine_GetAsyncIter_Generic(PyObject *obj) { + __Pyx_TypeName obj_type_name; #if PY_VERSION_HEX < 0x030500B1 { PyObject *iter = __Pyx_PyObject_CallMethod0(obj, PYIDENT("__aiter__")); @@ -262,11 +260,13 @@ static PyObject *__Pyx_Coroutine_GetAsyncIter_Generic(PyObject *obj) { } #else // avoid C warning about 'unused function' - if ((0)) (void) __Pyx_PyObject_CallMethod0(obj, PYIDENT("__aiter__")); + (void)&__Pyx_PyObject_CallMethod0; #endif - PyErr_Format(PyExc_TypeError, "'async for' requires an object with __aiter__ method, got %.100s", - Py_TYPE(obj)->tp_name); + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'async for' requires an object with __aiter__ method, got " __Pyx_FMT_TYPENAME, obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); return NULL; } @@ -299,8 +299,12 @@ static PyObject *__Pyx__Coroutine_AsyncIterNext(PyObject *obj) { // FIXME: for the sake of a nicely conforming exception message, assume any AttributeError meant '__anext__' if (PyErr_ExceptionMatches(PyExc_AttributeError)) #endif - PyErr_Format(PyExc_TypeError, "'async for' requires an object with __anext__ method, got %.100s", - Py_TYPE(obj)->tp_name); + { + __Pyx_TypeName obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'async for' requires an object with __anext__ method, got " __Pyx_FMT_TYPENAME, obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + } return NULL; } @@ -330,12 +334,13 @@ static void __Pyx_Generator_Replace_StopIteration(int in_async_gen); /*proto*/ //////////////////// pep479 //////////////////// //@requires: Exceptions.c::GetException -static void __Pyx_Generator_Replace_StopIteration(CYTHON_UNUSED int in_async_gen) { +static void __Pyx_Generator_Replace_StopIteration(int in_async_gen) { PyObject *exc, *val, *tb, *cur_exc; __Pyx_PyThreadState_declare #ifdef __Pyx_StopAsyncIteration_USED int is_async_stopiteration = 0; #endif + CYTHON_MAYBE_UNUSED_VAR(in_async_gen); cur_exc = PyErr_Occurred(); if (likely(!__Pyx_PyErr_GivenExceptionMatches(cur_exc, PyExc_StopIteration))) { @@ -366,7 +371,8 @@ static void __Pyx_Generator_Replace_StopIteration(CYTHON_UNUSED int in_async_gen //////////////////// CoroutineBase.proto //////////////////// //@substitute: naming -typedef PyObject *(*__pyx_coroutine_body_t)(PyObject *, PyThreadState *, PyObject *); +struct __pyx_CoroutineObject; +typedef PyObject *(*__pyx_coroutine_body_t)(struct __pyx_CoroutineObject *, PyThreadState *, PyObject *); #if CYTHON_USE_EXC_INFO_STACK // See https://bugs.python.org/issue25612 @@ -380,7 +386,7 @@ typedef struct { } __Pyx_ExcInfoStruct; #endif -typedef struct { +typedef struct __pyx_CoroutineObject { PyObject_HEAD __pyx_coroutine_body_t body; PyObject *closure; @@ -441,17 +447,15 @@ static CYTHON_INLINE void __Pyx_Coroutine_ResetFrameBackpointer(__Pyx_ExcInfoStr //////////////////// Coroutine.proto //////////////////// #define __Pyx_Coroutine_USED -static PyTypeObject *__pyx_CoroutineType = 0; -static PyTypeObject *__pyx_CoroutineAwaitType = 0; -#define __Pyx_Coroutine_CheckExact(obj) (Py_TYPE(obj) == __pyx_CoroutineType) +#define __Pyx_Coroutine_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_CoroutineType) // __Pyx_Coroutine_Check(obj): see override for IterableCoroutine below #define __Pyx_Coroutine_Check(obj) __Pyx_Coroutine_CheckExact(obj) -#define __Pyx_CoroutineAwait_CheckExact(obj) (Py_TYPE(obj) == __pyx_CoroutineAwaitType) +#define __Pyx_CoroutineAwait_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_CoroutineAwaitType) #define __Pyx_Coroutine_New(body, code, closure, name, qualname, module_name) \ __Pyx__Coroutine_New(__pyx_CoroutineType, body, code, closure, name, qualname, module_name) -static int __pyx_Coroutine_init(void); /*proto*/ +static int __pyx_Coroutine_init(PyObject *module); /*proto*/ static PyObject *__Pyx__Coroutine_await(PyObject *coroutine); /*proto*/ typedef struct { @@ -466,14 +470,13 @@ static PyObject *__Pyx_CoroutineAwait_Throw(__pyx_CoroutineAwaitObject *self, Py //////////////////// Generator.proto //////////////////// #define __Pyx_Generator_USED -static PyTypeObject *__pyx_GeneratorType = 0; -#define __Pyx_Generator_CheckExact(obj) (Py_TYPE(obj) == __pyx_GeneratorType) +#define __Pyx_Generator_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_GeneratorType) #define __Pyx_Generator_New(body, code, closure, name, qualname, module_name) \ __Pyx__Coroutine_New(__pyx_GeneratorType, body, code, closure, name, qualname, module_name) static PyObject *__Pyx_Generator_Next(PyObject *self); -static int __pyx_Generator_init(void); /*proto*/ +static int __pyx_Generator_init(PyObject *module); /*proto*/ //////////////////// AsyncGen //////////////////// @@ -489,10 +492,13 @@ static int __pyx_Generator_init(void); /*proto*/ //@requires: Exceptions.c::RaiseException //@requires: Exceptions.c::SaveResetException //@requires: ObjectHandling.c::PyObjectCallMethod1 +//@requires: ObjectHandling.c::PyObjectCallNoArg +//@requires: ObjectHandling.c::PyObjectFastCall //@requires: ObjectHandling.c::PyObjectGetAttrStr +//@requires: ObjectHandling.c::PyObjectGetAttrStrNoError //@requires: CommonStructures.c::FetchCommonType +//@requires: ModuleSetupCode.c::IncludeStructmemberH -#include <structmember.h> #include <frameobject.h> #if PY_VERSION_HEX >= 0x030b00a6 #ifndef Py_BUILD_CORE @@ -509,9 +515,10 @@ static int __pyx_Generator_init(void); /*proto*/ // Returns 0 if no exception or StopIteration is set. // If any other exception is set, returns -1 and leaves // pvalue unchanged. -static int __Pyx_PyGen__FetchStopIterationValue(CYTHON_UNUSED PyThreadState *$local_tstate_cname, PyObject **pvalue) { +static int __Pyx_PyGen__FetchStopIterationValue(PyThreadState *$local_tstate_cname, PyObject **pvalue) { PyObject *et, *ev, *tb; PyObject *value = NULL; + CYTHON_UNUSED_VAR($local_tstate_cname); __Pyx_ErrFetch(&et, &ev, &tb); @@ -530,7 +537,7 @@ static int __Pyx_PyGen__FetchStopIterationValue(CYTHON_UNUSED PyThreadState *$lo value = Py_None; } #if PY_VERSION_HEX >= 0x030300A0 - else if (Py_TYPE(ev) == (PyTypeObject*)PyExc_StopIteration) { + else if (likely(__Pyx_IS_TYPE(ev, (PyTypeObject*)PyExc_StopIteration))) { value = ((PyStopIterationObject *)ev)->value; Py_INCREF(value); Py_DECREF(ev); @@ -601,6 +608,9 @@ static int __Pyx_PyGen__FetchStopIterationValue(CYTHON_UNUSED PyThreadState *$lo static CYTHON_INLINE void __Pyx_Coroutine_ExceptionClear(__Pyx_ExcInfoStruct *exc_state) { +#if PY_VERSION_HEX >= 0x030B00a4 + Py_CLEAR(exc_state->exc_value); +#else PyObject *t, *v, *tb; t = exc_state->exc_type; v = exc_state->exc_value; @@ -613,11 +623,13 @@ void __Pyx_Coroutine_ExceptionClear(__Pyx_ExcInfoStruct *exc_state) { Py_XDECREF(t); Py_XDECREF(v); Py_XDECREF(tb); +#endif } #define __Pyx_Coroutine_AlreadyRunningError(gen) (__Pyx__Coroutine_AlreadyRunningError(gen), (PyObject*)NULL) -static void __Pyx__Coroutine_AlreadyRunningError(CYTHON_UNUSED __pyx_CoroutineObject *gen) { +static void __Pyx__Coroutine_AlreadyRunningError(__pyx_CoroutineObject *gen) { const char *msg; + CYTHON_MAYBE_UNUSED_VAR(gen); if ((0)) { #ifdef __Pyx_Coroutine_USED } else if (__Pyx_Coroutine_Check((PyObject*)gen)) { @@ -634,8 +646,9 @@ static void __Pyx__Coroutine_AlreadyRunningError(CYTHON_UNUSED __pyx_CoroutineOb } #define __Pyx_Coroutine_NotStartedError(gen) (__Pyx__Coroutine_NotStartedError(gen), (PyObject*)NULL) -static void __Pyx__Coroutine_NotStartedError(CYTHON_UNUSED PyObject *gen) { +static void __Pyx__Coroutine_NotStartedError(PyObject *gen) { const char *msg; + CYTHON_MAYBE_UNUSED_VAR(gen); if ((0)) { #ifdef __Pyx_Coroutine_USED } else if (__Pyx_Coroutine_Check(gen)) { @@ -652,7 +665,9 @@ static void __Pyx__Coroutine_NotStartedError(CYTHON_UNUSED PyObject *gen) { } #define __Pyx_Coroutine_AlreadyTerminatedError(gen, value, closing) (__Pyx__Coroutine_AlreadyTerminatedError(gen, value, closing), (PyObject*)NULL) -static void __Pyx__Coroutine_AlreadyTerminatedError(CYTHON_UNUSED PyObject *gen, PyObject *value, CYTHON_UNUSED int closing) { +static void __Pyx__Coroutine_AlreadyTerminatedError(PyObject *gen, PyObject *value, int closing) { + CYTHON_MAYBE_UNUSED_VAR(gen); + CYTHON_MAYBE_UNUSED_VAR(closing); #ifdef __Pyx_Coroutine_USED if (!closing && __Pyx_Coroutine_Check(gen)) { // `self` is an exhausted coroutine: raise an error, @@ -714,14 +729,21 @@ PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value, i // - do not touch external frames and tracebacks exc_state = &self->gi_exc_state; - if (exc_state->exc_type) { - #if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON + if (exc_state->exc_value) { + #if CYTHON_COMPILING_IN_PYPY // FIXME: what to do in PyPy? #else // Generators always return to their most recent caller, not // necessarily their creator. - if (exc_state->exc_traceback) { - PyTracebackObject *tb = (PyTracebackObject *) exc_state->exc_traceback; + PyObject *exc_tb; + #if PY_VERSION_HEX >= 0x030B00a4 + // owned reference! + exc_tb = PyException_GetTraceback(exc_state->exc_value); + #else + exc_tb = exc_state->exc_traceback; + #endif + if (exc_tb) { + PyTracebackObject *tb = (PyTracebackObject *) exc_tb; PyFrameObject *f = tb->tb_frame; assert(f->f_back == NULL); @@ -733,6 +755,9 @@ PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value, i Py_XINCREF(tstate->frame); f->f_back = tstate->frame; #endif + #if PY_VERSION_HEX >= 0x030B00a4 + Py_DECREF(exc_tb); + #endif } #endif } @@ -755,7 +780,7 @@ PyObject *__Pyx_Coroutine_SendEx(__pyx_CoroutineObject *self, PyObject *value, i #endif self->is_running = 1; - retval = self->body((PyObject *) self, tstate, value); + retval = self->body(self, tstate, value); self->is_running = 0; #if CYTHON_USE_EXC_INFO_STACK @@ -774,21 +799,34 @@ static CYTHON_INLINE void __Pyx_Coroutine_ResetFrameBackpointer(__Pyx_ExcInfoStr // Don't keep the reference to f_back any longer than necessary. It // may keep a chain of frames alive or it could create a reference // cycle. - PyObject *exc_tb = exc_state->exc_traceback; - - if (likely(exc_tb)) { -#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON +#if CYTHON_COMPILING_IN_PYPY // FIXME: what to do in PyPy? + CYTHON_UNUSED_VAR(exc_state); #else + PyObject *exc_tb; + + #if PY_VERSION_HEX >= 0x030B00a4 + if (!exc_state->exc_value) return; + // owned reference! + exc_tb = PyException_GetTraceback(exc_state->exc_value); + #else + exc_tb = exc_state->exc_traceback; + #endif + + if (likely(exc_tb)) { PyTracebackObject *tb = (PyTracebackObject *) exc_tb; PyFrameObject *f = tb->tb_frame; Py_CLEAR(f->f_back); -#endif + #if PY_VERSION_HEX >= 0x030B00a4 + Py_DECREF(exc_tb); + #endif } +#endif } static CYTHON_INLINE -PyObject *__Pyx_Coroutine_MethodReturn(CYTHON_UNUSED PyObject* gen, PyObject *retval) { +PyObject *__Pyx_Coroutine_MethodReturn(PyObject* gen, PyObject *retval) { + CYTHON_MAYBE_UNUSED_VAR(gen); if (unlikely(!retval)) { __Pyx_PyThreadState_declare __Pyx_PyThreadState_assign @@ -883,7 +921,7 @@ static PyObject *__Pyx_Coroutine_Send(PyObject *self, PyObject *value) { #endif { if (value == Py_None) - ret = Py_TYPE(yf)->tp_iternext(yf); + ret = __Pyx_PyObject_GetIterNextFunc(yf)(yf); else ret = __Pyx_PyObject_CallMethod1(yf, PYIDENT("send"), value); } @@ -937,16 +975,15 @@ static int __Pyx_Coroutine_CloseIter(__pyx_CoroutineObject *gen, PyObject *yf) { { PyObject *meth; gen->is_running = 1; - meth = __Pyx_PyObject_GetAttrStr(yf, PYIDENT("close")); + meth = __Pyx_PyObject_GetAttrStrNoError(yf, PYIDENT("close")); if (unlikely(!meth)) { - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + if (unlikely(PyErr_Occurred())) { PyErr_WriteUnraisable(yf); } - PyErr_Clear(); } else { - retval = PyObject_CallFunction(meth, NULL); + retval = __Pyx_PyObject_CallNoArg(meth); Py_DECREF(meth); - if (!retval) + if (unlikely(!retval)) err = -1; } gen->is_running = 0; @@ -982,7 +1019,7 @@ static PyObject *__Pyx_Generator_Next(PyObject *self) { ret = __Pyx_Coroutine_Send(yf, Py_None); } else #endif - ret = Py_TYPE(yf)->tp_iternext(yf); + ret = __Pyx_PyObject_GetIterNextFunc(yf)(yf); gen->is_running = 0; //Py_DECREF(yf); if (likely(ret)) { @@ -993,7 +1030,8 @@ static PyObject *__Pyx_Generator_Next(PyObject *self) { return __Pyx_Coroutine_SendEx(gen, Py_None, 0); } -static PyObject *__Pyx_Coroutine_Close_Method(PyObject *self, CYTHON_UNUSED PyObject *arg) { +static PyObject *__Pyx_Coroutine_Close_Method(PyObject *self, PyObject *arg) { + CYTHON_UNUSED_VAR(arg); return __Pyx_Coroutine_Close(self); } @@ -1084,23 +1122,23 @@ static PyObject *__Pyx__Coroutine_Throw(PyObject *self, PyObject *typ, PyObject ret = __Pyx__Coroutine_Throw(((__pyx_CoroutineAwaitObject*)yf)->coroutine, typ, val, tb, args, close_on_genexit); #endif } else { - PyObject *meth = __Pyx_PyObject_GetAttrStr(yf, PYIDENT("throw")); + PyObject *meth = __Pyx_PyObject_GetAttrStrNoError(yf, PYIDENT("throw")); if (unlikely(!meth)) { Py_DECREF(yf); - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { + if (unlikely(PyErr_Occurred())) { gen->is_running = 0; return NULL; } - PyErr_Clear(); __Pyx_Coroutine_Undelegate(gen); gen->is_running = 0; goto throw_here; } if (likely(args)) { - ret = PyObject_CallObject(meth, args); + ret = __Pyx_PyObject_Call(meth, args, NULL); } else { // "tb" or even "val" might be NULL, but that also correctly terminates the argument list - ret = PyObject_CallFunctionObjArgs(meth, typ, val, tb, NULL); + PyObject *cargs[4] = {NULL, typ, val, tb}; + ret = __Pyx_PyObject_FastCall(meth, cargs+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); } Py_DECREF(meth); } @@ -1121,16 +1159,20 @@ static PyObject *__Pyx_Coroutine_Throw(PyObject *self, PyObject *args) { PyObject *val = NULL; PyObject *tb = NULL; - if (!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb)) + if (unlikely(!PyArg_UnpackTuple(args, (char *)"throw", 1, 3, &typ, &val, &tb))) return NULL; return __Pyx__Coroutine_Throw(self, typ, val, tb, args, 1); } static CYTHON_INLINE int __Pyx_Coroutine_traverse_excstate(__Pyx_ExcInfoStruct *exc_state, visitproc visit, void *arg) { +#if PY_VERSION_HEX >= 0x030B00a4 + Py_VISIT(exc_state->exc_value); +#else Py_VISIT(exc_state->exc_type); Py_VISIT(exc_state->exc_value); Py_VISIT(exc_state->exc_traceback); +#endif return 0; } @@ -1172,10 +1214,10 @@ static void __Pyx_Coroutine_dealloc(PyObject *self) { // Generator is paused or unstarted, so we need to close PyObject_GC_Track(self); #if PY_VERSION_HEX >= 0x030400a1 && CYTHON_USE_TP_FINALIZE - if (PyObject_CallFinalizerFromDealloc(self)) + if (unlikely(PyObject_CallFinalizerFromDealloc(self))) #else Py_TYPE(gen)->tp_del(self); - if (Py_REFCNT(self) > 0) + if (unlikely(Py_REFCNT(self) > 0)) #endif { // resurrected. :( @@ -1193,7 +1235,7 @@ static void __Pyx_Coroutine_dealloc(PyObject *self) { } #endif __Pyx_Coroutine_clear(self); - PyObject_GC_Del(gen); + __Pyx_PyHeapTypeObject_GC_Del(gen); } static void __Pyx_Coroutine_del(PyObject *self) { @@ -1291,7 +1333,7 @@ static void __Pyx_Coroutine_del(PyObject *self) { // Undo the temporary resurrection; can't use DECREF here, it would // cause a recursive call. assert(Py_REFCNT(self) > 0); - if (--self->ob_refcnt == 0) { + if (likely(--self->ob_refcnt == 0)) { // this is the normal path out return; } @@ -1324,9 +1366,10 @@ static void __Pyx_Coroutine_del(PyObject *self) { } static PyObject * -__Pyx_Coroutine_get_name(__pyx_CoroutineObject *self, CYTHON_UNUSED void *context) +__Pyx_Coroutine_get_name(__pyx_CoroutineObject *self, void *context) { PyObject *name = self->gi_name; + CYTHON_UNUSED_VAR(context); // avoid NULL pointer dereference during garbage collection if (unlikely(!name)) name = Py_None; Py_INCREF(name); @@ -1334,10 +1377,9 @@ __Pyx_Coroutine_get_name(__pyx_CoroutineObject *self, CYTHON_UNUSED void *contex } static int -__Pyx_Coroutine_set_name(__pyx_CoroutineObject *self, PyObject *value, CYTHON_UNUSED void *context) +__Pyx_Coroutine_set_name(__pyx_CoroutineObject *self, PyObject *value, void *context) { - PyObject *tmp; - + CYTHON_UNUSED_VAR(context); #if PY_MAJOR_VERSION >= 3 if (unlikely(value == NULL || !PyUnicode_Check(value))) #else @@ -1348,17 +1390,16 @@ __Pyx_Coroutine_set_name(__pyx_CoroutineObject *self, PyObject *value, CYTHON_UN "__name__ must be set to a string object"); return -1; } - tmp = self->gi_name; Py_INCREF(value); - self->gi_name = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(self->gi_name, value); return 0; } static PyObject * -__Pyx_Coroutine_get_qualname(__pyx_CoroutineObject *self, CYTHON_UNUSED void *context) +__Pyx_Coroutine_get_qualname(__pyx_CoroutineObject *self, void *context) { PyObject *name = self->gi_qualname; + CYTHON_UNUSED_VAR(context); // avoid NULL pointer dereference during garbage collection if (unlikely(!name)) name = Py_None; Py_INCREF(name); @@ -1366,10 +1407,9 @@ __Pyx_Coroutine_get_qualname(__pyx_CoroutineObject *self, CYTHON_UNUSED void *co } static int -__Pyx_Coroutine_set_qualname(__pyx_CoroutineObject *self, PyObject *value, CYTHON_UNUSED void *context) +__Pyx_Coroutine_set_qualname(__pyx_CoroutineObject *self, PyObject *value, void *context) { - PyObject *tmp; - + CYTHON_UNUSED_VAR(context); #if PY_MAJOR_VERSION >= 3 if (unlikely(value == NULL || !PyUnicode_Check(value))) #else @@ -1380,18 +1420,16 @@ __Pyx_Coroutine_set_qualname(__pyx_CoroutineObject *self, PyObject *value, CYTHO "__qualname__ must be set to a string object"); return -1; } - tmp = self->gi_qualname; Py_INCREF(value); - self->gi_qualname = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(self->gi_qualname, value); return 0; } - static PyObject * -__Pyx_Coroutine_get_frame(__pyx_CoroutineObject *self, CYTHON_UNUSED void *context) +__Pyx_Coroutine_get_frame(__pyx_CoroutineObject *self, void *context) { PyObject *frame = self->gi_frame; + CYTHON_UNUSED_VAR(context); if (!frame) { if (unlikely(!self->gi_code)) { // Avoid doing something stupid, e.g. during garbage collection. @@ -1431,9 +1469,13 @@ static __pyx_CoroutineObject *__Pyx__Coroutine_NewInit( gen->resume_label = 0; gen->classobj = NULL; gen->yieldfrom = NULL; + #if PY_VERSION_HEX >= 0x030B00a4 + gen->gi_exc_state.exc_value = NULL; + #else gen->gi_exc_state.exc_type = NULL; gen->gi_exc_state.exc_value = NULL; gen->gi_exc_state.exc_traceback = NULL; + #endif #if CYTHON_USE_EXC_INFO_STACK gen->gi_exc_state.previous_item = NULL; #endif @@ -1461,7 +1503,7 @@ static __pyx_CoroutineObject *__Pyx__Coroutine_NewInit( static void __Pyx_CoroutineAwait_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); Py_CLEAR(((__pyx_CoroutineAwaitObject*)self)->coroutine); - PyObject_GC_Del(self); + __Pyx_PyHeapTypeObject_GC_Del(self); } static int __Pyx_CoroutineAwait_traverse(__pyx_CoroutineAwaitObject *self, visitproc visit, void *arg) { @@ -1486,7 +1528,8 @@ static PyObject *__Pyx_CoroutineAwait_Throw(__pyx_CoroutineAwaitObject *self, Py return __Pyx_Coroutine_Throw(self->coroutine, args); } -static PyObject *__Pyx_CoroutineAwait_Close(__pyx_CoroutineAwaitObject *self, CYTHON_UNUSED PyObject *arg) { +static PyObject *__Pyx_CoroutineAwait_Close(__pyx_CoroutineAwaitObject *self, PyObject *arg) { + CYTHON_UNUSED_VAR(arg); return __Pyx_Coroutine_Close(self->coroutine); } @@ -1496,12 +1539,31 @@ static PyObject *__Pyx_CoroutineAwait_self(PyObject *self) { } #if !CYTHON_COMPILING_IN_PYPY -static PyObject *__Pyx_CoroutineAwait_no_new(CYTHON_UNUSED PyTypeObject *type, CYTHON_UNUSED PyObject *args, CYTHON_UNUSED PyObject *kwargs) { +static PyObject *__Pyx_CoroutineAwait_no_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { + CYTHON_UNUSED_VAR(type); + CYTHON_UNUSED_VAR(args); + CYTHON_UNUSED_VAR(kwargs); PyErr_SetString(PyExc_TypeError, "cannot instantiate type, use 'await coroutine' instead"); return NULL; } #endif +// In earlier versions of Python an object with no __dict__ and not __slots__ is assumed +// to be pickleable by default. Coroutine-wrappers have significant state so shouldn't be. +// Therefore provide a default implementation. +// Something similar applies to heaptypes (i.e. with type_specs) with protocols 0 and 1 +// even in more recent versions. +// We are applying this to all Python versions (hence the commented out version guard) +// to make the behaviour explicit. +// #if PY_VERSION_HEX < 0x03060000 || CYTHON_USE_TYPE_SPECS +static PyObject *__Pyx_CoroutineAwait_reduce_ex(__pyx_CoroutineAwaitObject *self, PyObject *arg) { + CYTHON_UNUSED_VAR(arg); + PyErr_Format(PyExc_TypeError, "cannot pickle '%.200s' object", + Py_TYPE(self)->tp_name); + return NULL; +} +// #endif + static PyMethodDef __pyx_CoroutineAwait_methods[] = { {"send", (PyCFunction) __Pyx_CoroutineAwait_Send, METH_O, (char*) PyDoc_STR("send(arg) -> send 'arg' into coroutine,\nreturn next yielded value or raise StopIteration.")}, @@ -1509,12 +1571,40 @@ static PyMethodDef __pyx_CoroutineAwait_methods[] = { (char*) PyDoc_STR("throw(typ[,val[,tb]]) -> raise exception in coroutine,\nreturn next yielded value or raise StopIteration.")}, {"close", (PyCFunction) __Pyx_CoroutineAwait_Close, METH_NOARGS, (char*) PyDoc_STR("close() -> raise GeneratorExit inside coroutine.")}, +// only needed with type-specs or version<3.6, but included in all versions for clarity +// #if PY_VERSION_HEX < 0x03060000 || CYTHON_USE_TYPE_SPECS + {"__reduce_ex__", (PyCFunction) __Pyx_CoroutineAwait_reduce_ex, METH_O, 0}, + {"__reduce__", (PyCFunction) __Pyx_CoroutineAwait_reduce_ex, METH_NOARGS, 0}, +// #endif {0, 0, 0, 0} }; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_CoroutineAwaitType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_CoroutineAwait_dealloc}, + {Py_tp_traverse, (void *)__Pyx_CoroutineAwait_traverse}, + {Py_tp_clear, (void *)__Pyx_CoroutineAwait_clear}, +#if !CYTHON_COMPILING_IN_PYPY + {Py_tp_new, (void *)__Pyx_CoroutineAwait_no_new}, +#endif + {Py_tp_methods, (void *)__pyx_CoroutineAwait_methods}, + {Py_tp_iter, (void *)__Pyx_CoroutineAwait_self}, + {Py_tp_iternext, (void *)__Pyx_CoroutineAwait_Next}, + {0, 0}, +}; + +static PyType_Spec __pyx_CoroutineAwaitType_spec = { + __PYX_TYPE_MODULE_PREFIX "coroutine_wrapper", + sizeof(__pyx_CoroutineAwaitObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ + __pyx_CoroutineAwaitType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + static PyTypeObject __pyx_CoroutineAwaitType_type = { PyVarObject_HEAD_INIT(0, 0) - "coroutine_wrapper", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "coroutine_wrapper", /*tp_name*/ sizeof(__pyx_CoroutineAwaitObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __Pyx_CoroutineAwait_dealloc,/*tp_dealloc*/ @@ -1570,7 +1660,7 @@ static PyTypeObject __pyx_CoroutineAwaitType_type = { #if PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800) 0, /*tp_vectorcall*/ #endif -#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 +#if __PYX_NEED_TP_PRINT_SLOT 0, /*tp_print*/ #endif #if PY_VERSION_HEX >= 0x030C0000 @@ -1580,6 +1670,7 @@ static PyTypeObject __pyx_CoroutineAwaitType_type = { 0, /*tp_pypy_flags*/ #endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ #if PY_VERSION_HEX < 0x030500B1 || defined(__Pyx_IterableCoroutine_USED) || CYTHON_USE_ASYNC_SLOTS static CYTHON_INLINE PyObject *__Pyx__Coroutine_await(PyObject *coroutine) { @@ -1593,7 +1684,8 @@ static CYTHON_INLINE PyObject *__Pyx__Coroutine_await(PyObject *coroutine) { #endif #if PY_VERSION_HEX < 0x030500B1 -static PyObject *__Pyx_Coroutine_await_method(PyObject *coroutine, CYTHON_UNUSED PyObject *arg) { +static PyObject *__Pyx_Coroutine_await_method(PyObject *coroutine, PyObject *arg) { + CYTHON_UNUSED_VAR(arg); return __Pyx__Coroutine_await(coroutine); } #endif @@ -1641,7 +1733,10 @@ static PyMemberDef __pyx_Coroutine_memberlist[] = { {(char*) "cr_await", T_OBJECT, offsetof(__pyx_CoroutineObject, yieldfrom), READONLY, (char*) PyDoc_STR("object being awaited, or None")}, {(char*) "cr_code", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_code), READONLY, NULL}, - {(char *) "__module__", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_modulename), PY_WRITE_RESTRICTED, 0}, + {(char *) "__module__", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_modulename), 0, 0}, +#if CYTHON_USE_TYPE_SPECS + {(char *) "__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CoroutineObject, gi_weakreflist), READONLY, 0}, +#endif {0, 0, 0, 0, 0} }; @@ -1655,6 +1750,30 @@ static PyGetSetDef __pyx_Coroutine_getsets[] = { {0, 0, 0, 0, 0} }; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_CoroutineType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_Coroutine_dealloc}, + {Py_am_await, (void *)&__Pyx_Coroutine_await}, + {Py_tp_traverse, (void *)__Pyx_Coroutine_traverse}, + {Py_tp_methods, (void *)__pyx_Coroutine_methods}, + {Py_tp_members, (void *)__pyx_Coroutine_memberlist}, + {Py_tp_getset, (void *)__pyx_Coroutine_getsets}, + {Py_tp_getattro, (void *) __Pyx_PyObject_GenericGetAttrNoDict}, +#if CYTHON_USE_TP_FINALIZE + {Py_tp_finalize, (void *)__Pyx_Coroutine_del}, +#endif + {0, 0}, +}; + +static PyType_Spec __pyx_CoroutineType_spec = { + __PYX_TYPE_MODULE_PREFIX "coroutine", + sizeof(__pyx_CoroutineObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/ + __pyx_CoroutineType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + #if CYTHON_USE_ASYNC_SLOTS static __Pyx_PyAsyncMethodsStruct __pyx_Coroutine_as_async = { __Pyx_Coroutine_await, /*am_await*/ @@ -1668,7 +1787,7 @@ static __Pyx_PyAsyncMethodsStruct __pyx_Coroutine_as_async = { static PyTypeObject __pyx_CoroutineType_type = { PyVarObject_HEAD_INIT(0, 0) - "coroutine", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "coroutine", /*tp_name*/ sizeof(__pyx_CoroutineObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __Pyx_Coroutine_dealloc,/*tp_dealloc*/ @@ -1736,7 +1855,7 @@ static PyTypeObject __pyx_CoroutineType_type = { #if PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800) 0, /*tp_vectorcall*/ #endif -#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 +#if __PYX_NEED_TP_PRINT_SLOT 0, /*tp_print*/ #endif #if PY_VERSION_HEX >= 0x030C0000 @@ -1746,20 +1865,30 @@ static PyTypeObject __pyx_CoroutineType_type = { 0, /*tp_pypy_flags*/ #endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ -static int __pyx_Coroutine_init(void) { +static int __pyx_Coroutine_init(PyObject *module) { + CYTHON_MAYBE_UNUSED_VAR(module); // on Windows, C-API functions can't be used in slots statically +#if CYTHON_USE_TYPE_SPECS + __pyx_CoroutineType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx_CoroutineType_spec, NULL); +#else __pyx_CoroutineType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; __pyx_CoroutineType = __Pyx_FetchCommonType(&__pyx_CoroutineType_type); +#endif if (unlikely(!__pyx_CoroutineType)) return -1; #ifdef __Pyx_IterableCoroutine_USED - if (unlikely(__pyx_IterableCoroutine_init() == -1)) + if (unlikely(__pyx_IterableCoroutine_init(module) == -1)) return -1; #endif +#if CYTHON_USE_TYPE_SPECS + __pyx_CoroutineAwaitType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx_CoroutineAwaitType_spec, NULL); +#else __pyx_CoroutineAwaitType = __Pyx_FetchCommonType(&__pyx_CoroutineAwaitType_type); +#endif if (unlikely(!__pyx_CoroutineAwaitType)) return -1; return 0; @@ -1770,24 +1899,48 @@ static int __pyx_Coroutine_init(void) { #define __Pyx_IterableCoroutine_USED -static PyTypeObject *__pyx_IterableCoroutineType = 0; - #undef __Pyx_Coroutine_Check -#define __Pyx_Coroutine_Check(obj) (__Pyx_Coroutine_CheckExact(obj) || (Py_TYPE(obj) == __pyx_IterableCoroutineType)) +#define __Pyx_Coroutine_Check(obj) (__Pyx_Coroutine_CheckExact(obj) || __Pyx_IS_TYPE(obj, __pyx_IterableCoroutineType)) #define __Pyx_IterableCoroutine_New(body, code, closure, name, qualname, module_name) \ __Pyx__Coroutine_New(__pyx_IterableCoroutineType, body, code, closure, name, qualname, module_name) -static int __pyx_IterableCoroutine_init(void);/*proto*/ +static int __pyx_IterableCoroutine_init(PyObject *module);/*proto*/ //////////////////// IterableCoroutine //////////////////// //@requires: Coroutine //@requires: CommonStructures.c::FetchCommonType +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_IterableCoroutineType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_Coroutine_dealloc}, + {Py_am_await, (void *)&__Pyx_Coroutine_await}, + {Py_tp_traverse, (void *)__Pyx_Coroutine_traverse}, + {Py_tp_iter, (void *)__Pyx_Coroutine_await}, + {Py_tp_iternext, (void *)__Pyx_Generator_Next}, + {Py_tp_methods, (void *)__pyx_Coroutine_methods}, + {Py_tp_members, (void *)__pyx_Coroutine_memberlist}, + {Py_tp_getset, (void *)__pyx_Coroutine_getsets}, + {Py_tp_getattro, (void *) __Pyx_PyObject_GenericGetAttrNoDict}, +#if CYTHON_USE_TP_FINALIZE + {Py_tp_finalize, (void *)__Pyx_Coroutine_del}, +#endif + {0, 0}, +}; + +static PyType_Spec __pyx_IterableCoroutineType_spec = { + __PYX_TYPE_MODULE_PREFIX "iterable_coroutine", + sizeof(__pyx_CoroutineObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/ + __pyx_IterableCoroutineType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + static PyTypeObject __pyx_IterableCoroutineType_type = { PyVarObject_HEAD_INIT(0, 0) - "iterable_coroutine", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "iterable_coroutine", /*tp_name*/ sizeof(__pyx_CoroutineObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __Pyx_Coroutine_dealloc,/*tp_dealloc*/ @@ -1847,13 +2000,13 @@ static PyTypeObject __pyx_IterableCoroutineType_type = { __Pyx_Coroutine_del, /*tp_del*/ #endif 0, /*tp_version_tag*/ -#if PY_VERSION_HEX >= 0x030400a1 +#if PY_VERSION_HEX >= 0x030400a1 && !CYTHON_COMPILING_IN_PYPY __Pyx_Coroutine_del, /*tp_finalize*/ #endif #if PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800) 0, /*tp_vectorcall*/ #endif -#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 +#if __PYX_NEED_TP_PRINT_SLOT 0, /*tp_print*/ #endif #if PY_VERSION_HEX >= 0x030C0000 @@ -1863,11 +2016,17 @@ static PyTypeObject __pyx_IterableCoroutineType_type = { 0, /*tp_pypy_flags*/ #endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ -static int __pyx_IterableCoroutine_init(void) { +static int __pyx_IterableCoroutine_init(PyObject *module) { +#if CYTHON_USE_TYPE_SPECS + __pyx_IterableCoroutineType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx_IterableCoroutineType_spec, NULL); +#else + CYTHON_UNUSED_VAR(module); __pyx_IterableCoroutineType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; __pyx_IterableCoroutineType = __Pyx_FetchCommonType(&__pyx_IterableCoroutineType_type); +#endif if (unlikely(!__pyx_IterableCoroutineType)) return -1; return 0; @@ -1894,6 +2053,10 @@ static PyMemberDef __pyx_Generator_memberlist[] = { {(char*) "gi_yieldfrom", T_OBJECT, offsetof(__pyx_CoroutineObject, yieldfrom), READONLY, (char*) PyDoc_STR("object being iterated by 'yield from', or None")}, {(char*) "gi_code", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_code), READONLY, NULL}, + {(char *) "__module__", T_OBJECT, offsetof(__pyx_CoroutineObject, gi_modulename), 0, 0}, +#if CYTHON_USE_TYPE_SPECS + {(char *) "__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CoroutineObject, gi_weakreflist), READONLY, 0}, +#endif {0, 0, 0, 0, 0} }; @@ -1907,16 +2070,41 @@ static PyGetSetDef __pyx_Generator_getsets[] = { {0, 0, 0, 0, 0} }; +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_GeneratorType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_Coroutine_dealloc}, + {Py_tp_traverse, (void *)__Pyx_Coroutine_traverse}, + {Py_tp_iter, (void *)PyObject_SelfIter}, + {Py_tp_iternext, (void *)__Pyx_Generator_Next}, + {Py_tp_methods, (void *)__pyx_Generator_methods}, + {Py_tp_members, (void *)__pyx_Generator_memberlist}, + {Py_tp_getset, (void *)__pyx_Generator_getsets}, + {Py_tp_getattro, (void *) __Pyx_PyObject_GenericGetAttrNoDict}, +#if CYTHON_USE_TP_FINALIZE + {Py_tp_finalize, (void *)__Pyx_Coroutine_del}, +#endif + {0, 0}, +}; + +static PyType_Spec __pyx_GeneratorType_spec = { + __PYX_TYPE_MODULE_PREFIX "generator", + sizeof(__pyx_CoroutineObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_HAVE_FINALIZE, /*tp_flags*/ + __pyx_GeneratorType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + static PyTypeObject __pyx_GeneratorType_type = { PyVarObject_HEAD_INIT(0, 0) - "generator", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "generator", /*tp_name*/ sizeof(__pyx_CoroutineObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __Pyx_Coroutine_dealloc,/*tp_dealloc*/ 0, /*tp_print*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ - 0, /*tp_compare / tp_as_async*/ + 0, /*tp_as_async*/ 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ @@ -1967,7 +2155,7 @@ static PyTypeObject __pyx_GeneratorType_type = { #if PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800) 0, /*tp_vectorcall*/ #endif -#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 +#if __PYX_NEED_TP_PRINT_SLOT 0, /*tp_print*/ #endif #if PY_VERSION_HEX >= 0x030C0000 @@ -1977,13 +2165,18 @@ static PyTypeObject __pyx_GeneratorType_type = { 0, /*tp_pypy_flags*/ #endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ -static int __pyx_Generator_init(void) { +static int __pyx_Generator_init(PyObject *module) { +#if CYTHON_USE_TYPE_SPECS + __pyx_GeneratorType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx_GeneratorType_spec, NULL); +#else + CYTHON_UNUSED_VAR(module); // on Windows, C-API functions can't be used in slots statically __pyx_GeneratorType_type.tp_getattro = __Pyx_PyObject_GenericGetAttrNoDict; __pyx_GeneratorType_type.tp_iter = PyObject_SelfIter; - __pyx_GeneratorType = __Pyx_FetchCommonType(&__pyx_GeneratorType_type); +#endif if (unlikely(!__pyx_GeneratorType)) { return -1; } @@ -2010,7 +2203,7 @@ static void __Pyx__ReturnWithStopIteration(PyObject* value); /*proto*/ static void __Pyx__ReturnWithStopIteration(PyObject* value) { PyObject *exc, *args; -#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_PYSTON +#if CYTHON_COMPILING_IN_CPYTHON __Pyx_PyThreadState_declare if ((PY_VERSION_HEX >= 0x03030000 && PY_VERSION_HEX < 0x030500B1) || unlikely(PyTuple_Check(value) || PyExceptionInstance_Check(value))) { @@ -2029,7 +2222,7 @@ static void __Pyx__ReturnWithStopIteration(PyObject* value) { #if CYTHON_FAST_THREAD_STATE __Pyx_PyThreadState_assign #if CYTHON_USE_EXC_INFO_STACK - if (!$local_tstate_cname->exc_info->exc_type) + if (!$local_tstate_cname->exc_info->exc_value) #else if (!$local_tstate_cname->exc_type) #endif @@ -2139,7 +2332,7 @@ static int __Pyx_patch_abc(void) { if (CYTHON_REGISTER_ABCS && !abc_patched) { PyObject *module; module = PyImport_ImportModule((PY_MAJOR_VERSION >= 3) ? "collections.abc" : "collections"); - if (!module) { + if (unlikely(!module)) { PyErr_WriteUnraisable(NULL); if (unlikely(PyErr_WarnEx(PyExc_RuntimeWarning, ((PY_MAJOR_VERSION >= 3) ? @@ -2316,11 +2509,15 @@ old_types.add(_cython_generator_type) #define __Pyx_StopAsyncIteration_USED static PyObject *__Pyx_PyExc_StopAsyncIteration; -static int __pyx_StopAsyncIteration_init(void); /*proto*/ +static int __pyx_StopAsyncIteration_init(PyObject *module); /*proto*/ //////////////////// StopAsyncIteration //////////////////// #if PY_VERSION_HEX < 0x030500B1 +#if CYTHON_USE_TYPE_SPECS +#error Using async coroutines with type specs requires Python 3.5 or later. +#else + static PyTypeObject __Pyx__PyExc_StopAsyncIteration_type = { PyVarObject_HEAD_INIT(0, 0) "StopAsyncIteration", /*tp_name*/ @@ -2377,8 +2574,10 @@ static PyTypeObject __Pyx__PyExc_StopAsyncIteration_type = { #endif }; #endif +#endif -static int __pyx_StopAsyncIteration_init(void) { +static int __pyx_StopAsyncIteration_init(PyObject *module) { + CYTHON_UNUSED_VAR(module); #if PY_VERSION_HEX >= 0x030500B1 __Pyx_PyExc_StopAsyncIteration = PyExc_StopAsyncIteration; #else @@ -2400,7 +2599,7 @@ static int __pyx_StopAsyncIteration_init(void) { __Pyx_PyExc_StopAsyncIteration = (PyObject*) __Pyx_FetchCommonType(&__Pyx__PyExc_StopAsyncIteration_type); if (unlikely(!__Pyx_PyExc_StopAsyncIteration)) return -1; - if (builtins && unlikely(PyMapping_SetItemString(builtins, (char*) "StopAsyncIteration", __Pyx_PyExc_StopAsyncIteration) < 0)) + if (likely(builtins) && unlikely(PyMapping_SetItemString(builtins, (char*) "StopAsyncIteration", __Pyx_PyExc_StopAsyncIteration) < 0)) return -1; #endif return 0; diff --git a/Cython/Utility/CpdefEnums.pyx b/Cython/Utility/CpdefEnums.pyx index 148d776c2..39377b30c 100644 --- a/Cython/Utility/CpdefEnums.pyx +++ b/Cython/Utility/CpdefEnums.pyx @@ -6,10 +6,11 @@ cdef extern from *: int PY_VERSION_HEX cdef object __Pyx_OrderedDict -if PY_VERSION_HEX >= 0x02070000: - from collections import OrderedDict as __Pyx_OrderedDict -else: + +if PY_VERSION_HEX >= 0x03060000: __Pyx_OrderedDict = dict +else: + from collections import OrderedDict as __Pyx_OrderedDict @cython.internal cdef class __Pyx_EnumMeta(type): @@ -23,8 +24,7 @@ cdef class __Pyx_EnumMeta(type): # @cython.internal cdef object __Pyx_EnumBase -class __Pyx_EnumBase(int): - __metaclass__ = __Pyx_EnumMeta +class __Pyx_EnumBase(int, metaclass=__Pyx_EnumMeta): def __new__(cls, value, name=None): for v in cls: if v == value: @@ -44,23 +44,121 @@ class __Pyx_EnumBase(int): if PY_VERSION_HEX >= 0x03040000: from enum import IntEnum as __Pyx_EnumBase +cdef object __Pyx_FlagBase +class __Pyx_FlagBase(int, metaclass=__Pyx_EnumMeta): + def __new__(cls, value, name=None): + for v in cls: + if v == value: + return v + res = int.__new__(cls, value) + if name is None: + # some bitwise combination, no validation here + res.name = "" + else: + res.name = name + setattr(cls, name, res) + cls.__members__[name] = res + return res + def __repr__(self): + return "<%s.%s: %d>" % (self.__class__.__name__, self.name, self) + def __str__(self): + return "%s.%s" % (self.__class__.__name__, self.name) + +if PY_VERSION_HEX >= 0x03060000: + from enum import IntFlag as __Pyx_FlagBase + #################### EnumType #################### #@requires: EnumBase +cdef extern from *: + object {{enum_to_pyint_func}}({{name}} value) + cdef dict __Pyx_globals = globals() -if PY_VERSION_HEX >= 0x03040000: - # create new IntEnum() - {{name}} = __Pyx_EnumBase('{{name}}', __Pyx_OrderedDict([ +if PY_VERSION_HEX >= 0x03060000: + # create new IntFlag() - the assumption is that C enums are sufficiently commonly + # used as flags that this is the most appropriate base class + {{name}} = __Pyx_FlagBase('{{name}}', [ {{for item in items}} - ('{{item}}', {{item}}), + ('{{item}}', {{enum_to_pyint_func}}({{item}})), {{endfor}} - ])) + # Try to look up the module name dynamically if possible + ], module=__Pyx_globals.get("__module__", '{{static_modname}}')) + + if PY_VERSION_HEX >= 0x030B0000: + # Python 3.11 starts making the behaviour of flags stricter + # (only including powers of 2 when iterating). Since we're using + # "flag" because C enums *might* be used as flags, not because + # we want strict flag behaviour, manually undo some of this. + {{name}}._member_names_ = list({{name}}.__members__) + + {{if enum_doc is not None}} + {{name}}.__doc__ = {{ repr(enum_doc) }} + {{endif}} + {{for item in items}} __Pyx_globals['{{item}}'] = {{name}}.{{item}} {{endfor}} else: - class {{name}}(__Pyx_EnumBase): - pass + class {{name}}(__Pyx_FlagBase): + {{ repr(enum_doc) if enum_doc is not None else 'pass' }} + {{for item in items}} + __Pyx_globals['{{item}}'] = {{name}}({{enum_to_pyint_func}}({{item}}), '{{item}}') + {{endfor}} + +#################### CppScopedEnumType #################### +#@requires: EnumBase +cdef dict __Pyx_globals = globals() + +if PY_VERSION_HEX >= 0x03040000: + __Pyx_globals["{{name}}"] = __Pyx_EnumBase('{{name}}', [ + {{for item in items}} + ('{{item}}', <{{underlying_type}}>({{name}}.{{item}})), + {{endfor}} + ], module=__Pyx_globals.get("__module__", '{{static_modname}}')) +else: + __Pyx_globals["{{name}}"] = type('{{name}}', (__Pyx_EnumBase,), {}) {{for item in items}} - __Pyx_globals['{{item}}'] = {{name}}({{item}}, '{{item}}') + __Pyx_globals["{{name}}"](<{{underlying_type}}>({{name}}.{{item}}), '{{item}}') {{endfor}} + +{{if enum_doc is not None}} +__Pyx_globals["{{name}}"].__doc__ = {{ repr(enum_doc) }} +{{endif}} + + +#################### EnumTypeToPy #################### + +@cname("{{funcname}}") +cdef {{funcname}}({{name}} c_val): + cdef object __pyx_enum + # There's a complication here: the Python enum wrapping is only generated + # for enums defined in the same module that they're used in. Therefore, if + # the enum was cimported from a different module, we try to import it. + # If that fails we return an int equivalent as the next best option. +{{if module_name}} + try: + from {{module_name}} import {{name}} as __pyx_enum + except ImportError: + import warnings + warnings.warn( + f"enum class {{name}} not importable from {{module_name}}. " + "You are probably using a cpdef enum declared in a .pxd file that " + "does not have a .py or .pyx file.") + return <{{underlying_type}}>c_val +{{else}} + __pyx_enum = {{name}} +{{endif}} + # TODO - Cython only manages to optimize C enums to a switch currently + if 0: + pass +{{for item in items}} + elif c_val == {{name}}.{{item}}: + return __pyx_enum.{{item}} +{{endfor}} + else: + underlying_c_val = <{{underlying_type}}>c_val +{{if is_flag}} + return __pyx_enum(underlying_c_val) +{{else}} + raise ValueError(f"{underlying_c_val} is not a valid {{name}}") +{{endif}} diff --git a/Cython/Utility/CppConvert.pyx b/Cython/Utility/CppConvert.pyx index 03360e510..1c6239b2b 100644 --- a/Cython/Utility/CppConvert.pyx +++ b/Cython/Utility/CppConvert.pyx @@ -5,8 +5,8 @@ cdef extern from *: cdef cppclass string "{{type}}": - string() - string(char* c_str, size_t size) + string() except + + string(char* c_str, size_t size) except + cdef const char* __Pyx_PyObject_AsStringAndSize(object, Py_ssize_t*) except NULL @cname("{{cname}}") @@ -39,7 +39,7 @@ cdef inline object {{cname.replace("PyObject", py_type, 1)}}(const string& s): cdef extern from *: cdef cppclass vector "std::vector" [T]: - void push_back(T&) + void push_back(T&) except + @cname("{{cname}}") cdef vector[X] {{cname}}(object o) except *: @@ -52,20 +52,39 @@ cdef vector[X] {{cname}}(object o) except *: #################### vector.to_py #################### cdef extern from *: - cdef cppclass vector "const std::vector" [T]: + cdef cppclass vector "std::vector" [T]: size_t size() T& operator[](size_t) +cdef extern from "Python.h": + void Py_INCREF(object) + list PyList_New(Py_ssize_t size) + void PyList_SET_ITEM(object list, Py_ssize_t i, object o) + cdef Py_ssize_t PY_SSIZE_T_MAX + @cname("{{cname}}") -cdef object {{cname}}(vector[X]& v): - return [v[i] for i in range(v.size())] +cdef object {{cname}}(const vector[X]& v): + if v.size() > <size_t> PY_SSIZE_T_MAX: + raise MemoryError() + v_size_signed = <Py_ssize_t> v.size() + + o = PyList_New(v_size_signed) + + cdef Py_ssize_t i + cdef object item + for i in range(v_size_signed): + item = v[i] + Py_INCREF(item) + PyList_SET_ITEM(o, i, item) + + return o #################### list.from_py #################### cdef extern from *: cdef cppclass cpp_list "std::list" [T]: - void push_back(T&) + void push_back(T&) except + @cname("{{cname}}") cdef cpp_list[X] {{cname}}(object o) except *: @@ -87,14 +106,32 @@ cdef extern from *: bint operator!=(const_iterator) const_iterator begin() const_iterator end() + size_t size() + +cdef extern from "Python.h": + void Py_INCREF(object) + list PyList_New(Py_ssize_t size) + void PyList_SET_ITEM(object list, Py_ssize_t i, object o) + cdef Py_ssize_t PY_SSIZE_T_MAX @cname("{{cname}}") cdef object {{cname}}(const cpp_list[X]& v): - o = [] + if v.size() > <size_t> PY_SSIZE_T_MAX: + raise MemoryError() + + o = PyList_New(<Py_ssize_t> v.size()) + + cdef object item + cdef Py_ssize_t i = 0 cdef cpp_list[X].const_iterator iter = v.begin() + while iter != v.end(): - o.append(cython.operator.dereference(iter)) + item = cython.operator.dereference(iter) + Py_INCREF(item) + PyList_SET_ITEM(o, i, item) cython.operator.preincrement(iter) + i += 1 + return o @@ -102,7 +139,7 @@ cdef object {{cname}}(const cpp_list[X]& v): cdef extern from *: cdef cppclass set "std::{{maybe_unordered}}set" [T]: - void insert(T&) + void insert(T&) except + @cname("{{cname}}") cdef set[X] {{cname}}(object o) except *: @@ -127,19 +164,14 @@ cdef extern from *: @cname("{{cname}}") cdef object {{cname}}(const cpp_set[X]& s): - o = set() - cdef cpp_set[X].const_iterator iter = s.begin() - while iter != s.end(): - o.add(cython.operator.dereference(iter)) - cython.operator.preincrement(iter) - return o + return {v for v in s} #################### pair.from_py #################### cdef extern from *: cdef cppclass pair "std::pair" [T, U]: - pair() - pair(T&, U&) + pair() except + + pair(T&, U&) except + @cname("{{cname}}") cdef pair[X,Y] {{cname}}(object o) except *: @@ -163,19 +195,23 @@ cdef object {{cname}}(const pair[X,Y]& p): cdef extern from *: cdef cppclass pair "std::pair" [T, U]: - pair(T&, U&) + pair(T&, U&) except + cdef cppclass map "std::{{maybe_unordered}}map" [T, U]: - void insert(pair[T, U]&) + void insert(pair[T, U]&) except + cdef cppclass vector "std::vector" [T]: pass + int PY_MAJOR_VERSION @cname("{{cname}}") cdef map[X,Y] {{cname}}(object o) except *: - cdef dict d = o cdef map[X,Y] m - for key, value in d.iteritems(): - m.insert(pair[X,Y](<X>key, <Y>value)) + if PY_MAJOR_VERSION < 3: + for key, value in o.iteritems(): + m.insert(pair[X,Y](<X>key, <Y>value)) + else: + for key, value in o.items(): + m.insert(pair[X,Y](<X>key, <Y>value)) return m diff --git a/Cython/Utility/CppSupport.cpp b/Cython/Utility/CppSupport.cpp index b8fcff064..ba0002c94 100644 --- a/Cython/Utility/CppSupport.cpp +++ b/Cython/Utility/CppSupport.cpp @@ -56,3 +56,78 @@ auto __Pyx_pythran_to_python(T &&value) -> decltype(to_python( using returnable_type = typename pythonic::returnable<typename std::remove_cv<typename std::remove_reference<T>::type>::type>::type; return to_python(returnable_type{std::forward<T>(value)}); } + +#define __Pyx_PythranShapeAccessor(x) (pythonic::builtins::getattr(pythonic::types::attr::SHAPE{}, x)) + +////////////// MoveIfSupported.proto ////////////////// + +#if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600) + // move should be defined for these versions of MSVC, but __cplusplus isn't set usefully + #include <utility> + #define __PYX_STD_MOVE_IF_SUPPORTED(x) std::move(x) +#else + #define __PYX_STD_MOVE_IF_SUPPORTED(x) x +#endif + +////////////// EnumClassDecl.proto ////////////////// + +#if defined (_MSC_VER) + #if _MSC_VER >= 1910 + #define __PYX_ENUM_CLASS_DECL enum + #else + #define __PYX_ENUM_CLASS_DECL + #endif +#else + #define __PYX_ENUM_CLASS_DECL enum +#endif + +////////////// OptionalLocals.proto //////////////// +//@proto_block: utility_code_proto_before_types + +#include <utility> +#if defined(CYTHON_USE_BOOST_OPTIONAL) + // fallback mode - std::optional is preferred but this gives + // people with a less up-to-date compiler a chance + #include <boost/optional.hpp> + #define __Pyx_Optional_BaseType boost::optional +#else + #include <optional> + // since std::optional is a C++17 features, a templated using declaration should be safe + // (although it could be replaced with a define) + template <typename T> + using __Pyx_Optional_BaseType = std::optional<T>; +#endif + +// This class reuses as much of the implementation of std::optional as possible. +// The only place it differs significantly is the assignment operators, which use +// "emplace" (thus requiring move/copy constructors, but not move/copy +// assignment operators). This is preferred because it lets us work with assignable +// types (for example those with const members) +template <typename T> +class __Pyx_Optional_Type : private __Pyx_Optional_BaseType<T> { +public: + using __Pyx_Optional_BaseType<T>::__Pyx_Optional_BaseType; + using __Pyx_Optional_BaseType<T>::has_value; + using __Pyx_Optional_BaseType<T>::operator*; + using __Pyx_Optional_BaseType<T>::operator->; +#if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600) + __Pyx_Optional_Type& operator=(const __Pyx_Optional_Type& rhs) { + this->emplace(*rhs); + return *this; + } + __Pyx_Optional_Type& operator=(__Pyx_Optional_Type&& rhs) { + this->emplace(std::move(*rhs)); + return *this; + } + template <typename U=T> + __Pyx_Optional_Type& operator=(U&& rhs) { + this->emplace(std::forward<U>(rhs)); + return *this; + } +#else + // Note - the "cpp_locals" feature is designed to require C++14. + // This pre-c++11 fallback is largely untested, and definitely won't work + // in all the cases that the more modern version does + using __Pyx_Optional_BaseType<T>::operator=; // the chances are emplace can't work... +#endif +}; diff --git a/Cython/Utility/CythonFunction.c b/Cython/Utility/CythonFunction.c index 93f577f64..c2f85581a 100644 --- a/Cython/Utility/CythonFunction.c +++ b/Cython/Utility/CythonFunction.c @@ -1,16 +1,25 @@ //////////////////// CythonFunctionShared.proto //////////////////// -#define __Pyx_CyFunction_USED 1 +#define __Pyx_CyFunction_USED #define __Pyx_CYFUNCTION_STATICMETHOD 0x01 #define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 #define __Pyx_CYFUNCTION_CCLASS 0x04 +#define __Pyx_CYFUNCTION_COROUTINE 0x08 #define __Pyx_CyFunction_GetClosure(f) \ (((__pyx_CyFunctionObject *) (f))->func_closure) -#define __Pyx_CyFunction_GetClassObj(f) \ - (((__pyx_CyFunctionObject *) (f))->func_classobj) + +#if PY_VERSION_HEX < 0x030900B1 + #define __Pyx_CyFunction_GetClassObj(f) \ + (((__pyx_CyFunctionObject *) (f))->func_classobj) +#else + #define __Pyx_CyFunction_GetClassObj(f) \ + ((PyObject*) ((PyCMethodObject *) (f))->mm_class) +#endif +#define __Pyx_CyFunction_SetClassObj(f, classobj) \ + __Pyx__CyFunction_SetClassObj((__pyx_CyFunctionObject *) (f), (classobj)) #define __Pyx_CyFunction_Defaults(type, f) \ ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) @@ -19,7 +28,15 @@ typedef struct { +#if PY_VERSION_HEX < 0x030900B1 PyCFunctionObject func; +#else + // PEP-573: PyCFunctionObject + mm_class + PyCMethodObject func; +#endif +#if CYTHON_BACKPORT_VECTORCALL + __pyx_vectorcallfunc func_vectorcall; +#endif #if PY_VERSION_HEX < 0x030500A0 PyObject *func_weakreflist; #endif @@ -30,9 +47,10 @@ typedef struct { PyObject *func_globals; PyObject *func_code; PyObject *func_closure; +#if PY_VERSION_HEX < 0x030900B1 // No-args super() class cell PyObject *func_classobj; - +#endif // Dynamic default args and annotations void *defaults; int defaults_pyobjects; @@ -44,18 +62,22 @@ typedef struct { PyObject *defaults_kwdict; /* Const kwonly defaults dict */ PyObject *(*defaults_getter)(PyObject *); PyObject *func_annotations; /* function annotations dict */ -} __pyx_CyFunctionObject; -static PyTypeObject *__pyx_CyFunctionType = 0; + // Coroutine marker + PyObject *func_is_coroutine; +} __pyx_CyFunctionObject; -#define __Pyx_CyFunction_Check(obj) (__Pyx_TypeCheck(obj, __pyx_CyFunctionType)) +#define __Pyx_CyFunction_Check(obj) __Pyx_TypeCheck(obj, __pyx_CyFunctionType) +#define __Pyx_IsCyOrPyCFunction(obj) __Pyx_TypeCheck2(obj, __pyx_CyFunctionType, &PyCFunction_Type) +#define __Pyx_CyFunction_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_CyFunctionType) static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject* op, PyMethodDef *ml, int flags, PyObject* qualname, - PyObject *self, + PyObject *closure, PyObject *module, PyObject *globals, PyObject* code); +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj); static CYTHON_INLINE void *__Pyx_CyFunction_InitDefaults(PyObject *m, size_t size, int pyobjects); @@ -67,25 +89,51 @@ static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, PyObject *dict); -static int __pyx_CyFunction_init(void); +static int __pyx_CyFunction_init(PyObject *module); +#if CYTHON_METH_FASTCALL +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +#if CYTHON_BACKPORT_VECTORCALL +#define __Pyx_CyFunction_func_vectorcall(f) (((__pyx_CyFunctionObject*)f)->func_vectorcall) +#else +#define __Pyx_CyFunction_func_vectorcall(f) (((PyCFunctionObject*)f)->vectorcall) +#endif +#endif //////////////////// CythonFunctionShared //////////////////// //@substitute: naming //@requires: CommonStructures.c::FetchCommonType -////@requires: ObjectHandling.c::PyObjectGetAttrStr - -#include <structmember.h> +//@requires: ObjectHandling.c::PyMethodNew +//@requires: ObjectHandling.c::PyVectorcallFastCallDict +//@requires: ModuleSetupCode.c::IncludeStructmemberH +//@requires: ObjectHandling.c::PyObjectGetAttrStr + +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj) { +#if PY_VERSION_HEX < 0x030900B1 + __Pyx_Py_XDECREF_SET( + __Pyx_CyFunction_GetClassObj(f), + ((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#else + __Pyx_Py_XDECREF_SET( + // assigning to "mm_class", which is a "PyTypeObject*" + ((PyCMethodObject *) (f))->mm_class, + (PyTypeObject*)((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#endif +} static PyObject * -__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *closure) +__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, void *closure) { + CYTHON_UNUSED_VAR(closure); if (unlikely(op->func_doc == NULL)) { - if (op->func.m_ml->ml_doc) { + if (((PyCFunctionObject*)op)->m_ml->ml_doc) { #if PY_MAJOR_VERSION >= 3 - op->func_doc = PyUnicode_FromString(op->func.m_ml->ml_doc); + op->func_doc = PyUnicode_FromString(((PyCFunctionObject*)op)->m_ml->ml_doc); #else - op->func_doc = PyString_FromString(op->func.m_ml->ml_doc); + op->func_doc = PyString_FromString(((PyCFunctionObject*)op)->m_ml->ml_doc); #endif if (unlikely(op->func_doc == NULL)) return NULL; @@ -99,27 +147,27 @@ __Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *closure } static int -__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) +__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, void *context) { - PyObject *tmp = op->func_doc; + CYTHON_UNUSED_VAR(context); if (value == NULL) { // Mark as deleted value = Py_None; } Py_INCREF(value); - op->func_doc = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->func_doc, value); return 0; } static PyObject * -__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) +__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); if (unlikely(op->func_name == NULL)) { #if PY_MAJOR_VERSION >= 3 - op->func_name = PyUnicode_InternFromString(op->func.m_ml->ml_name); + op->func_name = PyUnicode_InternFromString(((PyCFunctionObject*)op)->m_ml->ml_name); #else - op->func_name = PyString_InternFromString(op->func.m_ml->ml_name); + op->func_name = PyString_InternFromString(((PyCFunctionObject*)op)->m_ml->ml_name); #endif if (unlikely(op->func_name == NULL)) return NULL; @@ -129,10 +177,9 @@ __Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *contex } static int -__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) +__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, void *context) { - PyObject *tmp; - + CYTHON_UNUSED_VAR(context); #if PY_MAJOR_VERSION >= 3 if (unlikely(value == NULL || !PyUnicode_Check(value))) #else @@ -143,25 +190,23 @@ __Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UN "__name__ must be set to a string object"); return -1; } - tmp = op->func_name; Py_INCREF(value); - op->func_name = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->func_name, value); return 0; } static PyObject * -__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) +__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); Py_INCREF(op->func_qualname); return op->func_qualname; } static int -__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) +__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, void *context) { - PyObject *tmp; - + CYTHON_UNUSED_VAR(context); #if PY_MAJOR_VERSION >= 3 if (unlikely(value == NULL || !PyUnicode_Check(value))) #else @@ -172,28 +217,15 @@ __Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, CYTHO "__qualname__ must be set to a string object"); return -1; } - tmp = op->func_qualname; Py_INCREF(value); - op->func_qualname = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->func_qualname, value); return 0; } static PyObject * -__Pyx_CyFunction_get_self(__pyx_CyFunctionObject *m, CYTHON_UNUSED void *closure) -{ - PyObject *self; - - self = m->func_closure; - if (self == NULL) - self = Py_None; - Py_INCREF(self); - return self; -} - -static PyObject * -__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) +__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); if (unlikely(op->func_dict == NULL)) { op->func_dict = PyDict_New(); if (unlikely(op->func_dict == NULL)) @@ -204,10 +236,9 @@ __Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *contex } static int -__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UNUSED void *context) +__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value, void *context) { - PyObject *tmp; - + CYTHON_UNUSED_VAR(context); if (unlikely(value == NULL)) { PyErr_SetString(PyExc_TypeError, "function's dictionary may not be deleted"); @@ -218,31 +249,33 @@ __Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value, CYTHON_UN "setting function's dictionary to a non-dict"); return -1; } - tmp = op->func_dict; Py_INCREF(value); - op->func_dict = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->func_dict, value); return 0; } static PyObject * -__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) +__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); Py_INCREF(op->func_globals); return op->func_globals; } static PyObject * -__Pyx_CyFunction_get_closure(CYTHON_UNUSED __pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) +__Pyx_CyFunction_get_closure(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(op); + CYTHON_UNUSED_VAR(context); Py_INCREF(Py_None); return Py_None; } static PyObject * -__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) +__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, void *context) { PyObject* result = (op->func_code) ? op->func_code : Py_None; + CYTHON_UNUSED_VAR(context); Py_INCREF(result); return result; } @@ -273,29 +306,30 @@ __Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { } static int -__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { - PyObject* tmp; +__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); if (!value) { // del => explicit None to prevent rebuilding value = Py_None; - } else if (value != Py_None && !PyTuple_Check(value)) { + } else if (unlikely(value != Py_None && !PyTuple_Check(value))) { PyErr_SetString(PyExc_TypeError, "__defaults__ must be set to a tuple object"); return -1; } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__defaults__ will not " + "currently affect the values used in function calls", 1); Py_INCREF(value); - tmp = op->defaults_tuple; - op->defaults_tuple = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->defaults_tuple, value); return 0; } static PyObject * -__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { +__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, void *context) { PyObject* result = op->defaults_tuple; + CYTHON_UNUSED_VAR(context); if (unlikely(!result)) { if (op->defaults_getter) { - if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; result = op->defaults_tuple; } else { result = Py_None; @@ -306,29 +340,30 @@ __Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *co } static int -__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { - PyObject* tmp; +__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); if (!value) { // del => explicit None to prevent rebuilding value = Py_None; - } else if (value != Py_None && !PyDict_Check(value)) { + } else if (unlikely(value != Py_None && !PyDict_Check(value))) { PyErr_SetString(PyExc_TypeError, "__kwdefaults__ must be set to a dict object"); return -1; } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__kwdefaults__ will not " + "currently affect the values used in function calls", 1); Py_INCREF(value); - tmp = op->defaults_kwdict; - op->defaults_kwdict = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->defaults_kwdict, value); return 0; } static PyObject * -__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { +__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, void *context) { PyObject* result = op->defaults_kwdict; + CYTHON_UNUSED_VAR(context); if (unlikely(!result)) { if (op->defaults_getter) { - if (__Pyx_CyFunction_init_defaults(op) < 0) return NULL; + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; result = op->defaults_kwdict; } else { result = Py_None; @@ -339,25 +374,24 @@ __Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, CYTHON_UNUSED void * } static int -__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, CYTHON_UNUSED void *context) { - PyObject* tmp; +__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); if (!value || value == Py_None) { value = NULL; - } else if (!PyDict_Check(value)) { + } else if (unlikely(!PyDict_Check(value))) { PyErr_SetString(PyExc_TypeError, "__annotations__ must be set to a dict object"); return -1; } Py_XINCREF(value); - tmp = op->func_annotations; - op->func_annotations = value; - Py_XDECREF(tmp); + __Pyx_Py_XDECREF_SET(op->func_annotations, value); return 0; } static PyObject * -__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { +__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, void *context) { PyObject* result = op->func_annotations; + CYTHON_UNUSED_VAR(context); if (unlikely(!result)) { result = PyDict_New(); if (unlikely(!result)) return NULL; @@ -367,10 +401,44 @@ __Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, CYTHON_UNUSED void return result; } +static PyObject * +__Pyx_CyFunction_get_is_coroutine(__pyx_CyFunctionObject *op, void *context) { + int is_coroutine; + CYTHON_UNUSED_VAR(context); + if (op->func_is_coroutine) { + return __Pyx_NewRef(op->func_is_coroutine); + } + + is_coroutine = op->flags & __Pyx_CYFUNCTION_COROUTINE; +#if PY_VERSION_HEX >= 0x03050000 + if (is_coroutine) { + PyObject *module, *fromlist, *marker = PYIDENT("_is_coroutine"); + fromlist = PyList_New(1); + if (unlikely(!fromlist)) return NULL; + Py_INCREF(marker); + PyList_SET_ITEM(fromlist, 0, marker); + module = PyImport_ImportModuleLevelObject(PYIDENT("asyncio.coroutines"), NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + if (unlikely(!module)) goto ignore; + op->func_is_coroutine = __Pyx_PyObject_GetAttrStr(module, marker); + Py_DECREF(module); + if (likely(op->func_is_coroutine)) { + return __Pyx_NewRef(op->func_is_coroutine); + } +ignore: + PyErr_Clear(); + } +#endif + + op->func_is_coroutine = __Pyx_PyBool_FromLong(is_coroutine); + return __Pyx_NewRef(op->func_is_coroutine); +} + //#if PY_VERSION_HEX >= 0x030400C1 //static PyObject * -//__Pyx_CyFunction_get_signature(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *context) { +//__Pyx_CyFunction_get_signature(__pyx_CyFunctionObject *op, void *context) { // PyObject *inspect_module, *signature_class, *signature; +// CYTHON_UNUSED_VAR(context); // // from inspect import Signature // inspect_module = PyImport_ImportModuleLevelObject(PYIDENT("inspect"), NULL, NULL, NULL, 0); // if (unlikely(!inspect_module)) @@ -398,7 +466,6 @@ static PyGetSetDef __pyx_CyFunction_getsets[] = { {(char *) "func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, {(char *) "__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, {(char *) "__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, - {(char *) "__self__", (getter)__Pyx_CyFunction_get_self, 0, 0, 0}, {(char *) "func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, {(char *) "__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, {(char *) "func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, @@ -411,6 +478,7 @@ static PyGetSetDef __pyx_CyFunction_getsets[] = { {(char *) "__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, {(char *) "__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, {(char *) "__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, + {(char *) "_is_coroutine", (getter)__Pyx_CyFunction_get_is_coroutine, 0, 0, 0}, //#if PY_VERSION_HEX >= 0x030400C1 // {(char *) "__signature__", (getter)__Pyx_CyFunction_get_signature, 0, 0, 0}, //#endif @@ -418,18 +486,34 @@ static PyGetSetDef __pyx_CyFunction_getsets[] = { }; static PyMemberDef __pyx_CyFunction_members[] = { - {(char *) "__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), PY_WRITE_RESTRICTED, 0}, + {(char *) "__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), 0, 0}, +#if CYTHON_USE_TYPE_SPECS + {(char *) "__dictoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_dict), READONLY, 0}, +#if CYTHON_METH_FASTCALL +#if CYTHON_BACKPORT_VECTORCALL + {(char *) "__vectorcalloffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_vectorcall), READONLY, 0}, +#else + {(char *) "__vectorcalloffset__", T_PYSSIZET, offsetof(PyCFunctionObject, vectorcall), READONLY, 0}, +#endif +#endif +#if PY_VERSION_HEX < 0x030500A0 + {(char *) "__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_weakreflist), READONLY, 0}, +#else + {(char *) "__weaklistoffset__", T_PYSSIZET, offsetof(PyCFunctionObject, m_weakreflist), READONLY, 0}, +#endif +#endif {0, 0, 0, 0, 0} }; static PyObject * -__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, CYTHON_UNUSED PyObject *args) +__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, PyObject *args) { + CYTHON_UNUSED_VAR(args); #if PY_MAJOR_VERSION >= 3 Py_INCREF(m->func_qualname); return m->func_qualname; #else - return PyString_FromString(m->func.m_ml->ml_name); + return PyString_FromString(((PyCFunctionObject*)m)->m_ml->ml_name); #endif } @@ -442,27 +526,32 @@ static PyMethodDef __pyx_CyFunction_methods[] = { #if PY_VERSION_HEX < 0x030500A0 #define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist) #else -#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func.m_weakreflist) +#define __Pyx_CyFunction_weakreflist(cyfunc) (((PyCFunctionObject*)cyfunc)->m_weakreflist) #endif static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject *op, PyMethodDef *ml, int flags, PyObject* qualname, PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { + PyCFunctionObject *cf = (PyCFunctionObject*) op; if (unlikely(op == NULL)) return NULL; op->flags = flags; __Pyx_CyFunction_weakreflist(op) = NULL; - op->func.m_ml = ml; - op->func.m_self = (PyObject *) op; + cf->m_ml = ml; + cf->m_self = (PyObject *) op; Py_XINCREF(closure); op->func_closure = closure; Py_XINCREF(module); - op->func.m_module = module; + cf->m_module = module; op->func_dict = NULL; op->func_name = NULL; Py_INCREF(qualname); op->func_qualname = qualname; op->func_doc = NULL; +#if PY_VERSION_HEX < 0x030900B1 op->func_classobj = NULL; +#else + ((PyCMethodObject*)op)->mm_class = NULL; +#endif op->func_globals = globals; Py_INCREF(op->func_globals); Py_XINCREF(code); @@ -475,6 +564,32 @@ static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject *op, PyMethodDef * op->defaults_kwdict = NULL; op->defaults_getter = NULL; op->func_annotations = NULL; + op->func_is_coroutine = NULL; +#if CYTHON_METH_FASTCALL + switch (ml->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS | METH_METHOD)) { + case METH_NOARGS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_NOARGS; + break; + case METH_O: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_O; + break; + // case METH_FASTCALL is not used + case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD; + break; + case METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS; + break; + // case METH_VARARGS is not used + case METH_VARARGS | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = NULL; + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + Py_DECREF(op); + return NULL; + } +#endif return (PyObject *) op; } @@ -482,17 +597,26 @@ static int __Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) { Py_CLEAR(m->func_closure); - Py_CLEAR(m->func.m_module); + Py_CLEAR(((PyCFunctionObject*)m)->m_module); Py_CLEAR(m->func_dict); Py_CLEAR(m->func_name); Py_CLEAR(m->func_qualname); Py_CLEAR(m->func_doc); Py_CLEAR(m->func_globals); Py_CLEAR(m->func_code); - Py_CLEAR(m->func_classobj); +#if PY_VERSION_HEX < 0x030900B1 + Py_CLEAR(__Pyx_CyFunction_GetClassObj(m)); +#else + { + PyObject *cls = (PyObject*) ((PyCMethodObject *) (m))->mm_class; + ((PyCMethodObject *) (m))->mm_class = NULL; + Py_XDECREF(cls); + } +#endif Py_CLEAR(m->defaults_tuple); Py_CLEAR(m->defaults_kwdict); Py_CLEAR(m->func_annotations); + Py_CLEAR(m->func_is_coroutine); if (m->defaults) { PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); @@ -513,7 +637,7 @@ static void __Pyx__CyFunction_dealloc(__pyx_CyFunctionObject *m) if (__Pyx_CyFunction_weakreflist(m) != NULL) PyObject_ClearWeakRefs((PyObject *) m); __Pyx_CyFunction_clear(m); - PyObject_GC_Del(m); + __Pyx_PyHeapTypeObject_GC_Del(m); } static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) @@ -525,16 +649,17 @@ static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) { Py_VISIT(m->func_closure); - Py_VISIT(m->func.m_module); + Py_VISIT(((PyCFunctionObject*)m)->m_module); Py_VISIT(m->func_dict); Py_VISIT(m->func_name); Py_VISIT(m->func_qualname); Py_VISIT(m->func_doc); Py_VISIT(m->func_globals); Py_VISIT(m->func_code); - Py_VISIT(m->func_classobj); + Py_VISIT(__Pyx_CyFunction_GetClassObj(m)); Py_VISIT(m->defaults_tuple); Py_VISIT(m->defaults_kwdict); + Py_VISIT(m->func_is_coroutine); if (m->defaults) { PyObject **pydefaults = __Pyx_CyFunction_Defaults(PyObject *, m); @@ -547,28 +672,6 @@ static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, return 0; } -static PyObject *__Pyx_CyFunction_descr_get(PyObject *func, PyObject *obj, PyObject *type) -{ -#if PY_MAJOR_VERSION < 3 - __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; - - if (m->flags & __Pyx_CYFUNCTION_STATICMETHOD) { - Py_INCREF(func); - return func; - } - - if (m->flags & __Pyx_CYFUNCTION_CLASSMETHOD) { - if (type == NULL) - type = (PyObject *)(Py_TYPE(obj)); - return __Pyx_PyMethod_New(func, type, (PyObject *)(Py_TYPE(type))); - } - - if (obj == Py_None) - obj = NULL; -#endif - return __Pyx_PyMethod_New(func, obj, type); -} - static PyObject* __Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) { @@ -628,10 +731,7 @@ static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, Py } break; default: - PyErr_SetString(PyExc_SystemError, "Bad call flags in " - "__Pyx_CyFunction_Call. METH_OLDARGS is no " - "longer supported!"); - + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); return NULL; } PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments", @@ -646,6 +746,22 @@ static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *a static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { PyObject *result; __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; + +#if CYTHON_METH_FASTCALL + // Prefer vectorcall if available. This is not the typical case, as + // CPython would normally use vectorcall directly instead of tp_call. + __pyx_vectorcallfunc vc = __Pyx_CyFunction_func_vectorcall(cyfunc); + if (vc) { +#if CYTHON_ASSUME_SAFE_MACROS + return __Pyx_PyVectorcall_FastCallDict(func, vc, &PyTuple_GET_ITEM(args, 0), (size_t)PyTuple_GET_SIZE(args), kw); +#else + // avoid unused function warning + (void) &__Pyx_PyVectorcall_FastCallDict; + return PyVectorcall_Call(func, args, kw); +#endif + } +#endif + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { Py_ssize_t argc; PyObject *new_args; @@ -682,19 +798,198 @@ static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, P return result; } +#if CYTHON_METH_FASTCALL +// Check that kwnames is empty (if you want to allow keyword arguments, +// simply pass kwnames=NULL) and figure out what to do with "self". +// Return value: +// 1: self = args[0] +// 0: self = cyfunc->func.m_self +// -1: error +static CYTHON_INLINE int __Pyx_CyFunction_Vectorcall_CheckArgs(__pyx_CyFunctionObject *cyfunc, Py_ssize_t nargs, PyObject *kwnames) +{ + int ret = 0; + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + if (unlikely(nargs < 1)) { + PyErr_Format(PyExc_TypeError, "%.200s() needs an argument", + ((PyCFunctionObject*)cyfunc)->m_ml->ml_name); + return -1; + } + ret = 1; + } + if (unlikely(kwnames) && unlikely(PyTuple_GET_SIZE(kwnames))) { + PyErr_Format(PyExc_TypeError, + "%.200s() takes no keyword arguments", ((PyCFunctionObject*)cyfunc)->m_ml->ml_name); + return -1; + } + return ret; +} + +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyMethodDef* def = ((PyCFunctionObject*)cyfunc)->m_ml; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: + self = ((PyCFunctionObject*)cyfunc)->m_self; + break; + default: + return NULL; + } + + if (unlikely(nargs != 0)) { + PyErr_Format(PyExc_TypeError, + "%.200s() takes no arguments (%" CYTHON_FORMAT_SSIZE_T "d given)", + def->ml_name, nargs); + return NULL; + } + return def->ml_meth(self, NULL); +} + +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyMethodDef* def = ((PyCFunctionObject*)cyfunc)->m_ml; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: + self = ((PyCFunctionObject*)cyfunc)->m_self; + break; + default: + return NULL; + } + + if (unlikely(nargs != 1)) { + PyErr_Format(PyExc_TypeError, + "%.200s() takes exactly one argument (%" CYTHON_FORMAT_SSIZE_T "d given)", + def->ml_name, nargs); + return NULL; + } + return def->ml_meth(self, args[0]); +} + +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyMethodDef* def = ((PyCFunctionObject*)cyfunc)->m_ml; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: + self = ((PyCFunctionObject*)cyfunc)->m_self; + break; + default: + return NULL; + } + + return ((_PyCFunctionFastWithKeywords)(void(*)(void))def->ml_meth)(self, args, nargs, kwnames); +} + +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyMethodDef* def = ((PyCFunctionObject*)cyfunc)->m_ml; + PyTypeObject *cls = (PyTypeObject *) __Pyx_CyFunction_GetClassObj(cyfunc); +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: + self = ((PyCFunctionObject*)cyfunc)->m_self; + break; + default: + return NULL; + } + + return ((__Pyx_PyCMethod)(void(*)(void))def->ml_meth)(self, cls, args, (size_t)nargs, kwnames); +} +#endif + +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_CyFunctionType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_CyFunction_dealloc}, + {Py_tp_repr, (void *)__Pyx_CyFunction_repr}, + {Py_tp_call, (void *)__Pyx_CyFunction_CallAsMethod}, + {Py_tp_traverse, (void *)__Pyx_CyFunction_traverse}, + {Py_tp_clear, (void *)__Pyx_CyFunction_clear}, + {Py_tp_methods, (void *)__pyx_CyFunction_methods}, + {Py_tp_members, (void *)__pyx_CyFunction_members}, + {Py_tp_getset, (void *)__pyx_CyFunction_getsets}, + {Py_tp_descr_get, (void *)__Pyx_PyMethod_New}, + {0, 0}, +}; + +static PyType_Spec __pyx_CyFunctionType_spec = { + __PYX_TYPE_MODULE_PREFIX "cython_function_or_method", + sizeof(__pyx_CyFunctionObject), + 0, +#ifdef Py_TPFLAGS_METHOD_DESCRIPTOR + Py_TPFLAGS_METHOD_DESCRIPTOR | +#endif +#if (defined(_Py_TPFLAGS_HAVE_VECTORCALL) && CYTHON_METH_FASTCALL) + _Py_TPFLAGS_HAVE_VECTORCALL | +#endif + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + __pyx_CyFunctionType_slots +}; +#else /* CYTHON_USE_TYPE_SPECS */ + static PyTypeObject __pyx_CyFunctionType_type = { PyVarObject_HEAD_INIT(0, 0) - "cython_function_or_method", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "cython_function_or_method", /*tp_name*/ sizeof(__pyx_CyFunctionObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __Pyx_CyFunction_dealloc, /*tp_dealloc*/ +#if !CYTHON_METH_FASTCALL 0, /*tp_print*/ +#elif CYTHON_BACKPORT_VECTORCALL + (printfunc)offsetof(__pyx_CyFunctionObject, func_vectorcall), /*tp_vectorcall_offset backported into tp_print*/ +#else + offsetof(PyCFunctionObject, vectorcall), /*tp_vectorcall_offset*/ +#endif 0, /*tp_getattr*/ 0, /*tp_setattr*/ #if PY_MAJOR_VERSION < 3 0, /*tp_compare*/ #else - 0, /*reserved*/ + 0, /*tp_as_async*/ #endif (reprfunc) __Pyx_CyFunction_repr, /*tp_repr*/ 0, /*tp_as_number*/ @@ -706,7 +1001,13 @@ static PyTypeObject __pyx_CyFunctionType_type = { 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ +#ifdef Py_TPFLAGS_METHOD_DESCRIPTOR + Py_TPFLAGS_METHOD_DESCRIPTOR | +#endif +#ifdef _Py_TPFLAGS_HAVE_VECTORCALL + _Py_TPFLAGS_HAVE_VECTORCALL | +#endif + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /*tp_flags*/ 0, /*tp_doc*/ (traverseproc) __Pyx_CyFunction_traverse, /*tp_traverse*/ (inquiry) __Pyx_CyFunction_clear, /*tp_clear*/ @@ -723,7 +1024,7 @@ static PyTypeObject __pyx_CyFunctionType_type = { __pyx_CyFunction_getsets, /*tp_getset*/ 0, /*tp_base*/ 0, /*tp_dict*/ - __Pyx_CyFunction_descr_get, /*tp_descr_get*/ + __Pyx_PyMethod_New, /*tp_descr_get*/ 0, /*tp_descr_set*/ offsetof(__pyx_CyFunctionObject, func_dict),/*tp_dictoffset*/ 0, /*tp_init*/ @@ -744,7 +1045,7 @@ static PyTypeObject __pyx_CyFunctionType_type = { #if PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800) 0, /*tp_vectorcall*/ #endif -#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 +#if __PYX_NEED_TP_PRINT_SLOT 0, /*tp_print*/ #endif #if PY_VERSION_HEX >= 0x030C0000 @@ -754,10 +1055,16 @@ static PyTypeObject __pyx_CyFunctionType_type = { 0, /*tp_pypy_flags*/ #endif }; +#endif /* CYTHON_USE_TYPE_SPECS */ -static int __pyx_CyFunction_init(void) { +static int __pyx_CyFunction_init(PyObject *module) { +#if CYTHON_USE_TYPE_SPECS + __pyx_CyFunctionType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx_CyFunctionType_spec, NULL); +#else + CYTHON_UNUSED_VAR(module); __pyx_CyFunctionType = __Pyx_FetchCommonType(&__pyx_CyFunctionType_type); +#endif if (unlikely(__pyx_CyFunctionType == NULL)) { return -1; } @@ -837,8 +1144,7 @@ static int __Pyx_CyFunction_InitClassCell(PyObject *cyfunctions, PyObject *class if (unlikely(!m)) return -1; #endif - Py_INCREF(classobj); - m->func_classobj = classobj; + __Pyx_CyFunction_SetClassObj(m, classobj); #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) Py_DECREF((PyObject*)m); #endif @@ -852,7 +1158,6 @@ static int __Pyx_CyFunction_InitClassCell(PyObject *cyfunctions, PyObject *class typedef struct { __pyx_CyFunctionObject func; PyObject *__signatures__; - PyObject *type; PyObject *self; } __pyx_FusedFunctionObject; @@ -862,8 +1167,7 @@ static PyObject *__pyx_FusedFunction_New(PyMethodDef *ml, int flags, PyObject *code); static int __pyx_FusedFunction_clear(__pyx_FusedFunctionObject *self); -static PyTypeObject *__pyx_FusedFunctionType = NULL; -static int __pyx_FusedFunction_init(void); +static int __pyx_FusedFunction_init(PyObject *module); #define __Pyx_FusedFunction_USED @@ -884,7 +1188,6 @@ __pyx_FusedFunction_New(PyMethodDef *ml, int flags, if (likely(op)) { __pyx_FusedFunctionObject *fusedfunc = (__pyx_FusedFunctionObject *) op; fusedfunc->__signatures__ = NULL; - fusedfunc->type = NULL; fusedfunc->self = NULL; PyObject_GC_Track(op); } @@ -896,7 +1199,6 @@ __pyx_FusedFunction_dealloc(__pyx_FusedFunctionObject *self) { PyObject_GC_UnTrack(self); Py_CLEAR(self->self); - Py_CLEAR(self->type); Py_CLEAR(self->__signatures__); __Pyx__CyFunction_dealloc((__pyx_CyFunctionObject *) self); } @@ -907,7 +1209,6 @@ __pyx_FusedFunction_traverse(__pyx_FusedFunctionObject *self, void *arg) { Py_VISIT(self->self); - Py_VISIT(self->type); Py_VISIT(self->__signatures__); return __Pyx_CyFunction_traverse((__pyx_CyFunctionObject *) self, visit, arg); } @@ -916,7 +1217,6 @@ static int __pyx_FusedFunction_clear(__pyx_FusedFunctionObject *self) { Py_CLEAR(self->self); - Py_CLEAR(self->type); Py_CLEAR(self->__signatures__); return __Pyx_CyFunction_clear((__pyx_CyFunctionObject *) self); } @@ -938,6 +1238,15 @@ __pyx_FusedFunction_descr_get(PyObject *self, PyObject *obj, PyObject *type) if (obj == Py_None) obj = NULL; + if (func->func.flags & __Pyx_CYFUNCTION_CLASSMETHOD) + obj = type; + + if (obj == NULL) { + // We aren't actually binding to anything, save the effort of rebinding + Py_INCREF(self); + return self; + } + meth = (__pyx_FusedFunctionObject *) __pyx_FusedFunction_New( ((PyCFunctionObject *) func)->m_ml, ((__pyx_CyFunctionObject *) func)->flags, @@ -946,7 +1255,7 @@ __pyx_FusedFunction_descr_get(PyObject *self, PyObject *obj, PyObject *type) ((PyCFunctionObject *) func)->m_module, ((__pyx_CyFunctionObject *) func)->func_globals, ((__pyx_CyFunctionObject *) func)->func_code); - if (!meth) + if (unlikely(!meth)) return NULL; // defaults needs copying fully rather than just copying the pointer @@ -956,9 +1265,10 @@ __pyx_FusedFunction_descr_get(PyObject *self, PyObject *obj, PyObject *type) PyObject **pydefaults; int i; - if (!__Pyx_CyFunction_InitDefaults((PyObject*)meth, - func->func.defaults_size, - func->func.defaults_pyobjects)) { + if (unlikely(!__Pyx_CyFunction_InitDefaults( + (PyObject*)meth, + func->func.defaults_size, + func->func.defaults_pyobjects))) { Py_XDECREF((PyObject*)meth); return NULL; } @@ -969,21 +1279,14 @@ __pyx_FusedFunction_descr_get(PyObject *self, PyObject *obj, PyObject *type) Py_XINCREF(pydefaults[i]); } - Py_XINCREF(func->func.func_classobj); - meth->func.func_classobj = func->func.func_classobj; + __Pyx_CyFunction_SetClassObj(meth, __Pyx_CyFunction_GetClassObj(func)); Py_XINCREF(func->__signatures__); meth->__signatures__ = func->__signatures__; - Py_XINCREF(type); - meth->type = type; - Py_XINCREF(func->func.defaults_tuple); meth->func.defaults_tuple = func->func.defaults_tuple; - if (func->func.flags & __Pyx_CYFUNCTION_CLASSMETHOD) - obj = type; - Py_XINCREF(obj); meth->self = obj; @@ -991,12 +1294,18 @@ __pyx_FusedFunction_descr_get(PyObject *self, PyObject *obj, PyObject *type) } static PyObject * -_obj_to_str(PyObject *obj) +_obj_to_string(PyObject *obj) { - if (PyType_Check(obj)) + if (PyUnicode_CheckExact(obj)) + return __Pyx_NewRef(obj); +#if PY_MAJOR_VERSION == 2 + else if (PyString_Check(obj)) + return PyUnicode_FromEncodedObject(obj, NULL, "strict"); +#endif + else if (PyType_Check(obj)) return PyObject_GetAttr(obj, PYIDENT("__name__")); else - return PyObject_Str(obj); + return PyObject_Unicode(obj); } static PyObject * @@ -1006,65 +1315,55 @@ __pyx_FusedFunction_getitem(__pyx_FusedFunctionObject *self, PyObject *idx) PyObject *unbound_result_func; PyObject *result_func = NULL; - if (self->__signatures__ == NULL) { + if (unlikely(self->__signatures__ == NULL)) { PyErr_SetString(PyExc_TypeError, "Function is not fused"); return NULL; } if (PyTuple_Check(idx)) { - PyObject *list = PyList_New(0); Py_ssize_t n = PyTuple_GET_SIZE(idx); - PyObject *sep = NULL; + PyObject *list = PyList_New(n); int i; if (unlikely(!list)) return NULL; for (i = 0; i < n; i++) { - int ret; PyObject *string; #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS PyObject *item = PyTuple_GET_ITEM(idx, i); #else PyObject *item = PySequence_ITEM(idx, i); if (unlikely(!item)) goto __pyx_err; #endif - string = _obj_to_str(item); + string = _obj_to_string(item); #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) Py_DECREF(item); #endif if (unlikely(!string)) goto __pyx_err; - ret = PyList_Append(list, string); - Py_DECREF(string); - if (unlikely(ret < 0)) goto __pyx_err; + PyList_SET_ITEM(list, i, string); } - sep = PyUnicode_FromString("|"); - if (likely(sep)) - signature = PyUnicode_Join(sep, list); -__pyx_err: -; + signature = PyUnicode_Join(PYUNICODE("|"), list); +__pyx_err:; Py_DECREF(list); - Py_XDECREF(sep); } else { - signature = _obj_to_str(idx); + signature = _obj_to_string(idx); } - if (!signature) + if (unlikely(!signature)) return NULL; unbound_result_func = PyObject_GetItem(self->__signatures__, signature); - if (unbound_result_func) { - if (self->self || self->type) { + if (likely(unbound_result_func)) { + if (self->self) { __pyx_FusedFunctionObject *unbound = (__pyx_FusedFunctionObject *) unbound_result_func; // TODO: move this to InitClassCell - Py_CLEAR(unbound->func.func_classobj); - Py_XINCREF(self->func.func_classobj); - unbound->func.func_classobj = self->func.func_classobj; + __Pyx_CyFunction_SetClassObj(unbound, __Pyx_CyFunction_GetClassObj(self)); result_func = __pyx_FusedFunction_descr_get(unbound_result_func, - self->self, self->type); + self->self, self->self); } else { result_func = unbound_result_func; Py_INCREF(result_func); @@ -1084,7 +1383,7 @@ __pyx_FusedFunction_callfunction(PyObject *func, PyObject *args, PyObject *kw) int static_specialized = (cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD && !((__pyx_FusedFunctionObject *) func)->__signatures__); - if (cyfunc->flags & __Pyx_CYFUNCTION_CCLASS && !static_specialized) { + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !static_specialized) { return __Pyx_CyFunction_CallAsMethod(func, args, kw); } else { return __Pyx_CyFunction_Call(func, args, kw); @@ -1105,23 +1404,21 @@ __pyx_FusedFunction_call(PyObject *func, PyObject *args, PyObject *kw) PyObject *new_args = NULL; __pyx_FusedFunctionObject *new_func = NULL; PyObject *result = NULL; - PyObject *self = NULL; int is_staticmethod = binding_func->func.flags & __Pyx_CYFUNCTION_STATICMETHOD; - int is_classmethod = binding_func->func.flags & __Pyx_CYFUNCTION_CLASSMETHOD; if (binding_func->self) { // Bound method call, put 'self' in the args tuple + PyObject *self; Py_ssize_t i; new_args = PyTuple_New(argc + 1); - if (!new_args) + if (unlikely(!new_args)) return NULL; self = binding_func->self; -#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) - Py_INCREF(self); -#endif + Py_INCREF(self); PyTuple_SET_ITEM(new_args, 0, self); + self = NULL; for (i = 0; i < argc; i++) { #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS @@ -1134,35 +1431,7 @@ __pyx_FusedFunction_call(PyObject *func, PyObject *args, PyObject *kw) } args = new_args; - } else if (binding_func->type) { - // Unbound method call - if (argc < 1) { - PyErr_SetString(PyExc_TypeError, "Need at least one argument, 0 given."); - return NULL; - } -#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS - self = PyTuple_GET_ITEM(args, 0); -#else - self = PySequence_ITEM(args, 0); if (unlikely(!self)) return NULL; -#endif - } - - if (self && !is_classmethod && !is_staticmethod) { - int is_instance = PyObject_IsInstance(self, binding_func->type); - if (unlikely(!is_instance)) { - PyErr_Format(PyExc_TypeError, - "First argument should be of type %.200s, got %.200s.", - ((PyTypeObject *) binding_func->type)->tp_name, - Py_TYPE(self)->tp_name); - goto bad; - } else if (unlikely(is_instance == -1)) { - goto bad; - } } -#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) - Py_XDECREF(self); - self = NULL; -#endif if (binding_func->__signatures__) { PyObject *tup; @@ -1186,18 +1455,13 @@ __pyx_FusedFunction_call(PyObject *func, PyObject *args, PyObject *kw) if (unlikely(!new_func)) goto bad; - Py_XINCREF(binding_func->func.func_classobj); - Py_CLEAR(new_func->func.func_classobj); - new_func->func.func_classobj = binding_func->func.func_classobj; + __Pyx_CyFunction_SetClassObj(new_func, __Pyx_CyFunction_GetClassObj(binding_func)); func = (PyObject *) new_func; } result = __pyx_FusedFunction_callfunction(func, args, kw); bad: -#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) - Py_XDECREF(self); -#endif Py_XDECREF(new_args); Py_XDECREF((PyObject *) new_func); return result; @@ -1209,9 +1473,41 @@ static PyMemberDef __pyx_FusedFunction_members[] = { offsetof(__pyx_FusedFunctionObject, __signatures__), READONLY, 0}, + {(char *) "__self__", T_OBJECT_EX, offsetof(__pyx_FusedFunctionObject, self), READONLY, 0}, {0, 0, 0, 0, 0}, }; +static PyGetSetDef __pyx_FusedFunction_getsets[] = { + // __doc__ is None for the fused function type, but we need it to be + // a descriptor for the instance's __doc__, so rebuild the descriptor in our subclass + // (all other descriptors are inherited) + {(char *) "__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {0, 0, 0, 0, 0} +}; + +#if CYTHON_USE_TYPE_SPECS +static PyType_Slot __pyx_FusedFunctionType_slots[] = { + {Py_tp_dealloc, (void *)__pyx_FusedFunction_dealloc}, + {Py_tp_call, (void *)__pyx_FusedFunction_call}, + {Py_tp_traverse, (void *)__pyx_FusedFunction_traverse}, + {Py_tp_clear, (void *)__pyx_FusedFunction_clear}, + {Py_tp_members, (void *)__pyx_FusedFunction_members}, + {Py_tp_getset, (void *)__pyx_FusedFunction_getsets}, + {Py_tp_descr_get, (void *)__pyx_FusedFunction_descr_get}, + {Py_mp_subscript, (void *)__pyx_FusedFunction_getitem}, + {0, 0}, +}; + +static PyType_Spec __pyx_FusedFunctionType_spec = { + __PYX_TYPE_MODULE_PREFIX "fused_cython_function", + sizeof(__pyx_FusedFunctionObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + __pyx_FusedFunctionType_slots +}; + +#else /* !CYTHON_USE_TYPE_SPECS */ + static PyMappingMethods __pyx_FusedFunction_mapping_methods = { 0, (binaryfunc) __pyx_FusedFunction_getitem, @@ -1220,7 +1516,7 @@ static PyMappingMethods __pyx_FusedFunction_mapping_methods = { static PyTypeObject __pyx_FusedFunctionType_type = { PyVarObject_HEAD_INIT(0, 0) - "fused_cython_function", /*tp_name*/ + __PYX_TYPE_MODULE_PREFIX "fused_cython_function", /*tp_name*/ sizeof(__pyx_FusedFunctionObject), /*tp_basicsize*/ 0, /*tp_itemsize*/ (destructor) __pyx_FusedFunction_dealloc, /*tp_dealloc*/ @@ -1230,7 +1526,7 @@ static PyTypeObject __pyx_FusedFunctionType_type = { #if PY_MAJOR_VERSION < 3 0, /*tp_compare*/ #else - 0, /*reserved*/ + 0, /*tp_as_async*/ #endif 0, /*tp_repr*/ 0, /*tp_as_number*/ @@ -1252,9 +1548,7 @@ static PyTypeObject __pyx_FusedFunctionType_type = { 0, /*tp_iternext*/ 0, /*tp_methods*/ __pyx_FusedFunction_members, /*tp_members*/ - // __doc__ is None for the fused function type, but we need it to be - // a descriptor for the instance's __doc__, so rebuild descriptors in our subclass - __pyx_CyFunction_getsets, /*tp_getset*/ + __pyx_FusedFunction_getsets, /*tp_getset*/ // NOTE: tp_base may be changed later during module initialisation when importing CyFunction across modules. &__pyx_CyFunctionType_type, /*tp_base*/ 0, /*tp_dict*/ @@ -1279,7 +1573,7 @@ static PyTypeObject __pyx_FusedFunctionType_type = { #if PY_VERSION_HEX >= 0x030800b1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07030800) 0, /*tp_vectorcall*/ #endif -#if PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000 +#if __PYX_NEED_TP_PRINT_SLOT 0, /*tp_print*/ #endif #if PY_VERSION_HEX >= 0x030C0000 @@ -1289,12 +1583,23 @@ static PyTypeObject __pyx_FusedFunctionType_type = { 0, /*tp_pypy_flags*/ #endif }; +#endif -static int __pyx_FusedFunction_init(void) { +static int __pyx_FusedFunction_init(PyObject *module) { +#if CYTHON_USE_TYPE_SPECS + PyObject *bases = PyTuple_Pack(1, __pyx_CyFunctionType); + if (unlikely(!bases)) { + return -1; + } + __pyx_FusedFunctionType = __Pyx_FetchCommonTypeFromSpec(module, &__pyx_FusedFunctionType_spec, bases); + Py_DECREF(bases); +#else + CYTHON_UNUSED_VAR(module); // Set base from __Pyx_FetchCommonTypeFromSpec, in case it's different from the local static value. __pyx_FusedFunctionType_type.tp_base = __pyx_CyFunctionType; __pyx_FusedFunctionType = __Pyx_FetchCommonType(&__pyx_FusedFunctionType_type); - if (__pyx_FusedFunctionType == NULL) { +#endif + if (unlikely(__pyx_FusedFunctionType == NULL)) { return -1; } return 0; @@ -1303,7 +1608,7 @@ static int __pyx_FusedFunction_init(void) { //////////////////// ClassMethod.proto //////////////////// #include "descrobject.h" -static CYTHON_UNUSED PyObject* __Pyx_Method_ClassMethod(PyObject *method); /*proto*/ +CYTHON_UNUSED static PyObject* __Pyx_Method_ClassMethod(PyObject *method); /*proto*/ //////////////////// ClassMethod //////////////////// @@ -1314,16 +1619,16 @@ static PyObject* __Pyx_Method_ClassMethod(PyObject *method) { return PyClassMethod_New(method); } #else -#if CYTHON_COMPILING_IN_PYSTON || CYTHON_COMPILING_IN_PYPY - // special C-API function only in Pyston and PyPy >= 5.9 +#if CYTHON_COMPILING_IN_PYPY + // special C-API function only in PyPy >= 5.9 if (PyMethodDescr_Check(method)) #else #if PY_MAJOR_VERSION == 2 // PyMethodDescr_Type is not exposed in the CPython C-API in Py2. static PyTypeObject *methoddescr_type = NULL; - if (methoddescr_type == NULL) { + if (unlikely(methoddescr_type == NULL)) { PyObject *meth = PyObject_GetAttrString((PyObject*)&PyList_Type, "append"); - if (!meth) return NULL; + if (unlikely(!meth)) return NULL; methoddescr_type = Py_TYPE(meth); Py_DECREF(meth); } diff --git a/Cython/Utility/Dataclasses.c b/Cython/Utility/Dataclasses.c new file mode 100644 index 000000000..fc6a88d94 --- /dev/null +++ b/Cython/Utility/Dataclasses.c @@ -0,0 +1,178 @@ +///////////////////// ModuleLoader.proto ////////////////////////// + +static PyObject* __Pyx_LoadInternalModule(const char* name, const char* fallback_code); /* proto */ + +//////////////////// ModuleLoader /////////////////////// +//@requires: CommonStructures.c::FetchSharedCythonModule + +static PyObject* __Pyx_LoadInternalModule(const char* name, const char* fallback_code) { + // We want to be able to use the contents of the standard library dataclasses module where available. + // If those objects aren't available (due to Python version) then a simple fallback is substituted + // instead, which largely just fails with a not-implemented error. + // + // The fallbacks are placed in the "shared abi module" as a convenient internal place to + // store them + + PyObject *shared_abi_module = 0, *module = 0; + + shared_abi_module = __Pyx_FetchSharedCythonABIModule(); + if (!shared_abi_module) return NULL; + + if (PyObject_HasAttrString(shared_abi_module, name)) { + PyObject* result = PyObject_GetAttrString(shared_abi_module, name); + Py_DECREF(shared_abi_module); + return result; + } + + // the best and simplest case is simply to defer to the standard library (if available) + module = PyImport_ImportModule(name); + if (!module) { + PyObject *localDict, *runValue, *builtins, *modulename; + if (!PyErr_ExceptionMatches(PyExc_ImportError)) goto bad; + PyErr_Clear(); // this is reasonably likely (especially on older versions of Python) +#if PY_MAJOR_VERSION < 3 + modulename = PyBytes_FromFormat("_cython_" CYTHON_ABI ".%s", name); +#else + modulename = PyUnicode_FromFormat("_cython_" CYTHON_ABI ".%s", name); +#endif + if (!modulename) goto bad; +#if PY_MAJOR_VERSION >= 3 && CYTHON_COMPILING_IN_CPYTHON + module = PyImport_AddModuleObject(modulename); // borrowed +#else + module = PyImport_AddModule(PyBytes_AsString(modulename)); // borrowed +#endif + Py_DECREF(modulename); + if (!module) goto bad; + Py_INCREF(module); + if (PyObject_SetAttrString(shared_abi_module, name, module) < 0) goto bad; + localDict = PyModule_GetDict(module); // borrowed + if (!localDict) goto bad; + builtins = PyEval_GetBuiltins(); // borrowed + if (!builtins) goto bad; + if (PyDict_SetItemString(localDict, "__builtins__", builtins) <0) goto bad; + + runValue = PyRun_String(fallback_code, Py_file_input, localDict, localDict); + if (!runValue) goto bad; + Py_DECREF(runValue); + } + goto shared_cleanup; + + bad: + Py_CLEAR(module); + shared_cleanup: + Py_XDECREF(shared_abi_module); + return module; +} + +///////////////////// SpecificModuleLoader.proto ////////////////////// +//@substitute: tempita + +static PyObject* __Pyx_Load_{{cname}}_Module(void); /* proto */ + + +//////////////////// SpecificModuleLoader /////////////////////// +//@requires: ModuleLoader + +static PyObject* __Pyx_Load_{{cname}}_Module(void) { + return __Pyx_LoadInternalModule("{{cname}}", {{py_code}}); +} + +//////////////////// DataclassesCallHelper.proto //////////////////////// + +static PyObject* __Pyx_DataclassesCallHelper(PyObject *callable, PyObject *kwds); /* proto */ + +//////////////////// DataclassesCallHelper //////////////////////// +//@substitute: naming + +// The signature of a few of the dataclasses module functions has +// been expanded over the years. Cython always passes the full set +// of arguments from the most recent version we know of, so needs +// to remove any arguments that don't exist on earlier versions. + +#if PY_MAJOR_VERSION >= 3 +static int __Pyx_DataclassesCallHelper_FilterToDict(PyObject *callable, PyObject *kwds, PyObject *new_kwds, PyObject *args_list, int is_kwonly) { + Py_ssize_t size, i; + size = PySequence_Size(args_list); + if (size == -1) return -1; + + for (i=0; i<size; ++i) { + PyObject *key, *value; + int setitem_result; + key = PySequence_GetItem(args_list, i); + if (!key) return -1; + + if (PyUnicode_Check(key) && ( + PyUnicode_CompareWithASCIIString(key, "self") == 0 || + // namedtuple constructor in fallback code + PyUnicode_CompareWithASCIIString(key, "_cls") == 0)) { + Py_DECREF(key); + continue; + } + + value = PyDict_GetItem(kwds, key); + if (!value) { + if (is_kwonly) { + Py_DECREF(key); + continue; + } else { + // The most likely reason for this is that Cython + // hasn't kept up to date with the Python dataclasses module. + // To be nice to our users, try not to fail, but ask them + // to report a bug so we can keep up to date. + value = Py_None; + if (PyErr_WarnFormat( + PyExc_RuntimeWarning, 1, + "Argument %S not passed to %R. This is likely a bug in Cython so please report it.", + key, callable) == -1) { + Py_DECREF(key); + return -1; + } + } + } + Py_INCREF(value); + setitem_result = PyDict_SetItem(new_kwds, key, value); + Py_DECREF(key); + Py_DECREF(value); + if (setitem_result == -1) return -1; + } + return 0; +} +#endif + +static PyObject* __Pyx_DataclassesCallHelper(PyObject *callable, PyObject *kwds) { +#if PY_MAJOR_VERSION < 3 + // We're falling back to our full replacement anyway + return PyObject_Call(callable, $empty_tuple, kwds); +#else + PyObject *new_kwds=NULL, *result=NULL; + PyObject *inspect; + PyObject *args_list=NULL, *kwonly_args_list=NULL, *getfullargspec_result=NULL; + + // Going via inspect to work out what arguments to pass is unlikely to be the + // fastest thing ever. However, it is compatible, and only happens once + // at module-import time. + inspect = PyImport_ImportModule("inspect"); + if (!inspect) goto bad; + getfullargspec_result = PyObject_CallMethodObjArgs(inspect, PYUNICODE("getfullargspec"), callable, NULL); + Py_DECREF(inspect); + if (!getfullargspec_result) goto bad; + args_list = PyObject_GetAttrString(getfullargspec_result, "args"); + if (!args_list) goto bad; + kwonly_args_list = PyObject_GetAttrString(getfullargspec_result, "kwonlyargs"); + if (!kwonly_args_list) goto bad; + + new_kwds = PyDict_New(); + if (!new_kwds) goto bad; + + // copy over only those arguments that are in the specification + if (__Pyx_DataclassesCallHelper_FilterToDict(callable, kwds, new_kwds, args_list, 0) == -1) goto bad; + if (__Pyx_DataclassesCallHelper_FilterToDict(callable, kwds, new_kwds, kwonly_args_list, 1) == -1) goto bad; + result = PyObject_Call(callable, $empty_tuple, new_kwds); +bad: + Py_XDECREF(getfullargspec_result); + Py_XDECREF(args_list); + Py_XDECREF(kwonly_args_list); + Py_XDECREF(new_kwds); + return result; +#endif +} diff --git a/Cython/Utility/Dataclasses.py b/Cython/Utility/Dataclasses.py new file mode 100644 index 000000000..2aa2d25a3 --- /dev/null +++ b/Cython/Utility/Dataclasses.py @@ -0,0 +1,112 @@ +################### Dataclasses_fallback ############################### + +# This is the fallback dataclass code if the stdlib module isn't available. +# It defines enough of the support types to be used with cdef classes +# and to fail if used on regular types. + +# (Intended to be included as py code - not compiled) + +from collections import namedtuple +try: + from types import MappingProxyType +except ImportError: + # mutable fallback if unavailable + MappingProxyType = lambda x: x + +class _MISSING_TYPE(object): + pass +MISSING = _MISSING_TYPE() + +_DataclassParams = namedtuple('_DataclassParams', + ["init", "repr", "eq", "order", "unsafe_hash", "frozen", + "match_args", "kw_only", "slots", "weakref_slot"]) +class Field(object): + __slots__ = ('name', + 'type', + 'default', + 'default_factory', + 'repr', + 'hash', + 'init', + 'compare', + 'metadata', + 'kw_only', + '_field_type', # Private: not to be used by user code. + ) + + def __init__(self, default, default_factory, init, repr, hash, compare, + metadata, kw_only): + self.name = None + self.type = None + self.default = default + self.default_factory = default_factory + self.init = init + self.repr = repr + self.hash = hash + self.compare = compare + # Be aware that if MappingProxyType is unavailable (i.e. py2?) then we + # don't enforce non-mutability that the real module does + self.metadata = (MappingProxyType({}) + if metadata is None else + MappingProxyType(metadata)) + self.kw_only = kw_only + self._field_type = None + + def __repr__(self): + return ('Field(' + 'name={0!r},' + 'type={1!r},' + 'default={2!r},' + 'default_factory={3!r},' + 'init={4!r},' + 'repr={5!r},' + 'hash={6!r},' + 'compare={7!r},' + 'metadata={8!r},' + 'kwonly={9!r},' + ')'.format(self.name, self.type, self.default, + self.default_factory, self.init, + self.repr, self.hash, self.compare, + self.metadata, self.kw_only)) + +# A sentinel object for default values to signal that a default +# factory will be used. This is given a nice repr() which will appear +# in the function signature of dataclasses' constructors. +class _HAS_DEFAULT_FACTORY_CLASS: + def __repr__(self): + return '<factory>' +_HAS_DEFAULT_FACTORY = _HAS_DEFAULT_FACTORY_CLASS() + +def dataclass(*args, **kwds): + raise NotImplementedError("Standard library 'dataclasses' module" + "is unavailable, likely due to the version of Python you're using.") + +# Markers for the various kinds of fields and pseudo-fields. +class _FIELD_BASE: + def __init__(self, name): + self.name = name + def __repr__(self): + return self.name +_FIELD = _FIELD_BASE('_FIELD') +_FIELD_CLASSVAR = _FIELD_BASE('_FIELD_CLASSVAR') +_FIELD_INITVAR = _FIELD_BASE('_FIELD_INITVAR') + +def field(*ignore, **kwds): + default = kwds.pop("default", MISSING) + default_factory = kwds.pop("default_factory", MISSING) + init = kwds.pop("init", True) + repr = kwds.pop("repr", True) + hash = kwds.pop("hash", None) + compare = kwds.pop("compare", True) + metadata = kwds.pop("metadata", None) + kw_only = kwds.pop("kw_only", None) + + if kwds: + raise ValueError("field received unexpected keyword arguments: %s" + % list(kwds.keys())) + if default is not MISSING and default_factory is not MISSING: + raise ValueError('cannot specify both default and default_factory') + if ignore: + raise ValueError("'field' does not take any positional arguments") + return Field(default, default_factory, init, + repr, hash, compare, metadata, kw_only) diff --git a/Cython/Utility/Embed.c b/Cython/Utility/Embed.c index 8f7e8f46e..3c827794e 100644 --- a/Cython/Utility/Embed.c +++ b/Cython/Utility/Embed.c @@ -5,12 +5,13 @@ #endif #if PY_MAJOR_VERSION < 3 -int %(main_method)s(int argc, char** argv) { -#elif defined(WIN32) || defined(MS_WINDOWS) -int %(wmain_method)s(int argc, wchar_t **argv) { +int %(main_method)s(int argc, char** argv) +#elif defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) +int %(wmain_method)s(int argc, wchar_t **argv) #else -static int __Pyx_main(int argc, wchar_t **argv) { +static int __Pyx_main(int argc, wchar_t **argv) #endif +{ /* 754 requires that FP exceptions run in "no stop" mode by default, * and until C vendors implement C99's ways to control FP exceptions, * Python requires non-stop mode. Alas, some platforms enable FP @@ -22,34 +23,60 @@ static int __Pyx_main(int argc, wchar_t **argv) { m = fpgetmask(); fpsetmask(m & ~FP_X_OFL); #endif +#if PY_VERSION_HEX < 0x03080000 if (argc && argv) Py_SetProgramName(argv[0]); +#endif + + #if PY_MAJOR_VERSION < 3 + if (PyImport_AppendInittab("%(module_name)s", init%(module_name)s) < 0) return 1; + #else + if (PyImport_AppendInittab("%(module_name)s", PyInit_%(module_name)s) < 0) return 1; + #endif + +#if PY_VERSION_HEX < 0x03080000 Py_Initialize(); if (argc && argv) PySys_SetArgv(argc, argv); +#else + { + PyStatus status; + + PyConfig config; + PyConfig_InitPythonConfig(&config); + // Disable parsing command line arguments + config.parse_argv = 0; + + if (argc && argv) { + status = PyConfig_SetString(&config, &config.program_name, argv[0]); + if (PyStatus_Exception(status)) { + PyConfig_Clear(&config); + return 1; + } + + status = PyConfig_SetArgv(&config, argc, argv); + if (PyStatus_Exception(status)) { + PyConfig_Clear(&config); + return 1; + } + } + + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + PyConfig_Clear(&config); + return 1; + } + + PyConfig_Clear(&config); + } +#endif + { /* init module '%(module_name)s' as '__main__' */ PyObject* m = NULL; %(module_is_main)s = 1; - #if PY_MAJOR_VERSION < 3 - init%(module_name)s(); - #elif CYTHON_PEP489_MULTI_PHASE_INIT - m = PyInit_%(module_name)s(); - if (!PyModule_Check(m)) { - PyModuleDef *mdef = (PyModuleDef *) m; - PyObject *modname = PyUnicode_FromString("__main__"); - m = NULL; - if (modname) { - // FIXME: not currently calling PyModule_FromDefAndSpec() here because we do not have a module spec! - // FIXME: not currently setting __file__, __path__, __spec__, ... - m = PyModule_NewObject(modname); - Py_DECREF(modname); - if (m) PyModule_ExecDef(m, mdef); - } - } - #else - m = PyInit_%(module_name)s(); - #endif - if (PyErr_Occurred()) { + m = PyImport_ImportModule("%(module_name)s"); + + if (!m && PyErr_Occurred()) { PyErr_Print(); /* This exits with the right code if SystemExit. */ #if PY_MAJOR_VERSION < 3 if (Py_FlushLine()) PyErr_Clear(); @@ -68,9 +95,11 @@ static int __Pyx_main(int argc, wchar_t **argv) { } -#if PY_MAJOR_VERSION >= 3 && !defined(WIN32) && !defined(MS_WINDOWS) +#if PY_MAJOR_VERSION >= 3 && !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) #include <locale.h> +#if PY_VERSION_HEX < 0x03050000 + static wchar_t* __Pyx_char2wchar(char* arg) { @@ -175,6 +204,8 @@ oom: return NULL; } +#endif + int %(main_method)s(int argc, char **argv) { @@ -197,7 +228,12 @@ int res = 0; setlocale(LC_ALL, ""); for (i = 0; i < argc; i++) { - argv_copy2[i] = argv_copy[i] = __Pyx_char2wchar(argv[i]); + argv_copy2[i] = argv_copy[i] = +#if PY_VERSION_HEX < 0x03050000 + __Pyx_char2wchar(argv[i]); +#else + Py_DecodeLocale(argv[i], NULL); +#endif if (!argv_copy[i]) res = 1; /* failure, but continue to simplify cleanup */ } setlocale(LC_ALL, oldloc); diff --git a/Cython/Utility/Exceptions.c b/Cython/Utility/Exceptions.c index 87d3a5cdd..abf95afda 100644 --- a/Cython/Utility/Exceptions.c +++ b/Cython/Utility/Exceptions.c @@ -6,6 +6,49 @@ // __Pyx_GetException() +/////////////// AssertionsEnabled.init /////////////// +__Pyx_init_assertions_enabled(); + +/////////////// AssertionsEnabled.proto /////////////// + +#define __Pyx_init_assertions_enabled() + +#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600 && !defined(Py_OptimizeFlag) + #define __pyx_assertions_enabled() (1) +#elif PY_VERSION_HEX < 0x03080000 || CYTHON_COMPILING_IN_PYPY || defined(Py_LIMITED_API) + #define __pyx_assertions_enabled() (!Py_OptimizeFlag) +#elif CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030900A6 + // Py3.8+ has PyConfig from PEP 587, but only Py3.9 added read access to it. + // Py_OptimizeFlag is deprecated in Py3.12+ + static int __pyx_assertions_enabled_flag; + #define __pyx_assertions_enabled() (__pyx_assertions_enabled_flag) + + #undef __Pyx_init_assertions_enabled + static void __Pyx_init_assertions_enabled(void) { + __pyx_assertions_enabled_flag = ! _PyInterpreterState_GetConfig(__Pyx_PyThreadState_Current->interp)->optimization_level; + } +#else + #define __pyx_assertions_enabled() (!Py_OptimizeFlag) +#endif + + +/////////////// ErrOccurredWithGIL.proto /////////////// +static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void); /* proto */ + +/////////////// ErrOccurredWithGIL /////////////// +static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void) { + int err; + #ifdef WITH_THREAD + PyGILState_STATE _save = PyGILState_Ensure(); + #endif + err = !!PyErr_Occurred(); + #ifdef WITH_THREAD + PyGILState_Release(_save); + #endif + return err; +} + + /////////////// PyThreadStateGet.proto /////////////// //@substitute: naming @@ -127,9 +170,9 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject // has changed quite a lot between the two versions. #if PY_MAJOR_VERSION < 3 -static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, - CYTHON_UNUSED PyObject *cause) { +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { __Pyx_PyThreadState_declare + CYTHON_UNUSED_VAR(cause); /* 'cause' is only used in Py3 */ Py_XINCREF(type); if (!value || value == Py_None) @@ -310,19 +353,19 @@ bad: /////////////// GetTopmostException.proto /////////////// -#if CYTHON_USE_EXC_INFO_STACK +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE static _PyErr_StackItem * __Pyx_PyErr_GetTopmostException(PyThreadState *tstate); #endif /////////////// GetTopmostException /////////////// -#if CYTHON_USE_EXC_INFO_STACK +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE // Copied from errors.c in CPython. static _PyErr_StackItem * __Pyx_PyErr_GetTopmostException(PyThreadState *tstate) { _PyErr_StackItem *exc_info = tstate->exc_info; - while ((exc_info->exc_type == NULL || exc_info->exc_type == Py_None) && + while ((exc_info->exc_value == NULL || exc_info->exc_value == Py_None) && exc_info->previous_item != NULL) { exc_info = exc_info->previous_item; @@ -388,12 +431,21 @@ static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) #if CYTHON_USE_EXC_INFO_STACK { _PyErr_StackItem *exc_info = tstate->exc_info; + #if PY_VERSION_HEX >= 0x030B00a4 + tmp_value = exc_info->exc_value; + exc_info->exc_value = local_value; + tmp_type = NULL; + tmp_tb = NULL; + Py_XDECREF(local_type); + Py_XDECREF(local_tb); + #else tmp_type = exc_info->exc_type; tmp_value = exc_info->exc_value; tmp_tb = exc_info->exc_traceback; exc_info->exc_type = local_type; exc_info->exc_value = local_value; exc_info->exc_traceback = local_tb; + #endif } #else tmp_type = tstate->exc_type; @@ -433,35 +485,44 @@ static CYTHON_INLINE void __Pyx_ReraiseException(void) { PyObject *type = NULL, *value = NULL, *tb = NULL; #if CYTHON_FAST_THREAD_STATE PyThreadState *tstate = PyThreadState_GET(); - #if CYTHON_USE_EXC_INFO_STACK + #if CYTHON_USE_EXC_INFO_STACK _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); - type = exc_info->exc_type; value = exc_info->exc_value; - tb = exc_info->exc_traceback; + #if PY_VERSION_HEX >= 0x030B00a4 + if (unlikely(value == Py_None)) { + value = NULL; + } else if (value) { + Py_INCREF(value); + type = (PyObject*) Py_TYPE(value); + Py_INCREF(type); + tb = PyException_GetTraceback(value); + } #else + type = exc_info->exc_type; + tb = exc_info->exc_traceback; + Py_XINCREF(type); + Py_XINCREF(value); + Py_XINCREF(tb); + #endif + #else type = tstate->exc_type; value = tstate->exc_value; tb = tstate->exc_traceback; - #endif + Py_XINCREF(type); + Py_XINCREF(value); + Py_XINCREF(tb); + #endif #else PyErr_GetExcInfo(&type, &value, &tb); #endif - if (!type || type == Py_None) { -#if !CYTHON_FAST_THREAD_STATE + if (unlikely(!type || type == Py_None)) { Py_XDECREF(type); Py_XDECREF(value); Py_XDECREF(tb); -#endif // message copied from Py3 PyErr_SetString(PyExc_RuntimeError, "No active exception to reraise"); } else { -#if CYTHON_FAST_THREAD_STATE - Py_INCREF(type); - Py_XINCREF(value); - Py_XINCREF(tb); - -#endif PyErr_Restore(type, value, tb); } } @@ -487,24 +548,49 @@ static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject #if CYTHON_FAST_THREAD_STATE static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { - #if CYTHON_USE_EXC_INFO_STACK + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + PyObject *exc_value = exc_info->exc_value; + if (exc_value == NULL || exc_value == Py_None) { + *value = NULL; + *type = NULL; + *tb = NULL; + } else { + *value = exc_value; + Py_INCREF(*value); + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + *tb = PyException_GetTraceback(exc_value); + } + #elif CYTHON_USE_EXC_INFO_STACK _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); *type = exc_info->exc_type; *value = exc_info->exc_value; *tb = exc_info->exc_traceback; - #else + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #else *type = tstate->exc_type; *value = tstate->exc_value; *tb = tstate->exc_traceback; - #endif Py_XINCREF(*type); Py_XINCREF(*value); Py_XINCREF(*tb); + #endif } static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + PyObject *tmp_value = exc_info->exc_value; + exc_info->exc_value = value; + Py_XDECREF(tmp_value); + // TODO: avoid passing these at all + Py_XDECREF(type); + Py_XDECREF(tb); + #else PyObject *tmp_type, *tmp_value, *tmp_tb; - #if CYTHON_USE_EXC_INFO_STACK _PyErr_StackItem *exc_info = tstate->exc_info; tmp_type = exc_info->exc_type; @@ -524,6 +610,7 @@ static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject Py_XDECREF(tmp_type); Py_XDECREF(tmp_value); Py_XDECREF(tmp_tb); + #endif } #endif @@ -543,8 +630,22 @@ static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, #if CYTHON_FAST_THREAD_STATE static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { PyObject *tmp_type, *tmp_value, *tmp_tb; - - #if CYTHON_USE_EXC_INFO_STACK + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_value = exc_info->exc_value; + exc_info->exc_value = *value; + if (tmp_value == NULL || tmp_value == Py_None) { + Py_XDECREF(tmp_value); + tmp_value = NULL; + tmp_type = NULL; + tmp_tb = NULL; + } else { + // TODO: avoid swapping these at all + tmp_type = (PyObject*) Py_TYPE(tmp_value); + Py_INCREF(tmp_type); + tmp_tb = PyException_GetTraceback(tmp_value); + } + #elif CYTHON_USE_EXC_INFO_STACK _PyErr_StackItem *exc_info = tstate->exc_info; tmp_type = exc_info->exc_type; tmp_value = exc_info->exc_value; @@ -553,7 +654,7 @@ static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject * exc_info->exc_type = *type; exc_info->exc_value = *value; exc_info->exc_traceback = *tb; - #else + #else tmp_type = tstate->exc_type; tmp_value = tstate->exc_value; tmp_tb = tstate->exc_traceback; @@ -561,7 +662,7 @@ static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject * tstate->exc_type = *type; tstate->exc_value = *value; tstate->exc_traceback = *tb; - #endif + #endif *type = tmp_type; *value = tmp_value; @@ -590,9 +691,9 @@ static void __Pyx_WriteUnraisable(const char *name, int clineno, //@requires: PyErrFetchRestore //@requires: PyThreadStateGet -static void __Pyx_WriteUnraisable(const char *name, CYTHON_UNUSED int clineno, - CYTHON_UNUSED int lineno, CYTHON_UNUSED const char *filename, - int full_traceback, CYTHON_UNUSED int nogil) { +static void __Pyx_WriteUnraisable(const char *name, int clineno, + int lineno, const char *filename, + int full_traceback, int nogil) { PyObject *old_exc, *old_val, *old_tb; PyObject *ctx; __Pyx_PyThreadState_declare @@ -600,9 +701,14 @@ static void __Pyx_WriteUnraisable(const char *name, CYTHON_UNUSED int clineno, PyGILState_STATE state; if (nogil) state = PyGILState_Ensure(); - /* initalize to suppress warning */ + /* arbitrary, to suppress warning */ else state = (PyGILState_STATE)0; #endif + CYTHON_UNUSED_VAR(clineno); + CYTHON_UNUSED_VAR(lineno); + CYTHON_UNUSED_VAR(filename); + CYTHON_MAYBE_UNUSED_VAR(nogil); + __Pyx_PyThreadState_assign __Pyx_ErrFetch(&old_exc, &old_val, &old_tb); if (full_traceback) { @@ -639,19 +745,21 @@ static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line);/*proto*/ #endif /////////////// CLineInTraceback /////////////// -//@requires: ObjectHandling.c::PyObjectGetAttrStr +//@requires: ObjectHandling.c::PyObjectGetAttrStrNoError //@requires: ObjectHandling.c::PyDictVersioning //@requires: PyErrFetchRestore //@substitute: naming #ifndef CYTHON_CLINE_IN_TRACEBACK -static int __Pyx_CLineForTraceback(CYTHON_UNUSED PyThreadState *tstate, int c_line) { +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) { PyObject *use_cline; PyObject *ptype, *pvalue, *ptraceback; #if CYTHON_COMPILING_IN_CPYTHON PyObject **cython_runtime_dict; #endif + CYTHON_MAYBE_UNUSED_VAR(tstate); + if (unlikely(!${cython_runtime_cname})) { // Very early error where the runtime module is not set up yet. return c_line; @@ -668,7 +776,7 @@ static int __Pyx_CLineForTraceback(CYTHON_UNUSED PyThreadState *tstate, int c_li } else #endif { - PyObject *use_cline_obj = __Pyx_PyObject_GetAttrStr(${cython_runtime_cname}, PYIDENT("cline_in_traceback")); + PyObject *use_cline_obj = __Pyx_PyObject_GetAttrStrNoError(${cython_runtime_cname}, PYIDENT("cline_in_traceback")); if (use_cline_obj) { use_cline = PyObject_Not(use_cline_obj) ? Py_False : Py_True; Py_DECREF(use_cline_obj); @@ -710,6 +818,17 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, #include "internal/pycore_frame.h" #endif +#if CYTHON_COMPILING_IN_LIMITED_API +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + if (c_line) { + // Avoid "unused" warning as long as we don't use this. + (void) $cfilenm_cname; + (void) __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line); + } + _PyTraceback_Add(funcname, filename, py_line); +} +#else static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( const char *funcname, int c_line, int py_line, const char *filename) { @@ -742,6 +861,7 @@ static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( #if PY_MAJOR_VERSION < 3 py_code = __Pyx_PyCode_New( 0, /*int argcount,*/ + 0, /*int posonlyargcount,*/ 0, /*int kwonlyargcount,*/ 0, /*int nlocals,*/ 0, /*int stacksize,*/ @@ -812,3 +932,4 @@ bad: Py_XDECREF(py_code); Py_XDECREF(py_frame); } +#endif diff --git a/Cython/Utility/ExtensionTypes.c b/Cython/Utility/ExtensionTypes.c index dc187ab49..8b73824dd 100644 --- a/Cython/Utility/ExtensionTypes.c +++ b/Cython/Utility/ExtensionTypes.c @@ -1,12 +1,107 @@ -/////////////// PyType_Ready.proto /////////////// +/////////////// FixUpExtensionType.proto /////////////// -static int __Pyx_PyType_Ready(PyTypeObject *t); +#if CYTHON_USE_TYPE_SPECS +static int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type); /*proto*/ +#endif -/////////////// PyType_Ready /////////////// +/////////////// FixUpExtensionType /////////////// +//@requires:ModuleSetupCode.c::IncludeStructmemberH +//@requires:StringTools.c::IncludeStringH -// Wrapper around PyType_Ready() with some runtime checks and fixes -// to deal with multiple inheritance. -static int __Pyx_PyType_Ready(PyTypeObject *t) { +#if CYTHON_USE_TYPE_SPECS +static int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type) { +#if PY_VERSION_HEX > 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + CYTHON_UNUSED_VAR(spec); + CYTHON_UNUSED_VAR(type); +#else + // Set tp_weakreflist, tp_dictoffset, tp_vectorcalloffset + // Copied and adapted from https://bugs.python.org/issue38140 + const PyType_Slot *slot = spec->slots; + while (slot && slot->slot && slot->slot != Py_tp_members) + slot++; + if (slot && slot->slot == Py_tp_members) { + int changed = 0; +#if !(PY_VERSION_HEX <= 0x030900b1 && CYTHON_COMPILING_IN_CPYTHON) + const +#endif + PyMemberDef *memb = (PyMemberDef*) slot->pfunc; + while (memb && memb->name) { + if (memb->name[0] == '_' && memb->name[1] == '_') { +#if PY_VERSION_HEX < 0x030900b1 + if (strcmp(memb->name, "__weaklistoffset__") == 0) { + // The PyMemberDef must be a Py_ssize_t and readonly. + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_weaklistoffset = memb->offset; + // FIXME: is it even worth calling PyType_Modified() here? + changed = 1; + } + else if (strcmp(memb->name, "__dictoffset__") == 0) { + // The PyMemberDef must be a Py_ssize_t and readonly. + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_dictoffset = memb->offset; + // FIXME: is it even worth calling PyType_Modified() here? + changed = 1; + } +#if CYTHON_METH_FASTCALL + else if (strcmp(memb->name, "__vectorcalloffset__") == 0) { + // The PyMemberDef must be a Py_ssize_t and readonly. + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); +#if PY_VERSION_HEX >= 0x030800b4 + type->tp_vectorcall_offset = memb->offset; +#else + type->tp_print = (printfunc) memb->offset; +#endif + // FIXME: is it even worth calling PyType_Modified() here? + changed = 1; + } +#endif +#else + if ((0)); +#endif +#if PY_VERSION_HEX <= 0x030900b1 && CYTHON_COMPILING_IN_CPYTHON + else if (strcmp(memb->name, "__module__") == 0) { + // PyType_FromSpec() in CPython <= 3.9b1 overwrites this field with a constant string. + // See https://bugs.python.org/issue40703 + PyObject *descr; + // The PyMemberDef must be an object and normally readable, possibly writable. + assert(memb->type == T_OBJECT); + assert(memb->flags == 0 || memb->flags == READONLY); + descr = PyDescr_NewMember(type, memb); + if (unlikely(!descr)) + return -1; + if (unlikely(PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr) < 0)) { + Py_DECREF(descr); + return -1; + } + Py_DECREF(descr); + changed = 1; + } +#endif + } + memb++; + } + if (changed) + PyType_Modified(type); + } +#endif + return 0; +} +#endif + + +/////////////// ValidateBasesTuple.proto /////////////// + +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS +static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases); /*proto*/ +#endif + +/////////////// ValidateBasesTuple /////////////// + +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_USE_TYPE_SPECS +static int __Pyx_validate_bases_tuple(const char *type_name, Py_ssize_t dictoffset, PyObject *bases) { // Loop over all bases (except the first) and check that those // really are heap types. Otherwise, it would not be safe to // subclass them. @@ -17,52 +112,97 @@ static int __Pyx_PyType_Ready(PyTypeObject *t) { // tp_dictoffset (i.e. there is no __dict__ attribute in the object // structure), we need to check that none of the base classes sets // it either. - int r; - PyObject *bases = t->tp_bases; - if (bases) + Py_ssize_t i, n = PyTuple_GET_SIZE(bases); + for (i = 1; i < n; i++) /* Skip first base */ { - Py_ssize_t i, n = PyTuple_GET_SIZE(bases); - for (i = 1; i < n; i++) /* Skip first base */ - { - PyObject *b0 = PyTuple_GET_ITEM(bases, i); - PyTypeObject *b; + PyObject *b0 = PyTuple_GET_ITEM(bases, i); + PyTypeObject *b; #if PY_MAJOR_VERSION < 3 - /* Disallow old-style classes */ - if (PyClass_Check(b0)) - { - PyErr_Format(PyExc_TypeError, "base class '%.200s' is an old-style class", - PyString_AS_STRING(((PyClassObject*)b0)->cl_name)); - return -1; - } + /* Disallow old-style classes */ + if (PyClass_Check(b0)) + { + PyErr_Format(PyExc_TypeError, "base class '%.200s' is an old-style class", + PyString_AS_STRING(((PyClassObject*)b0)->cl_name)); + return -1; + } #endif - b = (PyTypeObject*)b0; - if (!PyType_HasFeature(b, Py_TPFLAGS_HEAPTYPE)) - { - PyErr_Format(PyExc_TypeError, "base class '%.200s' is not a heap type", - b->tp_name); - return -1; - } - if (t->tp_dictoffset == 0 && b->tp_dictoffset) - { - PyErr_Format(PyExc_TypeError, - "extension type '%.200s' has no __dict__ slot, but base type '%.200s' has: " - "either add 'cdef dict __dict__' to the extension type " - "or add '__slots__ = [...]' to the base type", - t->tp_name, b->tp_name); - return -1; - } + b = (PyTypeObject*) b0; + if (!__Pyx_PyType_HasFeature(b, Py_TPFLAGS_HEAPTYPE)) + { + __Pyx_TypeName b_name = __Pyx_PyType_GetName(b); + PyErr_Format(PyExc_TypeError, + "base class '" __Pyx_FMT_TYPENAME "' is not a heap type", b_name); + __Pyx_DECREF_TypeName(b_name); + return -1; + } + if (dictoffset == 0 && b->tp_dictoffset) + { + __Pyx_TypeName b_name = __Pyx_PyType_GetName(b); + PyErr_Format(PyExc_TypeError, + "extension type '%.200s' has no __dict__ slot, " + "but base type '" __Pyx_FMT_TYPENAME "' has: " + "either add 'cdef dict __dict__' to the extension type " + "or add '__slots__ = [...]' to the base type", + type_name, b_name); + __Pyx_DECREF_TypeName(b_name); + return -1; } } + return 0; +} +#endif + + +/////////////// PyType_Ready.proto /////////////// + +// unused when using type specs +CYTHON_UNUSED static int __Pyx_PyType_Ready(PyTypeObject *t);/*proto*/ + +/////////////// PyType_Ready /////////////// +//@requires: ObjectHandling.c::PyObjectCallMethod0 +//@requires: ValidateBasesTuple + +// Wrapper around PyType_Ready() with some runtime checks and fixes +// to deal with multiple inheritance. +static int __Pyx_PyType_Ready(PyTypeObject *t) { + +// FIXME: is this really suitable for CYTHON_COMPILING_IN_LIMITED_API? +#if CYTHON_USE_TYPE_SPECS || !(CYTHON_COMPILING_IN_CPYTHON || CYTHON_COMPILING_IN_LIMITED_API) || defined(PYSTON_MAJOR_VERSION) + // avoid C warning about unused helper function + (void)__Pyx_PyObject_CallMethod0; +#if CYTHON_USE_TYPE_SPECS + (void)__Pyx_validate_bases_tuple; +#endif + + return PyType_Ready(t); + +#else + int r; + PyObject *bases = __Pyx_PyType_GetSlot(t, tp_bases, PyObject*); + if (bases && unlikely(__Pyx_validate_bases_tuple(t->tp_name, t->tp_dictoffset, bases) == -1)) + return -1; #if PY_VERSION_HEX >= 0x03050000 && !defined(PYSTON_MAJOR_VERSION) { // Make sure GC does not pick up our non-heap type as heap type with this hack! // For details, see https://github.com/cython/cython/issues/3603 - PyObject *ret, *py_status; int gc_was_enabled; - PyObject *gc = PyImport_Import(PYUNICODE("gc")); + #if PY_VERSION_HEX >= 0x030A00b1 + // finally added in Py3.10 :) + gc_was_enabled = PyGC_Disable(); + (void)__Pyx_PyObject_CallMethod0; + + #else + // Call gc.disable() as a backwards compatible fallback, but only if needed. + PyObject *ret, *py_status; + PyObject *gc = NULL; + #if PY_VERSION_HEX >= 0x030700a1 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM+0 >= 0x07030400) + // https://foss.heptapod.net/pypy/pypy/-/issues/3385 + gc = PyImport_GetModule(PYUNICODE("gc")); + #endif + if (unlikely(!gc)) gc = PyImport_Import(PYUNICODE("gc")); if (unlikely(!gc)) return -1; - py_status = PyObject_CallMethodObjArgs(gc, PYUNICODE("isenabled"), NULL); + py_status = __Pyx_PyObject_CallMethod0(gc, PYUNICODE("isenabled")); if (unlikely(!py_status)) { Py_DECREF(gc); return -1; @@ -70,7 +210,7 @@ static int __Pyx_PyType_Ready(PyTypeObject *t) { gc_was_enabled = __Pyx_PyObject_IsTrue(py_status); Py_DECREF(py_status); if (gc_was_enabled > 0) { - ret = PyObject_CallMethodObjArgs(gc, PYUNICODE("disable"), NULL); + ret = __Pyx_PyObject_CallMethod0(gc, PYUNICODE("disable")); if (unlikely(!ret)) { Py_DECREF(gc); return -1; @@ -80,6 +220,7 @@ static int __Pyx_PyType_Ready(PyTypeObject *t) { Py_DECREF(gc); return -1; } + #endif // As of https://bugs.python.org/issue22079 // PyType_Ready enforces that all bases of a non-heap type are @@ -89,6 +230,18 @@ static int __Pyx_PyType_Ready(PyTypeObject *t) { // Other than this check, the Py_TPFLAGS_HEAPTYPE flag is unused // in PyType_Ready(). t->tp_flags |= Py_TPFLAGS_HEAPTYPE; +#if PY_VERSION_HEX >= 0x030A0000 + // As of https://github.com/python/cpython/pull/25520 + // PyType_Ready marks types as immutable if they are static types + // and requires the Py_TPFLAGS_IMMUTABLETYPE flag to mark types as + // immutable + // Manually set the Py_TPFLAGS_IMMUTABLETYPE flag, since the type + // is immutable + t->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; +#endif +#else + // avoid C warning about unused helper function + (void)__Pyx_PyObject_CallMethod0; #endif r = PyType_Ready(t); @@ -96,29 +249,84 @@ static int __Pyx_PyType_Ready(PyTypeObject *t) { #if PY_VERSION_HEX >= 0x03050000 && !defined(PYSTON_MAJOR_VERSION) t->tp_flags &= ~Py_TPFLAGS_HEAPTYPE; + #if PY_VERSION_HEX >= 0x030A00b1 + if (gc_was_enabled) + PyGC_Enable(); + #else if (gc_was_enabled) { - PyObject *t, *v, *tb; - PyErr_Fetch(&t, &v, &tb); - ret = PyObject_CallMethodObjArgs(gc, PYUNICODE("enable"), NULL); + PyObject *tp, *v, *tb; + PyErr_Fetch(&tp, &v, &tb); + ret = __Pyx_PyObject_CallMethod0(gc, PYUNICODE("enable")); if (likely(ret || r == -1)) { Py_XDECREF(ret); // do not overwrite exceptions raised by PyType_Ready() above - PyErr_Restore(t, v, tb); + PyErr_Restore(tp, v, tb); } else { // PyType_Ready() succeeded, but gc.enable() failed. - Py_XDECREF(t); + Py_XDECREF(tp); Py_XDECREF(v); Py_XDECREF(tb); r = -1; } } Py_DECREF(gc); + #endif } #endif return r; +#endif } + +/////////////// PyTrashcan.proto /////////////// + +// These macros are taken from https://github.com/python/cpython/pull/11841 +// Unlike the Py_TRASHCAN_SAFE_BEGIN/Py_TRASHCAN_SAFE_END macros, they +// allow dealing correctly with subclasses. + +// This requires CPython version >= 2.7.4 +// (or >= 3.2.4 but we don't support such old Python 3 versions anyway) +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03080000 +// https://github.com/python/cpython/pull/11841 merged so Cython reimplementation +// is no longer necessary +#define __Pyx_TRASHCAN_BEGIN Py_TRASHCAN_BEGIN +#define __Pyx_TRASHCAN_END Py_TRASHCAN_END +#elif CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x02070400 +#define __Pyx_TRASHCAN_BEGIN_CONDITION(op, cond) \ + do { \ + PyThreadState *_tstate = NULL; \ + // If "cond" is false, then _tstate remains NULL and the deallocator + // is run normally without involving the trashcan + if (cond) { \ + _tstate = PyThreadState_GET(); \ + if (_tstate->trash_delete_nesting >= PyTrash_UNWIND_LEVEL) { \ + // Store the object (to be deallocated later) and jump past + // Py_TRASHCAN_END, skipping the body of the deallocator + _PyTrash_thread_deposit_object((PyObject*)(op)); \ + break; \ + } \ + ++_tstate->trash_delete_nesting; \ + } + // The body of the deallocator is here. +#define __Pyx_TRASHCAN_END \ + if (_tstate) { \ + --_tstate->trash_delete_nesting; \ + if (_tstate->trash_delete_later && _tstate->trash_delete_nesting <= 0) \ + _PyTrash_thread_destroy_chain(); \ + } \ + } while (0); + +#define __Pyx_TRASHCAN_BEGIN(op, dealloc) __Pyx_TRASHCAN_BEGIN_CONDITION(op, \ + __Pyx_PyObject_GetSlot(op, tp_dealloc, destructor) == (destructor)(dealloc)) + +#else +// The trashcan is a no-op on other Python implementations +// or old CPython versions +#define __Pyx_TRASHCAN_BEGIN(op, dealloc) +#define __Pyx_TRASHCAN_END +#endif + /////////////// CallNextTpDealloc.proto /////////////// static void __Pyx_call_next_tp_dealloc(PyObject* obj, destructor current_tp_dealloc); @@ -127,13 +335,14 @@ static void __Pyx_call_next_tp_dealloc(PyObject* obj, destructor current_tp_deal static void __Pyx_call_next_tp_dealloc(PyObject* obj, destructor current_tp_dealloc) { PyTypeObject* type = Py_TYPE(obj); + destructor tp_dealloc = NULL; /* try to find the first parent type that has a different tp_dealloc() function */ - while (type && type->tp_dealloc != current_tp_dealloc) - type = type->tp_base; - while (type && type->tp_dealloc == current_tp_dealloc) - type = type->tp_base; + while (type && __Pyx_PyType_GetSlot(type, tp_dealloc, destructor) != current_tp_dealloc) + type = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + while (type && (tp_dealloc = __Pyx_PyType_GetSlot(type, tp_dealloc, destructor)) == current_tp_dealloc) + type = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); if (type) - type->tp_dealloc(obj); + tp_dealloc(obj); } /////////////// CallNextTpTraverse.proto /////////////// @@ -144,48 +353,53 @@ static int __Pyx_call_next_tp_traverse(PyObject* obj, visitproc v, void *a, trav static int __Pyx_call_next_tp_traverse(PyObject* obj, visitproc v, void *a, traverseproc current_tp_traverse) { PyTypeObject* type = Py_TYPE(obj); + traverseproc tp_traverse = NULL; /* try to find the first parent type that has a different tp_traverse() function */ - while (type && type->tp_traverse != current_tp_traverse) - type = type->tp_base; - while (type && type->tp_traverse == current_tp_traverse) - type = type->tp_base; - if (type && type->tp_traverse) - return type->tp_traverse(obj, v, a); + while (type && __Pyx_PyType_GetSlot(type, tp_traverse, traverseproc) != current_tp_traverse) + type = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + while (type && (tp_traverse = __Pyx_PyType_GetSlot(type, tp_traverse, traverseproc)) == current_tp_traverse) + type = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + if (type && tp_traverse) + return tp_traverse(obj, v, a); // FIXME: really ignore? return 0; } /////////////// CallNextTpClear.proto /////////////// -static void __Pyx_call_next_tp_clear(PyObject* obj, inquiry current_tp_dealloc); +static void __Pyx_call_next_tp_clear(PyObject* obj, inquiry current_tp_clear); /////////////// CallNextTpClear /////////////// static void __Pyx_call_next_tp_clear(PyObject* obj, inquiry current_tp_clear) { PyTypeObject* type = Py_TYPE(obj); + inquiry tp_clear = NULL; /* try to find the first parent type that has a different tp_clear() function */ - while (type && type->tp_clear != current_tp_clear) - type = type->tp_base; - while (type && type->tp_clear == current_tp_clear) - type = type->tp_base; - if (type && type->tp_clear) - type->tp_clear(obj); + while (type && __Pyx_PyType_GetSlot(type, tp_clear, inquiry) != current_tp_clear) + type = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + while (type && (tp_clear = __Pyx_PyType_GetSlot(type, tp_clear, inquiry)) == current_tp_clear) + type = __Pyx_PyType_GetSlot(type, tp_base, PyTypeObject*); + if (type && tp_clear) + tp_clear(obj); } /////////////// SetupReduce.proto /////////////// +#if !CYTHON_COMPILING_IN_LIMITED_API static int __Pyx_setup_reduce(PyObject* type_obj); +#endif /////////////// SetupReduce /////////////// //@requires: ObjectHandling.c::PyObjectGetAttrStrNoError //@requires: ObjectHandling.c::PyObjectGetAttrStr //@substitute: naming +#if !CYTHON_COMPILING_IN_LIMITED_API static int __Pyx_setup_reduce_is_named(PyObject* meth, PyObject* name) { int ret; PyObject *name_attr; - name_attr = __Pyx_PyObject_GetAttrStr(meth, PYIDENT("__name__")); + name_attr = __Pyx_PyObject_GetAttrStrNoError(meth, PYIDENT("__name__")); if (likely(name_attr)) { ret = PyObject_RichCompareBool(name_attr, name, Py_EQ); } else { @@ -263,7 +477,7 @@ static int __Pyx_setup_reduce(PyObject* type_obj) { goto __PYX_BAD; } - setstate = __Pyx_PyObject_GetAttrStr(type_obj, PYIDENT("__setstate__")); + setstate = __Pyx_PyObject_GetAttrStrNoError(type_obj, PYIDENT("__setstate__")); if (!setstate) PyErr_Clear(); if (!setstate || __Pyx_setup_reduce_is_named(setstate, PYIDENT("__setstate_cython__"))) { setstate_cython = __Pyx_PyObject_GetAttrStrNoError(type_obj, PYIDENT("__setstate_cython__")); @@ -282,8 +496,13 @@ static int __Pyx_setup_reduce(PyObject* type_obj) { goto __PYX_GOOD; __PYX_BAD: - if (!PyErr_Occurred()) - PyErr_Format(PyExc_RuntimeError, "Unable to initialize pickling for %s", ((PyTypeObject*)type_obj)->tp_name); + if (!PyErr_Occurred()) { + __Pyx_TypeName type_obj_name = + __Pyx_PyType_GetName((PyTypeObject*)type_obj); + PyErr_Format(PyExc_RuntimeError, + "Unable to initialize pickling for " __Pyx_FMT_TYPENAME, type_obj_name); + __Pyx_DECREF_TypeName(type_obj_name); + } ret = -1; __PYX_GOOD: #if !CYTHON_USE_PYTYPE_LOOKUP @@ -299,3 +518,92 @@ __PYX_GOOD: Py_XDECREF(setstate_cython); return ret; } +#endif + + +/////////////// BinopSlot /////////////// + +static CYTHON_INLINE PyObject *{{func_name}}_maybe_call_slot(PyTypeObject* type, PyObject *left, PyObject *right {{extra_arg_decl}}) { + {{slot_type}} slot; +#if CYTHON_USE_TYPE_SLOTS || PY_MAJOR_VERSION < 3 || CYTHON_COMPILING_IN_PYPY + slot = type->tp_as_number ? type->tp_as_number->{{slot_name}} : NULL; +#else + slot = ({{slot_type}}) PyType_GetSlot(type, Py_{{slot_name}}); +#endif + return slot ? slot(left, right {{extra_arg}}) : __Pyx_NewRef(Py_NotImplemented); +} + +static PyObject *{{func_name}}(PyObject *left, PyObject *right {{extra_arg_decl}}) { + int maybe_self_is_left, maybe_self_is_right = 0; + maybe_self_is_left = Py_TYPE(left) == Py_TYPE(right) +#if CYTHON_USE_TYPE_SLOTS + || (Py_TYPE(left)->tp_as_number && Py_TYPE(left)->tp_as_number->{{slot_name}} == &{{func_name}}) +#endif + || __Pyx_TypeCheck(left, {{type_cname}}); + // Optimize for the common case where the left operation is defined (and successful). + if (!({{overloads_left}})) { + maybe_self_is_right = Py_TYPE(left) == Py_TYPE(right) +#if CYTHON_USE_TYPE_SLOTS + || (Py_TYPE(right)->tp_as_number && Py_TYPE(right)->tp_as_number->{{slot_name}} == &{{func_name}}) +#endif + || __Pyx_TypeCheck(right, {{type_cname}}); + } + if (maybe_self_is_left) { + PyObject *res; + if (maybe_self_is_right && {{overloads_right}} && !({{overloads_left}})) { + res = {{call_right}}; + if (res != Py_NotImplemented) return res; + Py_DECREF(res); + // Don't bother calling it again. + maybe_self_is_right = 0; + } + res = {{call_left}}; + if (res != Py_NotImplemented) return res; + Py_DECREF(res); + } + if (({{overloads_left}})) { + maybe_self_is_right = Py_TYPE(left) == Py_TYPE(right) +#if CYTHON_USE_TYPE_SLOTS + || (Py_TYPE(right)->tp_as_number && Py_TYPE(right)->tp_as_number->{{slot_name}} == &{{func_name}}) +#endif + || PyType_IsSubtype(Py_TYPE(right), {{type_cname}}); + } + if (maybe_self_is_right) { + return {{call_right}}; + } + return __Pyx_NewRef(Py_NotImplemented); +} + +/////////////// ValidateExternBase.proto /////////////// + +static int __Pyx_validate_extern_base(PyTypeObject *base); /* proto */ + +/////////////// ValidateExternBase /////////////// +//@requires: ObjectHandling.c::FormatTypeName + +static int __Pyx_validate_extern_base(PyTypeObject *base) { + Py_ssize_t itemsize; +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_itemsize; +#endif +#if !CYTHON_COMPILING_IN_LIMITED_API + itemsize = ((PyTypeObject *)base)->tp_itemsize; +#else + py_itemsize = PyObject_GetAttrString((PyObject*)base, "__itemsize__"); + if (!py_itemsize) + return -1; + itemsize = PyLong_AsSsize_t(py_itemsize); + Py_DECREF(py_itemsize); + py_itemsize = 0; + if (itemsize == (Py_ssize_t)-1 && PyErr_Occurred()) + return -1; +#endif + if (itemsize) { + __Pyx_TypeName b_name = __Pyx_PyType_GetName(base); + PyErr_Format(PyExc_TypeError, + "inheritance from PyVarObject types like '" __Pyx_FMT_TYPENAME "' not currently supported", b_name); + __Pyx_DECREF_TypeName(b_name); + return -1; + } + return 0; +} diff --git a/Cython/Utility/FunctionArguments.c b/Cython/Utility/FunctionArguments.c index 8333d9366..8bdaee562 100644 --- a/Cython/Utility/FunctionArguments.c +++ b/Cython/Utility/FunctionArguments.c @@ -2,7 +2,7 @@ #define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact) \ - ((likely((Py_TYPE(obj) == type) | (none_allowed && (obj == Py_None)))) ? 1 : \ + ((likely(__Pyx_IS_TYPE(obj, type) | (none_allowed && (obj == Py_None)))) ? 1 : \ __Pyx__ArgTypeTest(obj, type, name, exact)) static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); /*proto*/ @@ -11,6 +11,8 @@ static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *nam static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) { + __Pyx_TypeName type_name; + __Pyx_TypeName obj_type_name; if (unlikely(!type)) { PyErr_SetString(PyExc_SystemError, "Missing type object"); return 0; @@ -23,9 +25,13 @@ static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *nam else { if (likely(__Pyx_TypeCheck(obj, type))) return 1; } + type_name = __Pyx_PyType_GetName(type); + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); PyErr_Format(PyExc_TypeError, - "Argument '%.200s' has incorrect type (expected %.200s, got %.200s)", - name, type->tp_name, Py_TYPE(obj)->tp_name); + "Argument '%.200s' has incorrect type (expected " __Pyx_FMT_TYPENAME + ", got " __Pyx_FMT_TYPENAME ")", name, type_name, obj_type_name); + __Pyx_DECREF_TypeName(type_name); + __Pyx_DECREF_TypeName(obj_type_name); return 0; } @@ -111,22 +117,28 @@ static void __Pyx_RaiseMappingExpectedError(PyObject* arg); /*proto*/ //////////////////// RaiseMappingExpected //////////////////// static void __Pyx_RaiseMappingExpectedError(PyObject* arg) { - PyErr_Format(PyExc_TypeError, "'%.200s' object is not a mapping", Py_TYPE(arg)->tp_name); + __Pyx_TypeName arg_type_name = __Pyx_PyType_GetName(Py_TYPE(arg)); + PyErr_Format(PyExc_TypeError, + "'" __Pyx_FMT_TYPENAME "' object is not a mapping", arg_type_name); + __Pyx_DECREF_TypeName(arg_type_name); } //////////////////// KeywordStringCheck.proto //////////////////// -static int __Pyx_CheckKeywordStrings(PyObject *kwdict, const char* function_name, int kw_allowed); /*proto*/ +static int __Pyx_CheckKeywordStrings(PyObject *kw, const char* function_name, int kw_allowed); /*proto*/ //////////////////// KeywordStringCheck //////////////////// -// __Pyx_CheckKeywordStrings raises an error if non-string keywords -// were passed to a function, or if any keywords were passed to a -// function that does not accept them. +// __Pyx_CheckKeywordStrings raises an error if non-string keywords +// were passed to a function, or if any keywords were passed to a +// function that does not accept them. +// +// The "kw" argument is either a dict (for METH_VARARGS) or a tuple +// (for METH_FASTCALL). static int __Pyx_CheckKeywordStrings( - PyObject *kwdict, + PyObject *kw, const char* function_name, int kw_allowed) { @@ -134,18 +146,37 @@ static int __Pyx_CheckKeywordStrings( Py_ssize_t pos = 0; #if CYTHON_COMPILING_IN_PYPY /* PyPy appears to check keywords at call time, not at unpacking time => not much to do here */ - if (!kw_allowed && PyDict_Next(kwdict, &pos, &key, 0)) + if (!kw_allowed && PyDict_Next(kw, &pos, &key, 0)) goto invalid_keyword; return 1; #else - while (PyDict_Next(kwdict, &pos, &key, 0)) { + if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kw))) { + if (unlikely(PyTuple_GET_SIZE(kw) == 0)) + return 1; + if (!kw_allowed) { + key = PyTuple_GET_ITEM(kw, 0); + goto invalid_keyword; + } +#if PY_VERSION_HEX < 0x03090000 + // On CPython >= 3.9, the FASTCALL protocol guarantees that keyword + // names are strings (see https://bugs.python.org/issue37540) + for (pos = 0; pos < PyTuple_GET_SIZE(kw); pos++) { + key = PyTuple_GET_ITEM(kw, pos); + if (unlikely(!PyUnicode_Check(key))) + goto invalid_keyword_type; + } +#endif + return 1; + } + + while (PyDict_Next(kw, &pos, &key, 0)) { #if PY_MAJOR_VERSION < 3 if (unlikely(!PyString_Check(key))) #endif if (unlikely(!PyUnicode_Check(key))) goto invalid_keyword_type; } - if ((!kw_allowed) && unlikely(key)) + if (!kw_allowed && unlikely(key)) goto invalid_keyword; return 1; invalid_keyword_type: @@ -154,11 +185,12 @@ invalid_keyword_type: return 0; #endif invalid_keyword: - PyErr_Format(PyExc_TypeError, #if PY_MAJOR_VERSION < 3 + PyErr_Format(PyExc_TypeError, "%.200s() got an unexpected keyword argument '%.200s'", function_name, PyString_AsString(key)); #else + PyErr_Format(PyExc_TypeError, "%s() got an unexpected keyword argument '%U'", function_name, key); #endif @@ -168,17 +200,22 @@ invalid_keyword: //////////////////// ParseKeywords.proto //////////////////// -static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], \ - PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, \ +static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject *const *kwvalues, + PyObject **argnames[], + PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args, const char* function_name); /*proto*/ //////////////////// ParseKeywords //////////////////// //@requires: RaiseDoubleKeywords // __Pyx_ParseOptionalKeywords copies the optional/unknown keyword -// arguments from the kwds dict into kwds2. If kwds2 is NULL, unknown +// arguments from kwds into the dict kwds2. If kwds2 is NULL, unknown // keywords will raise an invalid keyword error. // +// When not using METH_FASTCALL, kwds is a dict and kwvalues is NULL. +// Otherwise, kwds is a tuple with keyword names and kwvalues is a C +// array with the corresponding values. +// // Three kinds of errors are checked: 1) non-string keywords, 2) // unexpected keywords and 3) overlap with positional arguments. // @@ -190,6 +227,7 @@ static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[], \ static int __Pyx_ParseOptionalKeywords( PyObject *kwds, + PyObject *const *kwvalues, PyObject **argnames[], PyObject *kwds2, PyObject *values[], @@ -200,8 +238,20 @@ static int __Pyx_ParseOptionalKeywords( Py_ssize_t pos = 0; PyObject*** name; PyObject*** first_kw_arg = argnames + num_pos_args; + int kwds_is_tuple = CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds)); + + while (1) { + if (kwds_is_tuple) { + if (pos >= PyTuple_GET_SIZE(kwds)) break; + key = PyTuple_GET_ITEM(kwds, pos); + value = kwvalues[pos]; + pos++; + } + else + { + if (!PyDict_Next(kwds, &pos, &key, &value)) break; + } - while (PyDict_Next(kwds, &pos, &key, &value)) { name = first_kw_arg; while (*name && (**name != key)) name++; if (*name) { @@ -237,12 +287,13 @@ static int __Pyx_ParseOptionalKeywords( #endif if (likely(PyUnicode_Check(key))) { while (*name) { - int cmp = (**name == key) ? 0 : + int cmp = ( #if !CYTHON_COMPILING_IN_PYPY && PY_MAJOR_VERSION >= 3 (__Pyx_PyUnicode_GET_LENGTH(**name) != __Pyx_PyUnicode_GET_LENGTH(key)) ? 1 : #endif // In Py2, we may need to convert the argument name from str to unicode for comparison. - PyUnicode_Compare(**name, key); + PyUnicode_Compare(**name, key) + ); if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; if (cmp == 0) { values[name-argnames] = value; @@ -284,11 +335,12 @@ invalid_keyword_type: "%.200s() keywords must be strings", function_name); goto bad; invalid_keyword: - PyErr_Format(PyExc_TypeError, #if PY_MAJOR_VERSION < 3 + PyErr_Format(PyExc_TypeError, "%.200s() got an unexpected keyword argument '%.200s'", function_name, PyString_AsString(key)); #else + PyErr_Format(PyExc_TypeError, "%s() got an unexpected keyword argument '%U'", function_name, key); #endif @@ -314,7 +366,7 @@ static int __Pyx_MergeKeywords(PyObject *kwdict, PyObject *source_mapping) { if (unlikely(!iter)) { // slow fallback: try converting to dict, then iterate PyObject *args; - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) goto bad; + if (unlikely(!PyErr_ExceptionMatches(PyExc_AttributeError))) goto bad; PyErr_Clear(); args = PyTuple_Pack(1, source_mapping); if (likely(args)) { @@ -350,3 +402,74 @@ bad: Py_XDECREF(iter); return -1; } + + +/////////////// fastcall.proto /////////////// + +// We define various functions and macros with two variants: +//..._FASTCALL and ..._VARARGS + +// The first is used when METH_FASTCALL is enabled and the second is used +// otherwise. If the Python implementation does not support METH_FASTCALL +// (because it's an old version of CPython or it's not CPython at all), +// then the ..._FASTCALL macros simply alias ..._VARARGS + +#define __Pyx_Arg_VARARGS(args, i) PyTuple_GET_ITEM(args, i) +#define __Pyx_NumKwargs_VARARGS(kwds) PyDict_Size(kwds) +#define __Pyx_KwValues_VARARGS(args, nargs) NULL +#define __Pyx_GetKwValue_VARARGS(kw, kwvalues, s) __Pyx_PyDict_GetItemStrWithError(kw, s) +#define __Pyx_KwargsAsDict_VARARGS(kw, kwvalues) PyDict_Copy(kw) +#if CYTHON_METH_FASTCALL + #define __Pyx_Arg_FASTCALL(args, i) args[i] + #define __Pyx_NumKwargs_FASTCALL(kwds) PyTuple_GET_SIZE(kwds) + #define __Pyx_KwValues_FASTCALL(args, nargs) ((args) + (nargs)) + static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s); + #define __Pyx_KwargsAsDict_FASTCALL(kw, kwvalues) _PyStack_AsDict(kwvalues, kw) +#else + #define __Pyx_Arg_FASTCALL __Pyx_Arg_VARARGS + #define __Pyx_NumKwargs_FASTCALL __Pyx_NumKwargs_VARARGS + #define __Pyx_KwValues_FASTCALL __Pyx_KwValues_VARARGS + #define __Pyx_GetKwValue_FASTCALL __Pyx_GetKwValue_VARARGS + #define __Pyx_KwargsAsDict_FASTCALL __Pyx_KwargsAsDict_VARARGS +#endif + +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_ArgsSlice_VARARGS(args, start, stop) __Pyx_PyTuple_FromArray(&__Pyx_Arg_VARARGS(args, start), stop - start) +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) __Pyx_PyTuple_FromArray(&__Pyx_Arg_FASTCALL(args, start), stop - start) +#else +/* Not CPython, so certainly no METH_FASTCALL support */ +#define __Pyx_ArgsSlice_VARARGS(args, start, stop) PyTuple_GetSlice(args, start, stop) +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) PyTuple_GetSlice(args, start, stop) +#endif + + +/////////////// fastcall /////////////// +//@requires: ObjectHandling.c::TupleAndListFromArray +//@requires: StringTools.c::UnicodeEquals + +#if CYTHON_METH_FASTCALL +// kwnames: tuple with names of keyword arguments +// kwvalues: C array with values of keyword arguments +// s: str with the keyword name to look for +static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s) +{ + // Search the kwnames array for s and return the corresponding value. + // We do two loops: a first one to compare pointers (which will find a + // match if the name in kwnames is interned, given that s is interned + // by Cython). A second loop compares the actual strings. + Py_ssize_t i, n = PyTuple_GET_SIZE(kwnames); + for (i = 0; i < n; i++) + { + if (s == PyTuple_GET_ITEM(kwnames, i)) return kwvalues[i]; + } + for (i = 0; i < n; i++) + { + int eq = __Pyx_PyUnicode_Equals(s, PyTuple_GET_ITEM(kwnames, i), Py_EQ); + if (unlikely(eq != 0)) { + if (unlikely(eq < 0)) return NULL; // error + return kwvalues[i]; + } + } + return NULL; // not found (no exception set) +} +#endif diff --git a/Cython/Utility/ImportExport.c b/Cython/Utility/ImportExport.c index d6f06ecd7..007f7b43a 100644 --- a/Cython/Utility/ImportExport.c +++ b/Cython/Utility/ImportExport.c @@ -1,12 +1,190 @@ -/////////////// PyIdentifierFromString.proto /////////////// +/////////////// ImportDottedModule.proto /////////////// -#if !defined(__Pyx_PyIdentifier_FromString) +static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple); /*proto*/ +#if PY_MAJOR_VERSION >= 3 +static PyObject *__Pyx_ImportDottedModule_WalkParts(PyObject *module, PyObject *name, PyObject *parts_tuple); /*proto*/ +#endif + +/////////////// ImportDottedModule /////////////// +//@requires: Import + +#if PY_MAJOR_VERSION >= 3 +static PyObject *__Pyx__ImportDottedModule_Error(PyObject *name, PyObject *parts_tuple, Py_ssize_t count) { + PyObject *partial_name = NULL, *slice = NULL, *sep = NULL; + if (unlikely(PyErr_Occurred())) { + PyErr_Clear(); + } + if (likely(PyTuple_GET_SIZE(parts_tuple) == count)) { + partial_name = name; + } else { + slice = PySequence_GetSlice(parts_tuple, 0, count); + if (unlikely(!slice)) + goto bad; + sep = PyUnicode_FromStringAndSize(".", 1); + if (unlikely(!sep)) + goto bad; + partial_name = PyUnicode_Join(sep, slice); + } + + PyErr_Format( +#if PY_MAJOR_VERSION < 3 + PyExc_ImportError, + "No module named '%s'", PyString_AS_STRING(partial_name)); +#else +#if PY_VERSION_HEX >= 0x030600B1 + PyExc_ModuleNotFoundError, +#else + PyExc_ImportError, +#endif + "No module named '%U'", partial_name); +#endif + +bad: + Py_XDECREF(sep); + Py_XDECREF(slice); + Py_XDECREF(partial_name); + return NULL; +} +#endif + +#if PY_MAJOR_VERSION >= 3 +static PyObject *__Pyx__ImportDottedModule_Lookup(PyObject *name) { + PyObject *imported_module; +#if PY_VERSION_HEX < 0x030700A1 || (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) + PyObject *modules = PyImport_GetModuleDict(); + if (unlikely(!modules)) + return NULL; + imported_module = __Pyx_PyDict_GetItemStr(modules, name); + Py_XINCREF(imported_module); +#else + imported_module = PyImport_GetModule(name); +#endif + return imported_module; +} +#endif + +#if PY_MAJOR_VERSION >= 3 +static PyObject *__Pyx_ImportDottedModule_WalkParts(PyObject *module, PyObject *name, PyObject *parts_tuple) { + Py_ssize_t i, nparts; + nparts = PyTuple_GET_SIZE(parts_tuple); + for (i=1; i < nparts && module; i++) { + PyObject *part, *submodule; +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + part = PyTuple_GET_ITEM(parts_tuple, i); +#else + part = PySequence_ITEM(parts_tuple, i); +#endif + submodule = __Pyx_PyObject_GetAttrStrNoError(module, part); + // We stop if the attribute isn't found, i.e. if submodule is NULL here. +#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(part); +#endif + Py_DECREF(module); + module = submodule; + } + if (unlikely(!module)) { + return __Pyx__ImportDottedModule_Error(name, parts_tuple, i); + } + return module; +} +#endif + +static PyObject *__Pyx__ImportDottedModule(PyObject *name, PyObject *parts_tuple) { #if PY_MAJOR_VERSION < 3 - #define __Pyx_PyIdentifier_FromString(s) PyString_FromString(s) + PyObject *module, *from_list, *star = PYIDENT("*"); + CYTHON_UNUSED_VAR(parts_tuple); + from_list = PyList_New(1); + if (unlikely(!from_list)) + return NULL; + Py_INCREF(star); + PyList_SET_ITEM(from_list, 0, star); + module = __Pyx_Import(name, from_list, 0); + Py_DECREF(from_list); + return module; #else - #define __Pyx_PyIdentifier_FromString(s) PyUnicode_FromString(s) + PyObject *imported_module; + PyObject *module = __Pyx_Import(name, NULL, 0); + if (!parts_tuple || unlikely(!module)) + return module; + + // Look up module in sys.modules, which is safer than the attribute lookups below. + imported_module = __Pyx__ImportDottedModule_Lookup(name); + if (likely(imported_module)) { + Py_DECREF(module); + return imported_module; + } + PyErr_Clear(); + return __Pyx_ImportDottedModule_WalkParts(module, name, parts_tuple); #endif +} + +static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030400B1 + PyObject *module = __Pyx__ImportDottedModule_Lookup(name); + if (likely(module)) { + // CPython guards against thread-concurrent initialisation in importlib. + // In this case, we let PyImport_ImportModuleLevelObject() handle the locking. + PyObject *spec = __Pyx_PyObject_GetAttrStrNoError(module, PYIDENT("__spec__")); + if (likely(spec)) { + PyObject *unsafe = __Pyx_PyObject_GetAttrStrNoError(spec, PYIDENT("_initializing")); + if (likely(!unsafe || !__Pyx_PyObject_IsTrue(unsafe))) { + Py_DECREF(spec); + spec = NULL; + } + Py_XDECREF(unsafe); + } + if (likely(!spec)) { + // Not in initialisation phase => use modules as is. + PyErr_Clear(); + return module; + } + Py_DECREF(spec); + Py_DECREF(module); + } else if (PyErr_Occurred()) { + PyErr_Clear(); + } +#endif + + return __Pyx__ImportDottedModule(name, parts_tuple); +} + + +/////////////// ImportDottedModuleRelFirst.proto /////////////// + +static PyObject *__Pyx_ImportDottedModuleRelFirst(PyObject *name, PyObject *parts_tuple); /*proto*/ + +/////////////// ImportDottedModuleRelFirst /////////////// +//@requires: ImportDottedModule +//@requires: Import + +static PyObject *__Pyx_ImportDottedModuleRelFirst(PyObject *name, PyObject *parts_tuple) { + PyObject *module; + PyObject *from_list = NULL; +#if PY_MAJOR_VERSION < 3 + PyObject *star = PYIDENT("*"); + from_list = PyList_New(1); + if (unlikely(!from_list)) + return NULL; + Py_INCREF(star); + PyList_SET_ITEM(from_list, 0, star); #endif + module = __Pyx_Import(name, from_list, -1); + Py_XDECREF(from_list); + if (module) { + #if PY_MAJOR_VERSION >= 3 + if (parts_tuple) { + module = __Pyx_ImportDottedModule_WalkParts(module, name, parts_tuple); + } + #endif + return module; + } + if (unlikely(!PyErr_ExceptionMatches(PyExc_ImportError))) + return NULL; + PyErr_Clear(); + // try absolute import + return __Pyx_ImportDottedModule(name, parts_tuple); +} + /////////////// Import.proto /////////////// @@ -17,30 +195,23 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); / //@substitute: naming static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { - PyObject *empty_list = 0; PyObject *module = 0; - PyObject *global_dict = 0; PyObject *empty_dict = 0; - PyObject *list; + PyObject *empty_list = 0; #if PY_MAJOR_VERSION < 3 PyObject *py_import; py_import = __Pyx_PyObject_GetAttrStr($builtins_cname, PYIDENT("__import__")); - if (!py_import) + if (unlikely(!py_import)) goto bad; - #endif - if (from_list) - list = from_list; - else { + if (!from_list) { empty_list = PyList_New(0); - if (!empty_list) + if (unlikely(!empty_list)) goto bad; - list = empty_list; + from_list = empty_list; } - global_dict = PyModule_GetDict($module_cname); - if (!global_dict) - goto bad; + #endif empty_dict = PyDict_New(); - if (!empty_dict) + if (unlikely(!empty_dict)) goto bad; { #if PY_MAJOR_VERSION >= 3 @@ -48,10 +219,15 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { // Avoid C compiler warning if strchr() evaluates to false at compile time. if ((1) && (strchr(__Pyx_MODULE_NAME, '.'))) { /* try package relative import first */ + #if CYTHON_COMPILING_IN_LIMITED_API module = PyImport_ImportModuleLevelObject( - name, global_dict, empty_dict, list, 1); - if (!module) { - if (!PyErr_ExceptionMatches(PyExc_ImportError)) + name, empty_dict, empty_dict, from_list, 1); + #else + module = PyImport_ImportModuleLevelObject( + name, $moddict_cname, empty_dict, from_list, 1); + #endif + if (unlikely(!module)) { + if (unlikely(!PyErr_ExceptionMatches(PyExc_ImportError))) goto bad; PyErr_Clear(); } @@ -62,23 +238,28 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { if (!module) { #if PY_MAJOR_VERSION < 3 PyObject *py_level = PyInt_FromLong(level); - if (!py_level) + if (unlikely(!py_level)) goto bad; module = PyObject_CallFunctionObjArgs(py_import, - name, global_dict, empty_dict, list, py_level, (PyObject *)NULL); + name, $moddict_cname, empty_dict, from_list, py_level, (PyObject *)NULL); Py_DECREF(py_level); #else + #if CYTHON_COMPILING_IN_LIMITED_API + module = PyImport_ImportModuleLevelObject( + name, empty_dict, empty_dict, from_list, level); + #else module = PyImport_ImportModuleLevelObject( - name, global_dict, empty_dict, list, level); + name, $moddict_cname, empty_dict, from_list, level); + #endif #endif } } bad: + Py_XDECREF(empty_dict); + Py_XDECREF(empty_list); #if PY_MAJOR_VERSION < 3 Py_XDECREF(py_import); #endif - Py_XDECREF(empty_list); - Py_XDECREF(empty_dict); return module; } @@ -93,6 +274,39 @@ static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); /*proto*/ static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { PyObject* value = __Pyx_PyObject_GetAttrStr(module, name); if (unlikely(!value) && PyErr_ExceptionMatches(PyExc_AttributeError)) { + // 'name' may refer to a (sub-)module which has not finished initialization + // yet, and may not be assigned as an attribute to its parent, so try + // finding it by full name. + const char* module_name_str = 0; + PyObject* module_name = 0; + PyObject* module_dot = 0; + PyObject* full_name = 0; + PyErr_Clear(); + module_name_str = PyModule_GetName(module); + if (unlikely(!module_name_str)) { goto modbad; } + module_name = PyUnicode_FromString(module_name_str); + if (unlikely(!module_name)) { goto modbad; } + module_dot = PyUnicode_Concat(module_name, PYUNICODE(".")); + if (unlikely(!module_dot)) { goto modbad; } + full_name = PyUnicode_Concat(module_dot, name); + if (unlikely(!full_name)) { goto modbad; } + #if PY_VERSION_HEX < 0x030700A1 || (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) + { + PyObject *modules = PyImport_GetModuleDict(); + if (unlikely(!modules)) + goto modbad; + value = PyObject_GetItem(modules, full_name); + } + #else + value = PyImport_GetModule(full_name); + #endif + + modbad: + Py_XDECREF(full_name); + Py_XDECREF(module_dot); + Py_XDECREF(module_name); + } + if (unlikely(!value)) { PyErr_Format(PyExc_ImportError, #if PY_MAJOR_VERSION < 3 "cannot import name %.230s", PyString_AS_STRING(name)); @@ -334,7 +548,7 @@ static PyTypeObject *__Pyx_ImportType_$cyversion(PyObject *module, const char *m char warning[200]; Py_ssize_t basicsize; Py_ssize_t itemsize; -#ifdef Py_LIMITED_API +#if CYTHON_COMPILING_IN_LIMITED_API PyObject *py_basicsize; PyObject *py_itemsize; #endif @@ -348,7 +562,7 @@ static PyTypeObject *__Pyx_ImportType_$cyversion(PyObject *module, const char *m module_name, class_name); goto bad; } -#ifndef Py_LIMITED_API +#if !CYTHON_COMPILING_IN_LIMITED_API basicsize = ((PyTypeObject *)result)->tp_basicsize; itemsize = ((PyTypeObject *)result)->tp_itemsize; #else @@ -385,14 +599,17 @@ static PyTypeObject *__Pyx_ImportType_$cyversion(PyObject *module, const char *m PyErr_Format(PyExc_ValueError, "%.200s.%.200s size changed, may indicate binary incompatibility. " "Expected %zd from C header, got %zd from PyObject", - module_name, class_name, size, basicsize); + module_name, class_name, size, basicsize+itemsize); goto bad; } - if (check_size == __Pyx_ImportType_CheckSize_Error_$cyversion && (size_t)basicsize != size) { + // varobjects almost have structs between basicsize and basicsize + itemsize + // but the struct isn't always one of the two limiting values + if (check_size == __Pyx_ImportType_CheckSize_Error_$cyversion && + ((size_t)basicsize > size || (size_t)(basicsize + itemsize) < size)) { PyErr_Format(PyExc_ValueError, "%.200s.%.200s size changed, may indicate binary incompatibility. " - "Expected %zd from C header, got %zd from PyObject", - module_name, class_name, size, basicsize); + "Expected %zd from C header, got %zd-%zd from PyObject", + module_name, class_name, size, basicsize, basicsize+itemsize); goto bad; } else if (check_size == __Pyx_ImportType_CheckSize_Warn_$cyversion && (size_t)basicsize > size) { @@ -438,7 +655,6 @@ static int __Pyx_ImportFunction_$cyversion(PyObject *module, const char *funcnam PyModule_GetName(module), funcname); goto bad; } -#if PY_VERSION_HEX >= 0x02070000 if (!PyCapsule_IsValid(cobj, sig)) { PyErr_Format(PyExc_TypeError, "C function %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", @@ -446,21 +662,6 @@ static int __Pyx_ImportFunction_$cyversion(PyObject *module, const char *funcnam goto bad; } tmp.p = PyCapsule_GetPointer(cobj, sig); -#else - {const char *desc, *s1, *s2; - desc = (const char *)PyCObject_GetDesc(cobj); - if (!desc) - goto bad; - s1 = desc; s2 = sig; - while (*s1 != '\0' && *s1 == *s2) { s1++; s2++; } - if (*s1 != *s2) { - PyErr_Format(PyExc_TypeError, - "C function %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", - PyModule_GetName(module), funcname, sig, desc); - goto bad; - } - tmp.p = PyCObject_AsVoidPtr(cobj);} -#endif *f = tmp.fp; if (!(*f)) goto bad; @@ -498,11 +699,7 @@ static int __Pyx_ExportFunction(const char *name, void (*f)(void), const char *s goto bad; } tmp.fp = f; -#if PY_VERSION_HEX >= 0x02070000 cobj = PyCapsule_New(tmp.p, sig, 0); -#else - cobj = PyCObject_FromVoidPtrAndDesc(tmp.p, (void *)sig, 0); -#endif if (!cobj) goto bad; if (PyDict_SetItemString(d, name, cobj) < 0) @@ -540,7 +737,6 @@ static int __Pyx_ImportVoidPtr_$cyversion(PyObject *module, const char *name, vo PyModule_GetName(module), name); goto bad; } -#if PY_VERSION_HEX >= 0x02070000 if (!PyCapsule_IsValid(cobj, sig)) { PyErr_Format(PyExc_TypeError, "C variable %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", @@ -548,21 +744,6 @@ static int __Pyx_ImportVoidPtr_$cyversion(PyObject *module, const char *name, vo goto bad; } *p = PyCapsule_GetPointer(cobj, sig); -#else - {const char *desc, *s1, *s2; - desc = (const char *)PyCObject_GetDesc(cobj); - if (!desc) - goto bad; - s1 = desc; s2 = sig; - while (*s1 != '\0' && *s1 == *s2) { s1++; s2++; } - if (*s1 != *s2) { - PyErr_Format(PyExc_TypeError, - "C variable %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", - PyModule_GetName(module), name, sig, desc); - goto bad; - } - *p = PyCObject_AsVoidPtr(cobj);} -#endif if (!(*p)) goto bad; Py_DECREF(d); @@ -594,11 +775,7 @@ static int __Pyx_ExportVoidPtr(PyObject *name, void *p, const char *sig) { if (__Pyx_PyObject_SetAttrStr($module_cname, PYIDENT("$api_name"), d) < 0) goto bad; } -#if PY_VERSION_HEX >= 0x02070000 cobj = PyCapsule_New(p, sig, 0); -#else - cobj = PyCObject_FromVoidPtrAndDesc(p, (void *)sig, 0); -#endif if (!cobj) goto bad; if (PyDict_SetItem(d, name, cobj) < 0) @@ -615,19 +792,19 @@ bad: /////////////// SetVTable.proto /////////////// -static int __Pyx_SetVtable(PyObject *dict, void *vtable); /*proto*/ +static int __Pyx_SetVtable(PyTypeObject* typeptr , void* vtable); /*proto*/ /////////////// SetVTable /////////////// -static int __Pyx_SetVtable(PyObject *dict, void *vtable) { -#if PY_VERSION_HEX >= 0x02070000 +static int __Pyx_SetVtable(PyTypeObject *type, void *vtable) { PyObject *ob = PyCapsule_New(vtable, 0, 0); + if (unlikely(!ob)) + goto bad; +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(PyObject_SetAttr((PyObject *) type, PYIDENT("__pyx_vtable__"), ob) < 0)) #else - PyObject *ob = PyCObject_FromVoidPtr(vtable, 0); + if (unlikely(PyDict_SetItem(type->tp_dict, PYIDENT("__pyx_vtable__"), ob) < 0)) #endif - if (!ob) - goto bad; - if (PyDict_SetItem(dict, PYIDENT("__pyx_vtable__"), ob) < 0) goto bad; Py_DECREF(ob); return 0; @@ -639,20 +816,20 @@ bad: /////////////// GetVTable.proto /////////////// -static void* __Pyx_GetVtable(PyObject *dict); /*proto*/ +static void* __Pyx_GetVtable(PyTypeObject *type); /*proto*/ /////////////// GetVTable /////////////// -static void* __Pyx_GetVtable(PyObject *dict) { +static void* __Pyx_GetVtable(PyTypeObject *type) { void* ptr; - PyObject *ob = PyObject_GetItem(dict, PYIDENT("__pyx_vtable__")); +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *ob = PyObject_GetAttr((PyObject *)type, PYIDENT("__pyx_vtable__")); +#else + PyObject *ob = PyObject_GetItem(type->tp_dict, PYIDENT("__pyx_vtable__")); +#endif if (!ob) goto bad; -#if PY_VERSION_HEX >= 0x02070000 ptr = PyCapsule_GetPointer(ob, 0); -#else - ptr = PyCObject_AsVoidPtr(ob); -#endif if (!ptr && !PyErr_Occurred()) PyErr_SetString(PyExc_RuntimeError, "invalid vtable found for imported type"); Py_DECREF(ob); @@ -666,13 +843,19 @@ bad: /////////////// MergeVTables.proto /////////////// //@requires: GetVTable +// TODO: find a way to make this work with the Limited API! +#if !CYTHON_COMPILING_IN_LIMITED_API static int __Pyx_MergeVtables(PyTypeObject *type); /*proto*/ +#endif /////////////// MergeVTables /////////////// +#if !CYTHON_COMPILING_IN_LIMITED_API static int __Pyx_MergeVtables(PyTypeObject *type) { int i; void** base_vtables; + __Pyx_TypeName tp_base_name; + __Pyx_TypeName base_name; void* unknown = (void*)-1; PyObject* bases = type->tp_bases; int base_depth = 0; @@ -692,13 +875,13 @@ static int __Pyx_MergeVtables(PyTypeObject *type) { // instance struct is so extended. (It would be good to also do this // check when a multiple-base class is created in pure Python as well.) for (i = 1; i < PyTuple_GET_SIZE(bases); i++) { - void* base_vtable = __Pyx_GetVtable(((PyTypeObject*)PyTuple_GET_ITEM(bases, i))->tp_dict); + void* base_vtable = __Pyx_GetVtable(((PyTypeObject*)PyTuple_GET_ITEM(bases, i))); if (base_vtable != NULL) { int j; PyTypeObject* base = type->tp_base; for (j = 0; j < base_depth; j++) { if (base_vtables[j] == unknown) { - base_vtables[j] = __Pyx_GetVtable(base->tp_dict); + base_vtables[j] = __Pyx_GetVtable(base); base_vtables[j + 1] = unknown; } if (base_vtables[j] == base_vtable) { @@ -715,13 +898,16 @@ static int __Pyx_MergeVtables(PyTypeObject *type) { free(base_vtables); return 0; bad: - PyErr_Format( - PyExc_TypeError, - "multiple bases have vtable conflict: '%s' and '%s'", - type->tp_base->tp_name, ((PyTypeObject*)PyTuple_GET_ITEM(bases, i))->tp_name); + tp_base_name = __Pyx_PyType_GetName(type->tp_base); + base_name = __Pyx_PyType_GetName((PyTypeObject*)PyTuple_GET_ITEM(bases, i)); + PyErr_Format(PyExc_TypeError, + "multiple bases have vtable conflict: '" __Pyx_FMT_TYPENAME "' and '" __Pyx_FMT_TYPENAME "'", tp_base_name, base_name); + __Pyx_DECREF_TypeName(tp_base_name); + __Pyx_DECREF_TypeName(base_name); free(base_vtables); return -1; } +#endif /////////////// ImportNumPyArray.proto /////////////// diff --git a/Cython/Utility/MemoryView.pyx b/Cython/Utility/MemoryView.pyx index 277c0bd87..40572d60e 100644 --- a/Cython/Utility/MemoryView.pyx +++ b/Cython/Utility/MemoryView.pyx @@ -1,5 +1,8 @@ #################### View.MemoryView #################### +# cython: language_level=3str +# cython: binding=False + # This utility provides cython.array and cython.view.memoryview from __future__ import absolute_import @@ -8,22 +11,22 @@ cimport cython # from cpython cimport ... cdef extern from "Python.h": + ctypedef struct PyObject int PyIndex_Check(object) - object PyLong_FromVoidPtr(void *) + PyObject *PyExc_IndexError + PyObject *PyExc_ValueError cdef extern from "pythread.h": ctypedef void *PyThread_type_lock PyThread_type_lock PyThread_allocate_lock() void PyThread_free_lock(PyThread_type_lock) - int PyThread_acquire_lock(PyThread_type_lock, int mode) nogil - void PyThread_release_lock(PyThread_type_lock) nogil cdef extern from "<string.h>": void *memset(void *b, int c, size_t len) cdef extern from *: - bint __PYX_CYTHON_ATOMICS_ENABLED() noexcept + bint __PYX_CYTHON_ATOMICS_ENABLED() int __Pyx_GetBuffer(object, Py_buffer *, int) except -1 void __Pyx_ReleaseBuffer(Py_buffer *) @@ -50,7 +53,7 @@ cdef extern from *: Py_ssize_t suboffsets[{{max_dims}}] void __PYX_INC_MEMVIEW({{memviewslice_name}} *memslice, int have_gil) - void __PYX_XDEC_MEMVIEW({{memviewslice_name}} *memslice, int have_gil) + void __PYX_XCLEAR_MEMVIEW({{memviewslice_name}} *memslice, int have_gil) ctypedef struct __pyx_buffer "Py_buffer": PyObject *obj @@ -72,17 +75,13 @@ cdef extern from *: ctypedef struct __Pyx_TypeInfo: pass - cdef object capsule "__pyx_capsule_create" (void *p, char *sig) - cdef int __pyx_array_getbuffer(PyObject *obj, Py_buffer view, int flags) - cdef int __pyx_memoryview_getbuffer(PyObject *obj, Py_buffer view, int flags) - cdef extern from *: ctypedef int __pyx_atomic_int {{memviewslice_name}} slice_copy_contig "__pyx_memoryview_copy_new_contig"( __Pyx_memviewslice *from_mvs, char *mode, int ndim, size_t sizeof_dtype, int contig_flag, - bint dtype_is_object) nogil except * + bint dtype_is_object) except * nogil bint slice_is_contig "__pyx_memviewslice_is_contig" ( {{memviewslice_name}} mvs, char order, int ndim) nogil bint slices_overlap "__pyx_slices_overlap" ({{memviewslice_name}} *slice1, @@ -95,13 +94,22 @@ cdef extern from "<stdlib.h>": void free(void *) nogil void *memcpy(void *dest, void *src, size_t n) nogil - - +# the sequence abstract base class +cdef object __pyx_collections_abc_Sequence "__pyx_collections_abc_Sequence" +try: + if __import__("sys").version_info >= (3, 3): + __pyx_collections_abc_Sequence = __import__("collections.abc").abc.Sequence + else: + __pyx_collections_abc_Sequence = __import__("collections").Sequence +except: + # it isn't a big problem if this fails + __pyx_collections_abc_Sequence = None # ### cython.array class # +@cython.collection_type("sequence") @cname("__pyx_array") cdef class array: @@ -115,7 +123,7 @@ cdef class array: Py_ssize_t itemsize unicode mode # FIXME: this should have been a simple 'char' bytes _format - void (*callback_free_data)(void *data) + void (*callback_free_data)(void *data) noexcept # cdef object _memview cdef bint free_data cdef bint dtype_is_object @@ -124,17 +132,16 @@ cdef class array: mode="c", bint allocate_buffer=True): cdef int idx - cdef Py_ssize_t i, dim - cdef PyObject **p + cdef Py_ssize_t dim self.ndim = <int> len(shape) self.itemsize = itemsize if not self.ndim: - raise ValueError("Empty shape tuple for cython.array") + raise ValueError, "Empty shape tuple for cython.array" if itemsize <= 0: - raise ValueError("itemsize <= 0 for cython.array") + raise ValueError, "itemsize <= 0 for cython.array" if not isinstance(format, bytes): format = format.encode('ASCII') @@ -146,76 +153,66 @@ cdef class array: self._strides = self._shape + self.ndim if not self._shape: - raise MemoryError("unable to allocate shape and strides.") + raise MemoryError, "unable to allocate shape and strides." # cdef Py_ssize_t dim, stride for idx, dim in enumerate(shape): if dim <= 0: - raise ValueError("Invalid shape in axis %d: %d." % (idx, dim)) + raise ValueError, f"Invalid shape in axis {idx}: {dim}." self._shape[idx] = dim cdef char order - if mode == 'fortran': - order = b'F' - self.mode = u'fortran' - elif mode == 'c': + if mode == 'c': order = b'C' self.mode = u'c' + elif mode == 'fortran': + order = b'F' + self.mode = u'fortran' else: - raise ValueError("Invalid mode, expected 'c' or 'fortran', got %s" % mode) + raise ValueError, f"Invalid mode, expected 'c' or 'fortran', got {mode}" - self.len = fill_contig_strides_array(self._shape, self._strides, - itemsize, self.ndim, order) + self.len = fill_contig_strides_array(self._shape, self._strides, itemsize, self.ndim, order) self.free_data = allocate_buffer self.dtype_is_object = format == b'O' - if allocate_buffer: - # use malloc() for backwards compatibility - # in case external code wants to change the data pointer - self.data = <char *>malloc(self.len) - if not self.data: - raise MemoryError("unable to allocate array data.") - if self.dtype_is_object: - p = <PyObject **> self.data - for i in range(self.len / itemsize): - p[i] = Py_None - Py_INCREF(Py_None) + if allocate_buffer: + _allocate_buffer(self) @cname('getbuffer') def __getbuffer__(self, Py_buffer *info, int flags): cdef int bufmode = -1 - if self.mode == u"c": - bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS - elif self.mode == u"fortran": - bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS - if not (flags & bufmode): - raise ValueError("Can only create a buffer that is contiguous in memory.") + if flags & (PyBUF_C_CONTIGUOUS | PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS): + if self.mode == u"c": + bufmode = PyBUF_C_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + elif self.mode == u"fortran": + bufmode = PyBUF_F_CONTIGUOUS | PyBUF_ANY_CONTIGUOUS + if not (flags & bufmode): + raise ValueError, "Can only create a buffer that is contiguous in memory." info.buf = self.data info.len = self.len - info.ndim = self.ndim - info.shape = self._shape - info.strides = self._strides - info.suboffsets = NULL - info.itemsize = self.itemsize - info.readonly = 0 - if flags & PyBUF_FORMAT: - info.format = self.format + if flags & PyBUF_STRIDES: + info.ndim = self.ndim + info.shape = self._shape + info.strides = self._strides else: - info.format = NULL + info.ndim = 1 + info.shape = &self.len if flags & PyBUF_ND else NULL + info.strides = NULL + info.suboffsets = NULL + info.itemsize = self.itemsize + info.readonly = 0 + info.format = self.format if flags & PyBUF_FORMAT else NULL info.obj = self - __pyx_getbuffer = capsule(<void *> &__pyx_array_getbuffer, "getbuffer(obj, view, flags)") - def __dealloc__(array self): if self.callback_free_data != NULL: self.callback_free_data(self.data) - elif self.free_data: + elif self.free_data and self.data is not NULL: if self.dtype_is_object: - refcount_objects_in_slice(self.data, self._shape, - self._strides, self.ndim, False) + refcount_objects_in_slice(self.data, self._shape, self._strides, self.ndim, inc=False) free(self.data) PyObject_Free(self._shape) @@ -240,17 +237,42 @@ cdef class array: def __setitem__(self, item, value): self.memview[item] = value + # Sequence methods + try: + count = __pyx_collections_abc_Sequence.count + index = __pyx_collections_abc_Sequence.index + except: + pass + +@cname("__pyx_array_allocate_buffer") +cdef int _allocate_buffer(array self) except -1: + # use malloc() for backwards compatibility + # in case external code wants to change the data pointer + cdef Py_ssize_t i + cdef PyObject **p + + self.free_data = True + self.data = <char *>malloc(self.len) + if not self.data: + raise MemoryError, "unable to allocate array data." + + if self.dtype_is_object: + p = <PyObject **> self.data + for i in range(self.len // self.itemsize): + p[i] = Py_None + Py_INCREF(Py_None) + return 0 + @cname("__pyx_array_new") -cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, - char *mode, char *buf): +cdef array array_cwrapper(tuple shape, Py_ssize_t itemsize, char *format, char *c_mode, char *buf): cdef array result + cdef str mode = "fortran" if c_mode[0] == b'f' else "c" # this often comes from a constant C string. - if buf == NULL: - result = array(shape, itemsize, format, mode.decode('ASCII')) + if buf is NULL: + result = array.__new__(array, shape, itemsize, format, mode) else: - result = array(shape, itemsize, format, mode.decode('ASCII'), - allocate_buffer=False) + result = array.__new__(array, shape, itemsize, format, mode, allocate_buffer=False) result.data = buf return result @@ -296,7 +318,7 @@ cdef indirect_contiguous = Enum("<contiguous and indirect>") @cname('__pyx_align_pointer') -cdef void *align_pointer(void *memory, size_t alignment) nogil: +cdef void *align_pointer(void *memory, size_t alignment) noexcept nogil: "Align pointer memory on a given boundary" cdef Py_intptr_t aligned_p = <Py_intptr_t> memory cdef size_t offset @@ -313,22 +335,16 @@ cdef void *align_pointer(void *memory, size_t alignment) nogil: # pre-allocate thread locks for reuse ## note that this could be implemented in a more beautiful way in "normal" Cython, ## but this code gets merged into the user module and not everything works there. -DEF THREAD_LOCKS_PREALLOCATED = 8 cdef int __pyx_memoryview_thread_locks_used = 0 -cdef PyThread_type_lock[THREAD_LOCKS_PREALLOCATED] __pyx_memoryview_thread_locks = [ - PyThread_allocate_lock(), - PyThread_allocate_lock(), - PyThread_allocate_lock(), - PyThread_allocate_lock(), - PyThread_allocate_lock(), - PyThread_allocate_lock(), - PyThread_allocate_lock(), +cdef PyThread_type_lock[{{THREAD_LOCKS_PREALLOCATED}}] __pyx_memoryview_thread_locks = [ +{{for _ in range(THREAD_LOCKS_PREALLOCATED)}} PyThread_allocate_lock(), +{{endfor}} ] @cname('__pyx_memoryview') -cdef class memoryview(object): +cdef class memoryview: cdef object obj cdef object _size @@ -354,7 +370,7 @@ cdef class memoryview(object): if not __PYX_CYTHON_ATOMICS_ENABLED(): global __pyx_memoryview_thread_locks_used - if __pyx_memoryview_thread_locks_used < THREAD_LOCKS_PREALLOCATED: + if __pyx_memoryview_thread_locks_used < {{THREAD_LOCKS_PREALLOCATED}}: self.lock = __pyx_memoryview_thread_locks[__pyx_memoryview_thread_locks_used] __pyx_memoryview_thread_locks_used += 1 if self.lock is NULL: @@ -417,7 +433,7 @@ cdef class memoryview(object): def __setitem__(memoryview self, object index, object value): if self.view.readonly: - raise TypeError("Cannot assign to read-only memoryview") + raise TypeError, "Cannot assign to read-only memoryview" have_slices, index = _unellipsify(index, self.view.ndim) @@ -443,10 +459,10 @@ cdef class memoryview(object): cdef setitem_slice_assignment(self, dst, src): cdef {{memviewslice_name}} dst_slice cdef {{memviewslice_name}} src_slice + cdef {{memviewslice_name}} msrc = get_slice_from_memview(src, &src_slice)[0] + cdef {{memviewslice_name}} mdst = get_slice_from_memview(dst, &dst_slice)[0] - memoryview_copy_contents(get_slice_from_memview(src, &src_slice)[0], - get_slice_from_memview(dst, &dst_slice)[0], - src.ndim, dst.ndim, self.dtype_is_object) + memoryview_copy_contents(msrc, mdst, src.ndim, dst.ndim, self.dtype_is_object) cdef setitem_slice_assign_scalar(self, memoryview dst, value): cdef int array[128] @@ -494,7 +510,7 @@ cdef class memoryview(object): try: result = struct.unpack(self.view.format, bytesitem) except struct.error: - raise ValueError("Unable to convert item to object") + raise ValueError, "Unable to convert item to object" else: if len(self.view.format) == 1: return result[0] @@ -519,7 +535,7 @@ cdef class memoryview(object): @cname('getbuffer') def __getbuffer__(self, Py_buffer *info, int flags): if flags & PyBUF_WRITABLE and self.view.readonly: - raise ValueError("Cannot create writable memory view from read-only memoryview") + raise ValueError, "Cannot create writable memory view from read-only memoryview" if flags & PyBUF_ND: info.shape = self.view.shape @@ -548,8 +564,6 @@ cdef class memoryview(object): info.readonly = self.view.readonly info.obj = self - __pyx_getbuffer = capsule(<void *> &__pyx_memoryview_getbuffer, "getbuffer(obj, view, flags)") - # Some properties that have the same semantics as in NumPy @property def T(self): @@ -559,6 +573,9 @@ cdef class memoryview(object): @property def base(self): + return self._get_base() + + cdef _get_base(self): return self.obj @property @@ -569,7 +586,7 @@ cdef class memoryview(object): def strides(self): if self.view.strides == NULL: # Note: we always ask for strides, so if this is not set it's a bug - raise ValueError("Buffer view does not expose strides") + raise ValueError, "Buffer view does not expose strides" return tuple([stride for stride in self.view.strides[:self.view.ndim]]) @@ -662,7 +679,7 @@ cdef memoryview_cwrapper(object o, int flags, bint dtype_is_object, __Pyx_TypeIn return result @cname('__pyx_memoryview_check') -cdef inline bint memoryview_check(object o): +cdef inline bint memoryview_check(object o) noexcept: return isinstance(o, memoryview) cdef tuple _unellipsify(object index, int ndim): @@ -670,39 +687,35 @@ cdef tuple _unellipsify(object index, int ndim): Replace all ellipses with full slices and fill incomplete indices with full slices. """ - if not isinstance(index, tuple): - tup = (index,) - else: - tup = index + cdef Py_ssize_t idx + tup = <tuple>index if isinstance(index, tuple) else (index,) - result = [] + result = [slice(None)] * ndim have_slices = False seen_ellipsis = False - for idx, item in enumerate(tup): + idx = 0 + for item in tup: if item is Ellipsis: if not seen_ellipsis: - result.extend([slice(None)] * (ndim - len(tup) + 1)) + idx += ndim - len(tup) seen_ellipsis = True - else: - result.append(slice(None)) have_slices = True else: - if not isinstance(item, slice) and not PyIndex_Check(item): - raise TypeError("Cannot index with type '%s'" % type(item)) - - have_slices = have_slices or isinstance(item, slice) - result.append(item) - - nslices = ndim - len(result) - if nslices: - result.extend([slice(None)] * nslices) - + if isinstance(item, slice): + have_slices = True + elif not PyIndex_Check(item): + raise TypeError, f"Cannot index with type '{type(item)}'" + result[idx] = item + idx += 1 + + nslices = ndim - idx return have_slices or nslices, tuple(result) -cdef assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim): +cdef int assert_direct_dimensions(Py_ssize_t *suboffsets, int ndim) except -1: for suboffset in suboffsets[:ndim]: if suboffset >= 0: - raise ValueError("Indirect dimensions not supported") + raise ValueError, "Indirect dimensions not supported" + return 0 # return type just used as an error flag # ### Slicing a memoryview @@ -742,15 +755,16 @@ cdef memoryview memview_slice(memoryview memview, object indices): # may not be as expected" cdef {{memviewslice_name}} *p_dst = &dst cdef int *p_suboffset_dim = &suboffset_dim - cdef Py_ssize_t start, stop, step + cdef Py_ssize_t start, stop, step, cindex cdef bint have_start, have_stop, have_step for dim, index in enumerate(indices): if PyIndex_Check(index): + cindex = index slice_memviewslice( p_dst, p_src.shape[dim], p_src.strides[dim], p_src.suboffsets[dim], dim, new_ndim, p_suboffset_dim, - index, 0, 0, # start, stop, step + cindex, 0, 0, # start, stop, step 0, 0, 0, # have_{start,stop,step} False) elif index is None: @@ -789,22 +803,6 @@ cdef memoryview memview_slice(memoryview memview, object indices): ### Slicing in a single dimension of a memoryviewslice # -cdef extern from "<stdlib.h>": - void abort() nogil - void printf(char *s, ...) nogil - -cdef extern from "<stdio.h>": - ctypedef struct FILE - FILE *stderr - int fputs(char *s, FILE *stream) - -cdef extern from "pystate.h": - void PyThreadState_Get() nogil - - # These are not actually nogil, but we check for the GIL before calling them - void PyErr_SetString(PyObject *type, char *msg) nogil - PyObject *PyErr_Format(PyObject *exc, char *msg, ...) nogil - @cname('__pyx_memoryview_slice_memviewslice') cdef int slice_memviewslice( {{memviewslice_name}} *dst, @@ -812,7 +810,7 @@ cdef int slice_memviewslice( int dim, int new_ndim, int *suboffset_dim, Py_ssize_t start, Py_ssize_t stop, Py_ssize_t step, int have_start, int have_stop, int have_step, - bint is_slice) nogil except -1: + bint is_slice) except -1 nogil: """ Create a new slice dst given slice src. @@ -831,13 +829,16 @@ cdef int slice_memviewslice( if start < 0: start += shape if not 0 <= start < shape: - _err_dim(IndexError, "Index out of bounds (axis %d)", dim) + _err_dim(PyExc_IndexError, "Index out of bounds (axis %d)", dim) else: # index is a slice - negative_step = have_step != 0 and step < 0 - - if have_step and step == 0: - _err_dim(ValueError, "Step may not be zero (axis %d)", dim) + if have_step: + negative_step = step < 0 + if step == 0: + _err_dim(PyExc_ValueError, "Step may not be zero (axis %d)", dim) + else: + negative_step = False + step = 1 # check our bounds and set defaults if have_start: @@ -869,9 +870,6 @@ cdef int slice_memviewslice( else: stop = shape - if not have_step: - step = 1 - # len = ceil( (stop - start) / step ) with cython.cdivision(True): new_shape = (stop - start) // step @@ -887,7 +885,7 @@ cdef int slice_memviewslice( dst.shape[new_ndim] = new_shape dst.suboffsets[new_ndim] = suboffset - # Add the slicing or idexing offsets to the right suboffset or base data * + # Add the slicing or indexing offsets to the right suboffset or base data * if suboffset_dim[0] < 0: dst.data += start * stride else: @@ -898,7 +896,7 @@ cdef int slice_memviewslice( if new_ndim == 0: dst.data = (<char **> dst.data)[0] + suboffset else: - _err_dim(IndexError, "All dimensions preceding dimension %d " + _err_dim(PyExc_IndexError, "All dimensions preceding dimension %d " "must be indexed and not sliced", dim) else: suboffset_dim[0] = new_ndim @@ -916,7 +914,7 @@ cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, cdef char *resultp if view.ndim == 0: - shape = view.len / itemsize + shape = view.len // itemsize stride = itemsize else: shape = view.shape[dim] @@ -927,10 +925,10 @@ cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, if index < 0: index += view.shape[dim] if index < 0: - raise IndexError("Out of bounds on buffer access (axis %d)" % dim) + raise IndexError, f"Out of bounds on buffer access (axis {dim})" if index >= shape: - raise IndexError("Out of bounds on buffer access (axis %d)" % dim) + raise IndexError, f"Out of bounds on buffer access (axis {dim})" resultp = bufp + index * stride if suboffset >= 0: @@ -942,7 +940,7 @@ cdef char *pybuffer_index(Py_buffer *view, char *bufp, Py_ssize_t index, ### Transposing a memoryviewslice # @cname('__pyx_memslice_transpose') -cdef int transpose_memslice({{memviewslice_name}} *memslice) nogil except 0: +cdef int transpose_memslice({{memviewslice_name}} *memslice) except -1 nogil: cdef int ndim = memslice.memview.view.ndim cdef Py_ssize_t *shape = memslice.shape @@ -950,19 +948,20 @@ cdef int transpose_memslice({{memviewslice_name}} *memslice) nogil except 0: # reverse strides and shape cdef int i, j - for i in range(ndim / 2): + for i in range(ndim // 2): j = ndim - 1 - i strides[i], strides[j] = strides[j], strides[i] shape[i], shape[j] = shape[j], shape[i] if memslice.suboffsets[i] >= 0 or memslice.suboffsets[j] >= 0: - _err(ValueError, "Cannot transpose memoryview with indirect dimensions") + _err(PyExc_ValueError, "Cannot transpose memoryview with indirect dimensions") - return 1 + return 0 # ### Creating new memoryview objects from slices and memoryviews # +@cython.collection_type("sequence") @cname('__pyx_memoryviewslice') cdef class _memoryviewslice(memoryview): "Internal class for passing memoryview slices to Python" @@ -976,7 +975,7 @@ cdef class _memoryviewslice(memoryview): cdef int (*to_dtype_func)(char *, object) except 0 def __dealloc__(self): - __PYX_XDEC_MEMVIEW(&self.from_slice, 1) + __PYX_XCLEAR_MEMVIEW(&self.from_slice, 1) cdef convert_item_to_object(self, char *itemp): if self.to_object_func != NULL: @@ -990,12 +989,25 @@ cdef class _memoryviewslice(memoryview): else: memoryview.assign_item_from_object(self, itemp, value) - @property - def base(self): + cdef _get_base(self): return self.from_object - __pyx_getbuffer = capsule(<void *> &__pyx_memoryview_getbuffer, "getbuffer(obj, view, flags)") + # Sequence methods + try: + count = __pyx_collections_abc_Sequence.count + index = __pyx_collections_abc_Sequence.index + except: + pass +try: + if __pyx_collections_abc_Sequence: + # The main value of registering _memoryviewslice as a + # Sequence is that it can be used in structural pattern + # matching in Python 3.10+ + __pyx_collections_abc_Sequence.register(_memoryviewslice) + __pyx_collections_abc_Sequence.register(array) +except: + pass # ignore failure, it's a minor issue @cname('__pyx_memoryview_fromslice') cdef memoryview_fromslice({{memviewslice_name}} memviewslice, @@ -1012,12 +1024,12 @@ cdef memoryview_fromslice({{memviewslice_name}} memviewslice, # assert 0 < ndim <= memviewslice.memview.view.ndim, ( # ndim, memviewslice.memview.view.ndim) - result = _memoryviewslice(None, 0, dtype_is_object) + result = _memoryviewslice.__new__(_memoryviewslice, None, 0, dtype_is_object) result.from_slice = memviewslice __PYX_INC_MEMVIEW(&memviewslice, 1) - result.from_object = (<memoryview> memviewslice.memview).base + result.from_object = (<memoryview> memviewslice.memview)._get_base() result.typeinfo = memviewslice.memview.typeinfo result.view = memviewslice.memview.view @@ -1062,7 +1074,7 @@ cdef {{memviewslice_name}} *get_slice_from_memview(memoryview memview, return mslice @cname('__pyx_memoryview_slice_copy') -cdef void slice_copy(memoryview memview, {{memviewslice_name}} *dst): +cdef void slice_copy(memoryview memview, {{memviewslice_name}} *dst) noexcept: cdef int dim cdef (Py_ssize_t*) shape, strides, suboffsets @@ -1108,14 +1120,11 @@ cdef memoryview_copy_from_slice(memoryview memview, {{memviewslice_name}} *memvi # ### Copy the contents of a memoryview slices # -cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) nogil: - if arg < 0: - return -arg - else: - return arg +cdef Py_ssize_t abs_py_ssize_t(Py_ssize_t arg) noexcept nogil: + return -arg if arg < 0 else arg @cname('__pyx_get_best_slice_order') -cdef char get_best_order({{memviewslice_name}} *mslice, int ndim) nogil: +cdef char get_best_order({{memviewslice_name}} *mslice, int ndim) noexcept nogil: """ Figure out the best memory access order for a given slice. """ @@ -1142,7 +1151,7 @@ cdef char get_best_order({{memviewslice_name}} *mslice, int ndim) nogil: cdef void _copy_strided_to_strided(char *src_data, Py_ssize_t *src_strides, char *dst_data, Py_ssize_t *dst_strides, Py_ssize_t *src_shape, Py_ssize_t *dst_shape, - int ndim, size_t itemsize) nogil: + int ndim, size_t itemsize) noexcept nogil: # Note: src_extent is 1 if we're broadcasting # dst_extent always >= src_extent as we don't do reductions cdef Py_ssize_t i @@ -1152,14 +1161,14 @@ cdef void _copy_strided_to_strided(char *src_data, Py_ssize_t *src_strides, cdef Py_ssize_t dst_stride = dst_strides[0] if ndim == 1: - if (src_stride > 0 and dst_stride > 0 and - <size_t> src_stride == itemsize == <size_t> dst_stride): - memcpy(dst_data, src_data, itemsize * dst_extent) - else: - for i in range(dst_extent): - memcpy(dst_data, src_data, itemsize) - src_data += src_stride - dst_data += dst_stride + if (src_stride > 0 and dst_stride > 0 and + <size_t> src_stride == itemsize == <size_t> dst_stride): + memcpy(dst_data, src_data, itemsize * dst_extent) + else: + for i in range(dst_extent): + memcpy(dst_data, src_data, itemsize) + src_data += src_stride + dst_data += dst_stride else: for i in range(dst_extent): _copy_strided_to_strided(src_data, src_strides + 1, @@ -1171,12 +1180,12 @@ cdef void _copy_strided_to_strided(char *src_data, Py_ssize_t *src_strides, cdef void copy_strided_to_strided({{memviewslice_name}} *src, {{memviewslice_name}} *dst, - int ndim, size_t itemsize) nogil: + int ndim, size_t itemsize) noexcept nogil: _copy_strided_to_strided(src.data, src.strides, dst.data, dst.strides, src.shape, dst.shape, ndim, itemsize) @cname('__pyx_memoryview_slice_get_size') -cdef Py_ssize_t slice_get_size({{memviewslice_name}} *src, int ndim) nogil: +cdef Py_ssize_t slice_get_size({{memviewslice_name}} *src, int ndim) noexcept nogil: "Return the size of the memory occupied by the slice in number of bytes" cdef Py_ssize_t shape, size = src.memview.view.itemsize @@ -1188,7 +1197,7 @@ cdef Py_ssize_t slice_get_size({{memviewslice_name}} *src, int ndim) nogil: @cname('__pyx_fill_contig_strides_array') cdef Py_ssize_t fill_contig_strides_array( Py_ssize_t *shape, Py_ssize_t *strides, Py_ssize_t stride, - int ndim, char order) nogil: + int ndim, char order) noexcept nogil: """ Fill the strides array for a slice with C or F contiguous strides. This is like PyBuffer_FillContiguousStrides, but compatible with py < 2.6 @@ -1210,7 +1219,7 @@ cdef Py_ssize_t fill_contig_strides_array( cdef void *copy_data_to_temp({{memviewslice_name}} *src, {{memviewslice_name}} *tmpslice, char order, - int ndim) nogil except NULL: + int ndim) except NULL nogil: """ Copy a direct slice to temporary contiguous memory. The caller should free the result when done. @@ -1223,7 +1232,7 @@ cdef void *copy_data_to_temp({{memviewslice_name}} *src, result = malloc(size) if not result: - _err(MemoryError, NULL) + _err_no_memory() # tmpslice[0] = src tmpslice.data = <char *> result @@ -1232,8 +1241,7 @@ cdef void *copy_data_to_temp({{memviewslice_name}} *src, tmpslice.shape[i] = src.shape[i] tmpslice.suboffsets[i] = -1 - fill_contig_strides_array(&tmpslice.shape[0], &tmpslice.strides[0], itemsize, - ndim, order) + fill_contig_strides_array(&tmpslice.shape[0], &tmpslice.strides[0], itemsize, ndim, order) # We need to broadcast strides again for i in range(ndim): @@ -1252,25 +1260,26 @@ cdef void *copy_data_to_temp({{memviewslice_name}} *src, @cname('__pyx_memoryview_err_extents') cdef int _err_extents(int i, Py_ssize_t extent1, Py_ssize_t extent2) except -1 with gil: - raise ValueError("got differing extents in dimension %d (got %d and %d)" % - (i, extent1, extent2)) + raise ValueError, f"got differing extents in dimension {i} (got {extent1} and {extent2})" @cname('__pyx_memoryview_err_dim') -cdef int _err_dim(object error, char *msg, int dim) except -1 with gil: - raise error(msg.decode('ascii') % dim) +cdef int _err_dim(PyObject *error, str msg, int dim) except -1 with gil: + raise <object>error, msg % dim @cname('__pyx_memoryview_err') -cdef int _err(object error, char *msg) except -1 with gil: - if msg != NULL: - raise error(msg.decode('ascii')) - else: - raise error +cdef int _err(PyObject *error, str msg) except -1 with gil: + raise <object>error, msg + +@cname('__pyx_memoryview_err_no_memory') +cdef int _err_no_memory() except -1 with gil: + raise MemoryError + @cname('__pyx_memoryview_copy_contents') cdef int memoryview_copy_contents({{memviewslice_name}} src, {{memviewslice_name}} dst, int src_ndim, int dst_ndim, - bint dtype_is_object) nogil except -1: + bint dtype_is_object) except -1 nogil: """ Copy memory from slice src to slice dst. Check for overlapping memory and verify the shapes. @@ -1299,7 +1308,7 @@ cdef int memoryview_copy_contents({{memviewslice_name}} src, _err_extents(i, dst.shape[i], src.shape[i]) if src.suboffsets[i] >= 0: - _err_dim(ValueError, "Dimension %d is not direct", i) + _err_dim(PyExc_ValueError, "Dimension %d is not direct", i) if slices_overlap(&src, &dst, ndim, itemsize): # slices overlap, copy to temp, copy temp to dst @@ -1319,9 +1328,9 @@ cdef int memoryview_copy_contents({{memviewslice_name}} src, if direct_copy: # Contiguous slices with same order - refcount_copying(&dst, dtype_is_object, ndim, False) + refcount_copying(&dst, dtype_is_object, ndim, inc=False) memcpy(dst.data, src.data, slice_get_size(&src, ndim)) - refcount_copying(&dst, dtype_is_object, ndim, True) + refcount_copying(&dst, dtype_is_object, ndim, inc=True) free(tmpdata) return 0 @@ -1331,9 +1340,9 @@ cdef int memoryview_copy_contents({{memviewslice_name}} src, transpose_memslice(&src) transpose_memslice(&dst) - refcount_copying(&dst, dtype_is_object, ndim, False) + refcount_copying(&dst, dtype_is_object, ndim, inc=False) copy_strided_to_strided(&src, &dst, ndim, itemsize) - refcount_copying(&dst, dtype_is_object, ndim, True) + refcount_copying(&dst, dtype_is_object, ndim, inc=True) free(tmpdata) return 0 @@ -1341,7 +1350,7 @@ cdef int memoryview_copy_contents({{memviewslice_name}} src, @cname('__pyx_memoryview_broadcast_leading') cdef void broadcast_leading({{memviewslice_name}} *mslice, int ndim, - int ndim_other) nogil: + int ndim_other) noexcept nogil: cdef int i cdef int offset = ndim_other - ndim @@ -1361,24 +1370,22 @@ cdef void broadcast_leading({{memviewslice_name}} *mslice, # @cname('__pyx_memoryview_refcount_copying') -cdef void refcount_copying({{memviewslice_name}} *dst, bint dtype_is_object, - int ndim, bint inc) nogil: - # incref or decref the objects in the destination slice if the dtype is - # object +cdef void refcount_copying({{memviewslice_name}} *dst, bint dtype_is_object, int ndim, bint inc) noexcept nogil: + # incref or decref the objects in the destination slice if the dtype is object if dtype_is_object: - refcount_objects_in_slice_with_gil(dst.data, dst.shape, - dst.strides, ndim, inc) + refcount_objects_in_slice_with_gil(dst.data, dst.shape, dst.strides, ndim, inc) @cname('__pyx_memoryview_refcount_objects_in_slice_with_gil') cdef void refcount_objects_in_slice_with_gil(char *data, Py_ssize_t *shape, Py_ssize_t *strides, int ndim, - bint inc) with gil: + bint inc) noexcept with gil: refcount_objects_in_slice(data, shape, strides, ndim, inc) @cname('__pyx_memoryview_refcount_objects_in_slice') cdef void refcount_objects_in_slice(char *data, Py_ssize_t *shape, - Py_ssize_t *strides, int ndim, bint inc): + Py_ssize_t *strides, int ndim, bint inc) noexcept: cdef Py_ssize_t i + cdef Py_ssize_t stride = strides[0] for i in range(shape[0]): if ndim == 1: @@ -1387,10 +1394,9 @@ cdef void refcount_objects_in_slice(char *data, Py_ssize_t *shape, else: Py_DECREF((<PyObject **> data)[0]) else: - refcount_objects_in_slice(data, shape + 1, strides + 1, - ndim - 1, inc) + refcount_objects_in_slice(data, shape + 1, strides + 1, ndim - 1, inc) - data += strides[0] + data += stride # ### Scalar to slice assignment @@ -1398,17 +1404,16 @@ cdef void refcount_objects_in_slice(char *data, Py_ssize_t *shape, @cname('__pyx_memoryview_slice_assign_scalar') cdef void slice_assign_scalar({{memviewslice_name}} *dst, int ndim, size_t itemsize, void *item, - bint dtype_is_object) nogil: - refcount_copying(dst, dtype_is_object, ndim, False) - _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, - itemsize, item) - refcount_copying(dst, dtype_is_object, ndim, True) + bint dtype_is_object) noexcept nogil: + refcount_copying(dst, dtype_is_object, ndim, inc=False) + _slice_assign_scalar(dst.data, dst.shape, dst.strides, ndim, itemsize, item) + refcount_copying(dst, dtype_is_object, ndim, inc=True) @cname('__pyx_memoryview__slice_assign_scalar') cdef void _slice_assign_scalar(char *data, Py_ssize_t *shape, Py_ssize_t *strides, int ndim, - size_t itemsize, void *item) nogil: + size_t itemsize, void *item) noexcept nogil: cdef Py_ssize_t i cdef Py_ssize_t stride = strides[0] cdef Py_ssize_t extent = shape[0] @@ -1419,8 +1424,7 @@ cdef void _slice_assign_scalar(char *data, Py_ssize_t *shape, data += stride else: for i in range(extent): - _slice_assign_scalar(data, shape + 1, strides + 1, - ndim - 1, itemsize, item) + _slice_assign_scalar(data, shape + 1, strides + 1, ndim - 1, itemsize, item) data += stride @@ -1433,27 +1437,27 @@ cdef extern from *: __PYX_BUF_FLAGS_INTEGER_COMPLEX ctypedef struct __Pyx_TypeInfo: - char* name - __Pyx_StructField* fields - size_t size - size_t arraysize[8] - int ndim - char typegroup - char is_unsigned - int flags + char* name + __Pyx_StructField* fields + size_t size + size_t arraysize[8] + int ndim + char typegroup + char is_unsigned + int flags ctypedef struct __Pyx_StructField: - __Pyx_TypeInfo* type - char* name - size_t offset + __Pyx_TypeInfo* type + char* name + size_t offset ctypedef struct __Pyx_BufFmt_StackElem: - __Pyx_StructField* field - size_t parent_offset + __Pyx_StructField* field + size_t parent_offset #ctypedef struct __Pyx_BufFmt_Context: # __Pyx_StructField root - __Pyx_BufFmt_StackElem* head + __Pyx_BufFmt_StackElem* head struct __pyx_typeinfo_string: char string[3] @@ -1466,6 +1470,7 @@ cdef bytes format_from_typeinfo(__Pyx_TypeInfo *type): cdef __Pyx_StructField *field cdef __pyx_typeinfo_string fmt cdef bytes part, result + cdef Py_ssize_t i if type.typegroup == 'S': assert type.fields != NULL @@ -1487,10 +1492,9 @@ cdef bytes format_from_typeinfo(__Pyx_TypeInfo *type): result = alignment.join(parts) + b'}' else: fmt = __Pyx_TypeInfoToFormat(type) + result = fmt.string if type.arraysize[0]: - extents = [unicode(type.arraysize[i]) for i in range(type.ndim)] - result = (u"(%s)" % u','.join(extents)).encode('ascii') + fmt.string - else: - result = fmt.string + extents = [f"{type.arraysize[i]}" for i in range(type.ndim)] + result = f"({u','.join(extents)})".encode('ascii') + result return result diff --git a/Cython/Utility/MemoryView_C.c b/Cython/Utility/MemoryView_C.c index 1b78b2a4e..774ec1767 100644 --- a/Cython/Utility/MemoryView_C.c +++ b/Cython/Utility/MemoryView_C.c @@ -29,8 +29,57 @@ typedef struct { #define __PYX_CYTHON_ATOMICS_ENABLED() CYTHON_ATOMICS #define __pyx_atomic_int_type int +#define __pyx_nonatomic_int_type int + +// For standard C/C++ atomics, get the headers first so we have ATOMIC_INT_LOCK_FREE +// defined when we decide to use them. +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) && \ + (__STDC_VERSION__ >= 201112L) && \ + !defined(__STDC_NO_ATOMICS__)) + #include <stdatomic.h> +#elif CYTHON_ATOMICS && (defined(__cplusplus) && ( \ + (__cplusplus >= 201103L) || \ + (defined(_MSC_VER) && _MSC_VER >= 1700))) + #include <atomic> +#endif -#if CYTHON_ATOMICS && (__GNUC__ >= 5 || (__GNUC__ == 4 && \ +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) && \ + (__STDC_VERSION__ >= 201112L) && \ + !defined(__STDC_NO_ATOMICS__) && \ + ATOMIC_INT_LOCK_FREE == 2) + // C11 atomics are available. + // Require ATOMIC_INT_LOCK_FREE because I'm nervous about the __pyx_atomic_int[2] + // alignment trick in MemoryView.pyx if it uses mutexes. + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type atomic_int + // TODO - it might be possible to use a less strict memory ordering here + #define __pyx_atomic_incr_aligned(value) atomic_fetch_add(value, 1) + #define __pyx_atomic_decr_aligned(value) atomic_fetch_sub(value, 1) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C atomics" + #endif +#elif CYTHON_ATOMICS && (defined(__cplusplus) && ( \ + (__cplusplus >= 201103L) || \ + /*_MSC_VER 1700 is Visual Studio 2012 */ \ + (defined(_MSC_VER) && _MSC_VER >= 1700)) && \ + ATOMIC_INT_LOCK_FREE == 2) + // C++11 atomics are available. + // Require ATOMIC_INT_LOCK_FREE because I'm nervous about the __pyx_atomic_int[2] + // alignment trick in MemoryView.pyx if it uses mutexes. + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type std::atomic_int + // TODO - it might be possible to use a less strict memory ordering here + #define __pyx_atomic_incr_aligned(value) std::atomic_fetch_add(value, 1) + #define __pyx_atomic_decr_aligned(value) std::atomic_fetch_sub(value, 1) + + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C++ atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C++ atomics" + #endif +#elif CYTHON_ATOMICS && (__GNUC__ >= 5 || (__GNUC__ == 4 && \ (__GNUC_MINOR__ > 1 || \ (__GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL__ >= 2)))) /* gcc >= 4.1.2 */ @@ -40,11 +89,12 @@ typedef struct { #ifdef __PYX_DEBUG_ATOMICS #warning "Using GNU atomics" #endif -#elif CYTHON_ATOMICS && defined(_MSC_VER) && CYTHON_COMPILING_IN_NOGIL +#elif CYTHON_ATOMICS && defined(_MSC_VER) /* msvc */ #include <intrin.h> #undef __pyx_atomic_int_type #define __pyx_atomic_int_type long + #define __pyx_nonatomic_int_type long #pragma intrinsic (_InterlockedExchangeAdd) #define __pyx_atomic_incr_aligned(value) _InterlockedExchangeAdd(value, 1) #define __pyx_atomic_decr_aligned(value) _InterlockedExchangeAdd(value, -1) @@ -109,9 +159,9 @@ static CYTHON_INLINE int __pyx_sub_acquisition_count_locked( #define __pyx_get_slice_count_pointer(memview) (memview->acquisition_count_aligned_p) #define __pyx_get_slice_count(memview) (*__pyx_get_slice_count_pointer(memview)) #define __PYX_INC_MEMVIEW(slice, have_gil) __Pyx_INC_MEMVIEW(slice, have_gil, __LINE__) -#define __PYX_XDEC_MEMVIEW(slice, have_gil) __Pyx_XDEC_MEMVIEW(slice, have_gil, __LINE__) +#define __PYX_XCLEAR_MEMVIEW(slice, have_gil) __Pyx_XCLEAR_MEMVIEW(slice, have_gil, __LINE__) static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *, int, int); -static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *, int, int); +static CYTHON_INLINE void __Pyx_XCLEAR_MEMVIEW({{memviewslice_name}} *, int, int); /////////////// MemviewSliceIndex.proto /////////////// @@ -226,8 +276,9 @@ fail: } static int -__pyx_check_suboffsets(Py_buffer *buf, int dim, CYTHON_UNUSED int ndim, int spec) +__pyx_check_suboffsets(Py_buffer *buf, int dim, int ndim, int spec) { + CYTHON_UNUSED_VAR(ndim); // Todo: without PyBUF_INDIRECT we may not have suboffset information, i.e., the // ptr may not be set to NULL but may be uninitialized? if (spec & __Pyx_MEMVIEW_DIRECT) { @@ -483,47 +534,49 @@ __pyx_sub_acquisition_count_locked(__pyx_atomic_int *acquisition_count, static CYTHON_INLINE void __Pyx_INC_MEMVIEW({{memviewslice_name}} *memslice, int have_gil, int lineno) { - int first_time; + __pyx_nonatomic_int_type old_acquisition_count; struct {{memview_struct_name}} *memview = memslice->memview; - if (unlikely(!memview || (PyObject *) memview == Py_None)) - return; /* allow uninitialized memoryview assignment */ - - if (unlikely(__pyx_get_slice_count(memview) < 0)) - __pyx_fatalerror("Acquisition count is %d (line %d)", - __pyx_get_slice_count(memview), lineno); - - first_time = __pyx_add_acquisition_count(memview) == 0; + if (unlikely(!memview || (PyObject *) memview == Py_None)) { + // Allow uninitialized memoryview assignment and do not ref-count None. + return; + } - if (unlikely(first_time)) { - if (have_gil) { - Py_INCREF((PyObject *) memview); + old_acquisition_count = __pyx_add_acquisition_count(memview); + if (unlikely(old_acquisition_count <= 0)) { + if (likely(old_acquisition_count == 0)) { + // First acquisition => keep the memoryview object alive. + if (have_gil) { + Py_INCREF((PyObject *) memview); + } else { + PyGILState_STATE _gilstate = PyGILState_Ensure(); + Py_INCREF((PyObject *) memview); + PyGILState_Release(_gilstate); + } } else { - PyGILState_STATE _gilstate = PyGILState_Ensure(); - Py_INCREF((PyObject *) memview); - PyGILState_Release(_gilstate); + __pyx_fatalerror("Acquisition count is %d (line %d)", + __pyx_get_slice_count(memview), lineno); } } } -static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *memslice, +static CYTHON_INLINE void __Pyx_XCLEAR_MEMVIEW({{memviewslice_name}} *memslice, int have_gil, int lineno) { - int last_time; + __pyx_nonatomic_int_type old_acquisition_count; struct {{memview_struct_name}} *memview = memslice->memview; if (unlikely(!memview || (PyObject *) memview == Py_None)) { - // we do not ref-count None + // Do not ref-count None. memslice->memview = NULL; return; } - if (unlikely(__pyx_get_slice_count(memview) <= 0)) - __pyx_fatalerror("Acquisition count is %d (line %d)", - __pyx_get_slice_count(memview), lineno); - - last_time = __pyx_sub_acquisition_count(memview) == 1; + old_acquisition_count = __pyx_sub_acquisition_count(memview); memslice->data = NULL; - - if (unlikely(last_time)) { + if (likely(old_acquisition_count > 1)) { + // Still other slices out there => we do not own the reference. + memslice->memview = NULL; + } else if (likely(old_acquisition_count == 1)) { + // Last slice => discard owned Python reference to memoryview object. if (have_gil) { Py_CLEAR(memslice->memview); } else { @@ -532,7 +585,8 @@ static CYTHON_INLINE void __Pyx_XDEC_MEMVIEW({{memviewslice_name}} *memslice, PyGILState_Release(_gilstate); } } else { - memslice->memview = NULL; + __pyx_fatalerror("Acquisition count is %d (line %d)", + __pyx_get_slice_count(memview), lineno); } } @@ -770,7 +824,7 @@ static CYTHON_INLINE PyObject *{{get_function}}(const char *itemp) { {{if from_py_function}} static CYTHON_INLINE int {{set_function}}(const char *itemp, PyObject *obj) { {{dtype}} value = {{from_py_function}}(obj); - if ({{error_condition}}) + if (unlikely({{error_condition}})) return 0; *({{dtype}} *) itemp = value; return 1; @@ -921,7 +975,7 @@ __pyx_fill_slice_{{dtype_name}}({{type_decl}} *p, Py_ssize_t extent, Py_ssize_t ////////// FillStrided1DScalar ////////// /* Fill a slice with a scalar value. The dimension is direct and strided or contiguous */ -/* This can be used as a callback for the memoryview object to efficienty assign a scalar */ +/* This can be used as a callback for the memoryview object to efficiently assign a scalar */ /* Currently unused */ static void __pyx_fill_slice_{{dtype_name}}({{type_decl}} *p, Py_ssize_t extent, Py_ssize_t stride, diff --git a/Cython/Utility/ModuleSetupCode.c b/Cython/Utility/ModuleSetupCode.c index 13bc7a8b3..cc398f6ff 100644 --- a/Cython/Utility/ModuleSetupCode.c +++ b/Cython/Utility/ModuleSetupCode.c @@ -1,3 +1,16 @@ +/////////////// InitLimitedAPI /////////////// + +#if defined(CYTHON_LIMITED_API) && 0 /* disabled: enabling Py_LIMITED_API needs more work */ + #ifndef Py_LIMITED_API + #if CYTHON_LIMITED_API+0 > 0x03030000 + #define Py_LIMITED_API CYTHON_LIMITED_API + #else + #define Py_LIMITED_API 0x03030000 + #endif + #endif +#endif + + /////////////// CModulePreamble /////////////// #include <stddef.h> /* For offsetof */ @@ -5,7 +18,7 @@ #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) #endif -#if !defined(WIN32) && !defined(MS_WINDOWS) +#if !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) #ifndef __stdcall #define __stdcall #endif @@ -29,9 +42,7 @@ #ifndef HAVE_LONG_LONG // CPython has required PY_LONG_LONG support for years, even if HAVE_LONG_LONG is not defined for us - #if PY_VERSION_HEX >= 0x02070000 - #define HAVE_LONG_LONG - #endif + #define HAVE_LONG_LONG #endif #ifndef PY_LONG_LONG @@ -42,14 +53,78 @@ #define Py_HUGE_VAL HUGE_VAL #endif -#ifdef PYPY_VERSION +#if defined(GRAALVM_PYTHON) + /* For very preliminary testing purposes. Most variables are set the same as PyPy. + The existence of this section does not imply that anything works or is even tested */ + // GRAALVM_PYTHON test comes before PyPy test because GraalPython unhelpfully defines PYPY_VERSION + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 1 + #define CYTHON_COMPILING_IN_NOGIL 0 + + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #if PY_VERSION_HEX < 0x03050000 + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #elif !defined(CYTHON_USE_ASYNC_SLOTS) + #define CYTHON_USE_ASYNC_SLOTS 1 + #endif + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS (PY_MAJOR_VERSION >= 3) + #endif + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + +#elif defined(PYPY_VERSION) #define CYTHON_COMPILING_IN_PYPY 1 - #define CYTHON_COMPILING_IN_PYSTON 0 #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 #define CYTHON_COMPILING_IN_NOGIL 0 #undef CYTHON_USE_TYPE_SLOTS #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 #undef CYTHON_USE_PYTYPE_LOOKUP #define CYTHON_USE_PYTYPE_LOOKUP 0 #if PY_VERSION_HEX < 0x03050000 @@ -74,14 +149,23 @@ #define CYTHON_UNPACK_METHODS 0 #undef CYTHON_FAST_THREAD_STATE #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 #undef CYTHON_FAST_PYCALL #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS (PY_MAJOR_VERSION >= 3) + #endif #if PY_VERSION_HEX < 0x03090000 #undef CYTHON_PEP489_MULTI_PHASE_INIT #define CYTHON_PEP489_MULTI_PHASE_INIT 0 #elif !defined(CYTHON_PEP489_MULTI_PHASE_INIT) #define CYTHON_PEP489_MULTI_PHASE_INIT 1 #endif + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 #undef CYTHON_USE_TP_FINALIZE #define CYTHON_USE_TP_FINALIZE 0 #undef CYTHON_USE_DICT_VERSIONS @@ -92,45 +176,60 @@ #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 #endif -#elif defined(PYSTON_VERSION) +#elif defined(CYTHON_LIMITED_API) + // EXPERIMENTAL !! #define CYTHON_COMPILING_IN_PYPY 0 - #define CYTHON_COMPILING_IN_PYSTON 1 #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 1 + #define CYTHON_COMPILING_IN_GRAAL 0 #define CYTHON_COMPILING_IN_NOGIL 0 - #ifndef CYTHON_USE_TYPE_SLOTS - #define CYTHON_USE_TYPE_SLOTS 1 - #endif + // CYTHON_CLINE_IN_TRACEBACK is currently disabled for the Limited API + #undef CYTHON_CLINE_IN_TRACEBACK + #define CYTHON_CLINE_IN_TRACEBACK 0 + + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 1 #undef CYTHON_USE_PYTYPE_LOOKUP #define CYTHON_USE_PYTYPE_LOOKUP 0 #undef CYTHON_USE_ASYNC_SLOTS #define CYTHON_USE_ASYNC_SLOTS 0 #undef CYTHON_USE_PYLIST_INTERNALS #define CYTHON_USE_PYLIST_INTERNALS 0 - #ifndef CYTHON_USE_UNICODE_INTERNALS - #define CYTHON_USE_UNICODE_INTERNALS 1 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 #endif - #undef CYTHON_USE_UNICODE_WRITER - #define CYTHON_USE_UNICODE_WRITER 0 #undef CYTHON_USE_PYLONG_INTERNALS #define CYTHON_USE_PYLONG_INTERNALS 0 #ifndef CYTHON_AVOID_BORROWED_REFS #define CYTHON_AVOID_BORROWED_REFS 0 #endif - #ifndef CYTHON_ASSUME_SAFE_MACROS - #define CYTHON_ASSUME_SAFE_MACROS 1 - #endif - #ifndef CYTHON_UNPACK_METHODS - #define CYTHON_UNPACK_METHODS 1 - #endif + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 #undef CYTHON_FAST_THREAD_STATE #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 #undef CYTHON_FAST_PYCALL #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif #undef CYTHON_PEP489_MULTI_PHASE_INIT #define CYTHON_PEP489_MULTI_PHASE_INIT 0 - #undef CYTHON_USE_TP_FINALIZE - #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 1 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 1 + #endif #undef CYTHON_USE_DICT_VERSIONS #define CYTHON_USE_DICT_VERSIONS 0 #undef CYTHON_USE_EXC_INFO_STACK @@ -141,8 +240,9 @@ #elif defined(PY_NOGIL) #define CYTHON_COMPILING_IN_PYPY 0 - #define CYTHON_COMPILING_IN_PYSTON 0 #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 #define CYTHON_COMPILING_IN_NOGIL 1 #ifndef CYTHON_USE_TYPE_SLOTS @@ -188,18 +288,18 @@ #else #define CYTHON_COMPILING_IN_PYPY 0 - #define CYTHON_COMPILING_IN_PYSTON 0 #define CYTHON_COMPILING_IN_CPYTHON 1 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 #define CYTHON_COMPILING_IN_NOGIL 0 #ifndef CYTHON_USE_TYPE_SLOTS #define CYTHON_USE_TYPE_SLOTS 1 #endif - #if PY_VERSION_HEX < 0x02070000 - // looks like calling _PyType_Lookup() isn't safe in Py<=2.6/3.1 - #undef CYTHON_USE_PYTYPE_LOOKUP - #define CYTHON_USE_PYTYPE_LOOKUP 0 - #elif !defined(CYTHON_USE_PYTYPE_LOOKUP) + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #ifndef CYTHON_USE_PYTYPE_LOOKUP #define CYTHON_USE_PYTYPE_LOOKUP 1 #endif #if PY_MAJOR_VERSION < 3 @@ -208,12 +308,8 @@ #elif !defined(CYTHON_USE_ASYNC_SLOTS) #define CYTHON_USE_ASYNC_SLOTS 1 #endif - #if PY_VERSION_HEX < 0x02070000 - #undef CYTHON_USE_PYLONG_INTERNALS - #define CYTHON_USE_PYLONG_INTERNALS 0 - #elif !defined(CYTHON_USE_PYLONG_INTERNALS) - // PyLong internals changed in Py3.12. - #define CYTHON_USE_PYLONG_INTERNALS (PY_VERSION_HEX < 0x030C00A5) + #ifndef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 1 #endif #ifndef CYTHON_USE_PYLIST_INTERNALS #define CYTHON_USE_PYLIST_INTERNALS 1 @@ -238,33 +334,54 @@ #ifndef CYTHON_UNPACK_METHODS #define CYTHON_UNPACK_METHODS 1 #endif - #if PY_VERSION_HEX >= 0x030B00A4 - #undef CYTHON_FAST_THREAD_STATE - #define CYTHON_FAST_THREAD_STATE 0 - #elif !defined(CYTHON_FAST_THREAD_STATE) - #define CYTHON_FAST_THREAD_STATE 1 + #ifndef CYTHON_FAST_THREAD_STATE + // CPython 3.12a6 made PyThreadState an opaque struct. + #define CYTHON_FAST_THREAD_STATE (PY_VERSION_HEX < 0x030C00A6) + #endif + #ifndef CYTHON_FAST_GIL + // Py3<3.5.2 does not support _PyThreadState_UncheckedGet(). + // FIXME: FastGIL can probably be supported also in CPython 3.12 but needs to be adapted. + #define CYTHON_FAST_GIL (PY_MAJOR_VERSION < 3 || PY_VERSION_HEX >= 0x03060000 && PY_VERSION_HEX < 0x030C00A6) + #endif + #ifndef CYTHON_METH_FASTCALL + // CPython 3.6 introduced METH_FASTCALL but with slightly different + // semantics. It became stable starting from CPython 3.7. + #define CYTHON_METH_FASTCALL (PY_VERSION_HEX >= 0x030700A1) #endif #ifndef CYTHON_FAST_PYCALL - // Python 3.11 deleted localplus argument from frame object, which is used in our - // fast_pycall code - // On Python 3.10 it causes issues when used while profiling/debugging - #define CYTHON_FAST_PYCALL (PY_VERSION_HEX < 0x030A0000) + #define CYTHON_FAST_PYCALL 1 #endif - #ifndef CYTHON_PEP489_MULTI_PHASE_INIT - #define CYTHON_PEP489_MULTI_PHASE_INIT (PY_VERSION_HEX >= 0x03050000) + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 #endif - #ifndef CYTHON_USE_TP_FINALIZE - #define CYTHON_USE_TP_FINALIZE (PY_VERSION_HEX >= 0x030400a1) + #if PY_VERSION_HEX < 0x03050000 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #elif !defined(CYTHON_PEP489_MULTI_PHASE_INIT) + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 #endif - #ifndef CYTHON_USE_DICT_VERSIONS - // The dict version field is now deprecated in Py3.12. - #define CYTHON_USE_DICT_VERSIONS ((PY_VERSION_HEX >= 0x030600B1) && (PY_VERSION_HEX < 0x030C00A5)) + #ifndef CYTHON_USE_MODULE_STATE + // EXPERIMENTAL !! + #define CYTHON_USE_MODULE_STATE 0 #endif - #if PY_VERSION_HEX >= 0x030B00A4 + #if PY_VERSION_HEX < 0x030400a1 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #elif !defined(CYTHON_USE_TP_FINALIZE) + #define CYTHON_USE_TP_FINALIZE 1 + #endif + #if PY_VERSION_HEX < 0x030600B1 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #elif !defined(CYTHON_USE_DICT_VERSIONS) + // Python 3.12a5 deprecated "ma_version_tag" + #define CYTHON_USE_DICT_VERSIONS (PY_VERSION_HEX < 0x030C00A5) + #endif + #if PY_VERSION_HEX < 0x030700A3 #undef CYTHON_USE_EXC_INFO_STACK #define CYTHON_USE_EXC_INFO_STACK 0 #elif !defined(CYTHON_USE_EXC_INFO_STACK) - #define CYTHON_USE_EXC_INFO_STACK (PY_VERSION_HEX >= 0x030700A3) + #define CYTHON_USE_EXC_INFO_STACK 1 #endif #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC #define CYTHON_UPDATE_DESCRIPTOR_DOC 1 @@ -275,6 +392,13 @@ #define CYTHON_FAST_PYCCALL (CYTHON_FAST_PYCALL && PY_VERSION_HEX >= 0x030600B1) #endif +#if !defined(CYTHON_VECTORCALL) +#define CYTHON_VECTORCALL (CYTHON_FAST_PYCCALL && PY_VERSION_HEX >= 0x030800B1) +#endif + +/* Whether to use METH_FASTCALL with a fake backported implementation of vectorcall */ +#define CYTHON_BACKPORT_VECTORCALL (CYTHON_METH_FASTCALL && PY_VERSION_HEX < 0x030800B1) + #if CYTHON_USE_PYLONG_INTERNALS #if PY_MAJOR_VERSION < 3 #include "longintrepr.h" @@ -312,6 +436,17 @@ // unused attribute #ifndef CYTHON_UNUSED + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(maybe_unused) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(maybe_unused) + #define CYTHON_UNUSED [[maybe_unused]] + #endif + #endif + #endif +#endif +#ifndef CYTHON_UNUSED # if defined(__GNUC__) # if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) # define CYTHON_UNUSED __attribute__ ((__unused__)) @@ -325,14 +460,18 @@ # endif #endif -#ifndef CYTHON_MAYBE_UNUSED_VAR +#ifndef CYTHON_UNUSED_VAR # if defined(__cplusplus) - template<class T> void CYTHON_MAYBE_UNUSED_VAR( const T& ) { } + template<class T> void CYTHON_UNUSED_VAR( const T& ) { } # else -# define CYTHON_MAYBE_UNUSED_VAR(x) (void)(x) +# define CYTHON_UNUSED_VAR(x) (void)(x) # endif #endif +#ifndef CYTHON_MAYBE_UNUSED_VAR + #define CYTHON_MAYBE_UNUSED_VAR(x) CYTHON_UNUSED_VAR(x) +#endif + #ifndef CYTHON_NCP_UNUSED # if CYTHON_COMPILING_IN_CPYTHON # define CYTHON_NCP_UNUSED @@ -346,26 +485,50 @@ #ifdef _MSC_VER #ifndef _MSC_STDINT_H_ #if _MSC_VER < 1300 - typedef unsigned char uint8_t; - typedef unsigned int uint32_t; + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; #else - typedef unsigned __int8 uint8_t; - typedef unsigned __int32 uint32_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + #endif + #endif + #if _MSC_VER < 1300 + #ifdef _WIN64 + typedef unsigned long long __pyx_uintptr_t; + #else + typedef unsigned int __pyx_uintptr_t; + #endif + #else + #ifdef _WIN64 + typedef unsigned __int64 __pyx_uintptr_t; + #else + typedef unsigned __int32 __pyx_uintptr_t; #endif #endif #else - #include <stdint.h> + #include <stdint.h> + typedef uintptr_t __pyx_uintptr_t; #endif #ifndef CYTHON_FALLTHROUGH - #if defined(__cplusplus) && __cplusplus >= 201103L - #if __has_cpp_attribute(fallthrough) - #define CYTHON_FALLTHROUGH [[fallthrough]] - #elif __has_cpp_attribute(clang::fallthrough) - #define CYTHON_FALLTHROUGH [[clang::fallthrough]] - #elif __has_cpp_attribute(gnu::fallthrough) - #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(fallthrough) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(fallthrough) + #define CYTHON_FALLTHROUGH [[fallthrough]] + #endif + #endif + + #ifndef CYTHON_FALLTHROUGH + #if __has_cpp_attribute(clang::fallthrough) + #define CYTHON_FALLTHROUGH [[clang::fallthrough]] + #elif __has_cpp_attribute(gnu::fallthrough) + #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #endif #endif #endif @@ -377,7 +540,7 @@ #endif #endif - #if defined(__clang__ ) && defined(__apple_build_version__) + #if defined(__clang__) && defined(__apple_build_version__) #if __apple_build_version__ < 7000000 /* Xcode < 7.0 */ #undef CYTHON_FALLTHROUGH #define CYTHON_FALLTHROUGH @@ -385,6 +548,27 @@ #endif #endif +#ifdef __cplusplus + template <typename T> + struct __PYX_IS_UNSIGNED_IMPL {static const bool value = T(0) < T(-1);}; + #define __PYX_IS_UNSIGNED(type) (__PYX_IS_UNSIGNED_IMPL<type>::value) +#else + #define __PYX_IS_UNSIGNED(type) (((type)-1) > 0) +#endif + +#if CYTHON_COMPILING_IN_PYPY == 1 + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x030A0000) +#else + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000) +#endif +// reinterpret + +// TODO: refactor existing code to use those macros +#define __PYX_REINTERPRET_FUNCION(func_pointer, other_pointer) ((func_pointer)(void(*)(void))(other_pointer)) +// #define __PYX_REINTERPRET_POINTER(pointer_type, pointer) ((pointer_type)(void *)(pointer)) +// #define __PYX_RUNTIME_REINTERPRET(type, var) (*(type *)(&var)) + + /////////////// CInitCode /////////////// // inline attribute @@ -418,7 +602,7 @@ #endif #endif -// Work around clang bug http://stackoverflow.com/questions/21847816/c-invoke-nested-template-class-destructor +// Work around clang bug https://stackoverflow.com/questions/21847816/c-invoke-nested-template-class-destructor template<typename T> void __Pyx_call_destructor(T& x) { x.~T(); @@ -436,8 +620,10 @@ class __Pyx_FakeReference { T *operator&() { return ptr; } operator T&() { return *ptr; } // TODO(robertwb): Delegate all operators (or auto-generate unwrapping code where needed). - template<typename U> bool operator ==(U other) { return *ptr == other; } - template<typename U> bool operator !=(U other) { return *ptr != other; } + template<typename U> bool operator ==(const U& other) const { return *ptr == other; } + template<typename U> bool operator !=(const U& other) const { return *ptr != other; } + template<typename U> bool operator==(const __Pyx_FakeReference<U>& other) const { return *ptr == *other.ptr; } + template<typename U> bool operator!=(const __Pyx_FakeReference<U>& other) const { return *ptr != *other.ptr; } private: T *ptr; }; @@ -445,33 +631,29 @@ class __Pyx_FakeReference { /////////////// PythonCompatibility /////////////// -#if CYTHON_COMPILING_IN_PYPY && PY_VERSION_HEX < 0x02070600 && !defined(Py_OptimizeFlag) - #define Py_OptimizeFlag 0 -#endif - #define __PYX_BUILD_PY_SSIZE_T "n" #define CYTHON_FORMAT_SSIZE_T "z" #if PY_MAJOR_VERSION < 3 #define __Pyx_BUILTIN_MODULE_NAME "__builtin__" - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ - PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #define __Pyx_DefaultClassType PyClass_Type + #define __Pyx_PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ + PyCode_New(a+k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #else #define __Pyx_BUILTIN_MODULE_NAME "builtins" #define __Pyx_DefaultClassType PyType_Type #if PY_VERSION_HEX >= 0x030B00A1 - static CYTHON_INLINE PyCodeObject* __Pyx_PyCode_New(int a, int k, int l, int s, int f, + static CYTHON_INLINE PyCodeObject* __Pyx_PyCode_New(int a, int p, int k, int l, int s, int f, PyObject *code, PyObject *c, PyObject* n, PyObject *v, PyObject *fv, PyObject *cell, PyObject* fn, PyObject *name, int fline, PyObject *lnos) { // TODO - currently written to be simple and work in limited API etc. // A more optimized version would be good PyObject *kwds=NULL, *argcount=NULL, *posonlyargcount=NULL, *kwonlyargcount=NULL; - PyObject *nlocals=NULL, *stacksize=NULL, *flags=NULL, *replace=NULL, *call_result=NULL, *empty=NULL; + PyObject *nlocals=NULL, *stacksize=NULL, *flags=NULL, *replace=NULL, *empty=NULL; const char *fn_cstr=NULL; const char *name_cstr=NULL; - PyCodeObject* co=NULL; + PyCodeObject *co=NULL, *result=NULL; PyObject *type, *value, *traceback; // we must be able to call this while an exception is happening - thus clear then restore the state @@ -480,7 +662,7 @@ class __Pyx_FakeReference { if (!(kwds=PyDict_New())) goto end; if (!(argcount=PyLong_FromLong(a))) goto end; if (PyDict_SetItemString(kwds, "co_argcount", argcount) != 0) goto end; - if (!(posonlyargcount=PyLong_FromLong(0))) goto end; + if (!(posonlyargcount=PyLong_FromLong(p))) goto end; if (PyDict_SetItemString(kwds, "co_posonlyargcount", posonlyargcount) != 0) goto end; if (!(kwonlyargcount=PyLong_FromLong(k))) goto end; if (PyDict_SetItemString(kwds, "co_kwonlyargcount", kwonlyargcount) != 0) goto end; @@ -502,20 +684,14 @@ class __Pyx_FakeReference { if (!(name_cstr=PyUnicode_AsUTF8AndSize(name, NULL))) goto end; if (!(co = PyCode_NewEmpty(fn_cstr, name_cstr, fline))) goto end; - if (!(replace = PyObject_GetAttrString((PyObject*)co, "replace"))) goto cleanup_code_too; - if (!(empty = PyTuple_New(0))) goto cleanup_code_too; // unfortunately __pyx_empty_tuple isn't available here - if (!(call_result = PyObject_Call(replace, empty, kwds))) goto cleanup_code_too; + if (!(replace = PyObject_GetAttrString((PyObject*)co, "replace"))) goto end; + // unfortunately, __pyx_empty_tuple isn't available here + if (!(empty = PyTuple_New(0))) goto end; - Py_XDECREF((PyObject*)co); - co = (PyCodeObject*)call_result; - call_result = NULL; + result = (PyCodeObject*) PyObject_Call(replace, empty, kwds); - if (0) { - cleanup_code_too: - Py_XDECREF((PyObject*)co); - co = NULL; - } - end: + end: + Py_XDECREF((PyObject*) co); Py_XDECREF(kwds); Py_XDECREF(argcount); Py_XDECREF(posonlyargcount); @@ -523,18 +699,55 @@ class __Pyx_FakeReference { Py_XDECREF(nlocals); Py_XDECREF(stacksize); Py_XDECREF(replace); - Py_XDECREF(call_result); Py_XDECREF(empty); if (type) { PyErr_Restore(type, value, traceback); } - return co; + return result; } +#elif PY_VERSION_HEX >= 0x030800B2 && !CYTHON_COMPILING_IN_PYPY + + #define __Pyx_PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ + PyCode_NewWithPosOnlyArgs(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #else - #define __Pyx_PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ + #define __Pyx_PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) \ PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) #endif - #define __Pyx_DefaultClassType PyType_Type +#endif + +#if PY_VERSION_HEX >= 0x030900A4 || defined(Py_IS_TYPE) + #define __Pyx_IS_TYPE(ob, type) Py_IS_TYPE(ob, type) +#else + #define __Pyx_IS_TYPE(ob, type) (((const PyObject*)ob)->ob_type == (type)) +#endif + +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_Is) + #define __Pyx_Py_Is(x, y) Py_Is(x, y) +#else + #define __Pyx_Py_Is(x, y) ((x) == (y)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsNone) + #define __Pyx_Py_IsNone(ob) Py_IsNone(ob) +#else + #define __Pyx_Py_IsNone(ob) __Pyx_Py_Is((ob), Py_None) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsTrue) + #define __Pyx_Py_IsTrue(ob) Py_IsTrue(ob) +#else + #define __Pyx_Py_IsTrue(ob) __Pyx_Py_Is((ob), Py_True) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsFalse) + #define __Pyx_Py_IsFalse(ob) Py_IsFalse(ob) +#else + #define __Pyx_Py_IsFalse(ob) __Pyx_Py_Is((ob), Py_False) +#endif +#define __Pyx_NoneAsNull(obj) (__Pyx_Py_IsNone(obj) ? NULL : (obj)) + +#ifndef CO_COROUTINE + #define CO_COROUTINE 0x80 +#endif +#ifndef CO_ASYNC_GENERATOR + #define CO_ASYNC_GENERATOR 0x200 #endif #ifndef Py_TPFLAGS_CHECKTYPES @@ -549,6 +762,12 @@ class __Pyx_FakeReference { #ifndef Py_TPFLAGS_HAVE_FINALIZE #define Py_TPFLAGS_HAVE_FINALIZE 0 #endif +#ifndef Py_TPFLAGS_SEQUENCE + #define Py_TPFLAGS_SEQUENCE 0 +#endif +#ifndef Py_TPFLAGS_MAPPING + #define Py_TPFLAGS_MAPPING 0 +#endif #ifndef METH_STACKLESS // already defined for Stackless Python (all versions) and C-Python >= 3.7 @@ -572,11 +791,41 @@ class __Pyx_FakeReference { #define __Pyx_PyCFunctionFast _PyCFunctionFast #define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords #endif -#if CYTHON_FAST_PYCCALL -#define __Pyx_PyFastCFunction_Check(func) \ - ((PyCFunction_Check(func) && (METH_FASTCALL == (PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS | METH_STACKLESS))))) + +#if CYTHON_METH_FASTCALL + #define __Pyx_METH_FASTCALL METH_FASTCALL + #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast + #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords +#else + #define __Pyx_METH_FASTCALL METH_VARARGS + #define __Pyx_PyCFunction_FastCall PyCFunction + #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords +#endif + +#if CYTHON_VECTORCALL + #define __pyx_vectorcallfunc vectorcallfunc + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET PY_VECTORCALL_ARGUMENTS_OFFSET + #define __Pyx_PyVectorcall_NARGS(n) PyVectorcall_NARGS((size_t)(n)) +#elif CYTHON_BACKPORT_VECTORCALL + typedef PyObject *(*__pyx_vectorcallfunc)(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames); + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET ((size_t)1 << (8 * sizeof(size_t) - 1)) + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(((size_t)(n)) & ~__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)) +#else + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET 0 + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(n)) +#endif + +// PEP-573: PyCFunction holds reference to defining class (PyCMethodObject) +#if PY_VERSION_HEX < 0x030900B1 + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) ((void)m, PyType_FromSpecWithBases(s, b)) + typedef PyObject *(*__Pyx_PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *); #else -#define __Pyx_PyFastCFunction_Check(func) 0 + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) PyType_FromModuleAndSpec(m, s, b) + #define __Pyx_PyCMethod PyCMethod +#endif +#ifndef METH_METHOD + #define METH_METHOD 0x200 #endif #if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) @@ -585,22 +834,17 @@ class __Pyx_FakeReference { #define PyObject_Realloc(p) PyMem_Realloc(p) #endif -#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030400A1 - #define PyMem_RawMalloc(n) PyMem_Malloc(n) - #define PyMem_RawRealloc(p, n) PyMem_Realloc(p, n) - #define PyMem_RawFree(p) PyMem_Free(p) -#endif - -#if CYTHON_COMPILING_IN_PYSTON - // special C-API functions only in Pyston - #define __Pyx_PyCode_HasFreeVars(co) PyCode_HasFreeVars(co) - #define __Pyx_PyFrame_SetLineNumber(frame, lineno) PyFrame_SetLineNumber(frame, lineno) +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) #else #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) #endif -#if !CYTHON_FAST_THREAD_STATE || PY_VERSION_HEX < 0x02070000 +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyThreadState_Current PyThreadState_Get() +#elif !CYTHON_FAST_THREAD_STATE #define __Pyx_PyThreadState_Current PyThreadState_GET() #elif PY_VERSION_HEX >= 0x03060000 //#elif PY_VERSION_HEX >= 0x03050200 @@ -612,6 +856,25 @@ class __Pyx_FakeReference { #define __Pyx_PyThreadState_Current _PyThreadState_Current #endif +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE void *__Pyx_PyModule_GetState(PyObject *op) +{ + void *result; + + result = PyModule_GetState(op); + if (!result) + Py_FatalError("Couldn't find the module state"); + return result; +} +#endif + +#define __Pyx_PyObject_GetSlot(obj, name, func_ctype) __Pyx_PyType_GetSlot(Py_TYPE(obj), name, func_ctype) +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((func_ctype) PyType_GetSlot((type), Py_##name)) +#else + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((type)->name) +#endif + // TSS (Thread Specific Storage) API #if PY_VERSION_HEX < 0x030700A2 && !defined(PyThread_tss_create) && !defined(Py_tss_NEEDS_INIT) #include "pythread.h" @@ -642,10 +905,40 @@ static CYTHON_INLINE int PyThread_tss_set(Py_tss_t *key, void *value) { static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { return PyThread_get_key_value(*key); } -// PyThread_delete_key_value(key) is equalivalent to PyThread_set_key_value(key, NULL) +// PyThread_delete_key_value(key) is equivalent to PyThread_set_key_value(key, NULL) // PyThread_ReInitTLS() is a no-op #endif /* TSS (Thread Specific Storage) API */ + +#if PY_MAJOR_VERSION < 3 + #if CYTHON_COMPILING_IN_PYPY + #if PYPY_VERSION_NUM < 0x07030600 + #if defined(__cplusplus) && __cplusplus >= 201402L + [[deprecated("`with nogil:` inside a nogil function will not release the GIL in PyPy2 < 7.3.6")]] + #elif defined(__GNUC__) || defined(__clang__) + __attribute__ ((__deprecated__("`with nogil:` inside a nogil function will not release the GIL in PyPy2 < 7.3.6"))) + #elif defined(_MSC_VER) + __declspec(deprecated("`with nogil:` inside a nogil function will not release the GIL in PyPy2 < 7.3.6")) + #endif + static CYTHON_INLINE int PyGILState_Check(void) { + // PyGILState_Check is used to decide whether to release the GIL when we don't + // know that we have it. For PyPy2 it isn't possible to check. + // Therefore assume that we don't have the GIL (which causes us not to release it, + // but is "safe") + return 0; + } + #else // PYPY_VERSION_NUM < 0x07030600 + // PyPy2 >= 7.3.6 has PyGILState_Check + #endif // PYPY_VERSION_NUM < 0x07030600 + #else + // https://stackoverflow.com/a/25666624 + static CYTHON_INLINE int PyGILState_Check(void) { + PyThreadState * tstate = _PyThreadState_Current; + return tstate && (tstate == PyGILState_GetThisThreadState()); + } + #endif +#endif + #if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) #define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) #else @@ -660,14 +953,85 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceDivide(x,y) #endif -#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 && CYTHON_USE_UNICODE_INTERNALS -#define __Pyx_PyDict_GetItemStr(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX > 0x030600B4 && CYTHON_USE_UNICODE_INTERNALS +// _PyDict_GetItem_KnownHash() exists since CPython 3.5, but it was +// dropping exceptions. Since 3.6, exceptions are kept. +#define __Pyx_PyDict_GetItemStrWithError(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) { + PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name); + if (res == NULL) PyErr_Clear(); + return res; +} +#elif PY_MAJOR_VERSION >= 3 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000) +#define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#else +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) { + // This is tricky - we should return a borrowed reference but not swallow non-KeyError exceptions. 8-| + // But: this function is only used in Py2 and older PyPys, + // and currently only for argument parsing and other non-correctness-critical lookups + // and we know that 'name' is an interned 'str' with pre-calculated hash value (only comparisons can fail), + // thus, performance matters more than correctness here, especially in the "not found" case. +#if CYTHON_COMPILING_IN_PYPY + // So we ignore any exceptions in old PyPys ... + return PyDict_GetItem(dict, name); #else -#define __Pyx_PyDict_GetItemStr(dict, name) PyDict_GetItem(dict, name) + // and hack together a stripped-down and modified PyDict_GetItem() in CPython 2. + PyDictEntry *ep; + PyDictObject *mp = (PyDictObject*) dict; + long hash = ((PyStringObject *) name)->ob_shash; + assert(hash != -1); /* hash values of interned strings are always initialised */ + ep = (mp->ma_lookup)(mp, name, hash); + if (ep == NULL) { + // error occurred + return NULL; + } + // found or not found + return ep->me_value; +#endif +} +#define __Pyx_PyDict_GetItemStr PyDict_GetItem #endif -/* new Py3.3 unicode type (PEP 393) */ -#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) +/* Type slots */ + +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetFlags(tp) (((PyTypeObject *)tp)->tp_flags) + #define __Pyx_PyType_HasFeature(type, feature) ((__Pyx_PyType_GetFlags(type) & (feature)) != 0) + #define __Pyx_PyObject_GetIterNextFunc(obj) (Py_TYPE(obj)->tp_iternext) +#else + #define __Pyx_PyType_GetFlags(tp) (PyType_GetFlags((PyTypeObject *)tp)) + #define __Pyx_PyType_HasFeature(type, feature) PyType_HasFeature(type, feature) + #define __Pyx_PyObject_GetIterNextFunc(obj) PyIter_Next +#endif + +#if CYTHON_USE_TYPE_SPECS && PY_VERSION_HEX >= 0x03080000 +// In Py3.8+, instances of heap types need to decref their type on deallocation. +// https://bugs.python.org/issue35810 +#define __Pyx_PyHeapTypeObject_GC_Del(obj) { \ + PyTypeObject *type = Py_TYPE(obj); \ + assert(__Pyx_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)); \ + PyObject_GC_Del(obj); \ + Py_DECREF(type); \ +} +#else +#define __Pyx_PyHeapTypeObject_GC_Del(obj) PyObject_GC_Del(obj) +#endif + +#if CYTHON_COMPILING_IN_LIMITED_API + #define CYTHON_PEP393_ENABLED 1 + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GetLength(u) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((void)u, 1114111U) + #define __Pyx_PyUnicode_KIND(u) ((void)u, (0)) + // __Pyx_PyUnicode_DATA() and __Pyx_PyUnicode_READ() must go together, e.g. for iteration. + #define __Pyx_PyUnicode_DATA(u) ((void*)u) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)k, PyUnicode_ReadChar((PyObject*)(d), i)) + //#define __Pyx_PyUnicode_WRITE(k, d, i, ch) /* not available */ + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GetLength(u)) +#elif PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) + /* new Py3.3 unicode type (PEP 393) */ #define CYTHON_PEP393_ENABLED 1 #if PY_VERSION_HEX >= 0x030C0000 @@ -681,7 +1045,7 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) - #define __Pyx_PyUnicode_KIND(u) PyUnicode_KIND(u) + #define __Pyx_PyUnicode_KIND(u) ((int)PyUnicode_KIND(u)) #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, ch) @@ -704,10 +1068,10 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #define __Pyx_PyUnicode_READY(op) (0) #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) - #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((sizeof(Py_UNICODE) == 2) ? 65535 : 1114111) - #define __Pyx_PyUnicode_KIND(u) (sizeof(Py_UNICODE)) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((sizeof(Py_UNICODE) == 2) ? 65535U : 1114111U) + #define __Pyx_PyUnicode_KIND(u) ((int)sizeof(Py_UNICODE)) #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AS_UNICODE(u)) - /* (void)(k) => avoid unused variable warning due to macro: */ + // (void)(k) => avoid unused variable warning due to macro: #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((Py_UNICODE*)d)[i])) #define __Pyx_PyUnicode_WRITE(k, d, i, ch) (((void)(k)), ((Py_UNICODE*)d)[i] = ch) #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_SIZE(u)) @@ -722,16 +1086,20 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) #endif -#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_Contains) - #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) -#endif - -#if CYTHON_COMPILING_IN_PYPY && !defined(PyByteArray_Check) - #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) -#endif - -#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Format) - #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) +#if CYTHON_COMPILING_IN_PYPY + #if !defined(PyUnicode_DecodeUnicodeEscape) + #define PyUnicode_DecodeUnicodeEscape(s, size, errors) PyUnicode_Decode(s, size, "unicode_escape", errors) + #endif + #if !defined(PyUnicode_Contains) || (PY_MAJOR_VERSION == 2 && PYPY_VERSION_NUM < 0x07030500) + #undef PyUnicode_Contains + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) + #endif + #if !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) + #endif + #if !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) + #endif #endif // ("..." % x) must call PyNumber_Remainder() if x is a string subclass that implements "__rmod__()". @@ -768,10 +1136,16 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #define __Pyx_PyBaseString_CheckExact(obj) (PyString_CheckExact(obj) || PyUnicode_CheckExact(obj)) #endif -#ifndef PySet_CheckExact - #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) +#if CYTHON_COMPILING_IN_CPYTHON + #define __Pyx_PySequence_ListKeepNew(obj) \ + (likely(PyList_CheckExact(obj) && Py_REFCNT(obj) == 1) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#else + #define __Pyx_PySequence_ListKeepNew(obj) PySequence_List(obj) #endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) __Pyx_IS_TYPE(obj, &PySet_Type) +#endif #if PY_VERSION_HEX >= 0x030900A4 #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt) @@ -793,6 +1167,8 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #define PyInt_Type PyLong_Type #define PyInt_Check(op) PyLong_Check(op) #define PyInt_CheckExact(op) PyLong_CheckExact(op) + #define __Pyx_Py3Int_Check(op) PyLong_Check(op) + #define __Pyx_Py3Int_CheckExact(op) PyLong_CheckExact(op) #define PyInt_FromString PyLong_FromString #define PyInt_FromUnicode PyLong_FromUnicode #define PyInt_FromLong PyLong_FromLong @@ -804,6 +1180,9 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask #define PyNumber_Int PyNumber_Long +#else + #define __Pyx_Py3Int_Check(op) (PyLong_Check(op) || PyInt_Check(op)) + #define __Pyx_Py3Int_CheckExact(op) (PyLong_CheckExact(op) || PyInt_CheckExact(op)) #endif #if PY_MAJOR_VERSION >= 3 @@ -825,12 +1204,6 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #define __Pyx_PyInt_AsHash_t __Pyx_PyIndex_AsSsize_t #endif -#if PY_MAJOR_VERSION >= 3 - #define __Pyx_PyMethod_New(func, self, klass) ((self) ? ((void)(klass), PyMethod_New(func, self)) : __Pyx_NewRef(func)) -#else - #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) -#endif - // backport of PyAsyncMethods from Py3.5 to older Py3.x versions // (mis-)using the "tp_reserved" type slot which is re-activated as "tp_as_async" in Py3.5 #if CYTHON_USE_ASYNC_SLOTS @@ -852,6 +1225,11 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #endif +/////////////// IncludeStructmemberH.proto /////////////// + +#include <structmember.h> + + /////////////// SmallCodeConfig.proto /////////////// #ifndef CYTHON_SMALL_CODE @@ -892,14 +1270,18 @@ static CYTHON_INLINE void * PyThread_tss_get(Py_tss_t *key) { #if CYTHON_COMPILING_IN_CPYTHON #define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) __Pyx_IsAnySubtype2(Py_TYPE(obj), (PyTypeObject *)type1, (PyTypeObject *)type2) static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b);/*proto*/ +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b);/*proto*/ static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type);/*proto*/ static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2);/*proto*/ #else #define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) (PyObject_TypeCheck(obj, (PyTypeObject *)type1) || PyObject_TypeCheck(obj, (PyTypeObject *)type2)) #define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) #define __Pyx_PyErr_GivenExceptionMatches2(err, type1, type2) (PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2)) #endif +#define __Pyx_PyErr_ExceptionMatches2(err1, err2) __Pyx_PyErr_GivenExceptionMatches2(__Pyx_PyErr_Occurred(), err1, err2) #define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) @@ -910,7 +1292,7 @@ static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObj #if CYTHON_COMPILING_IN_CPYTHON static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { while (a) { - a = a->tp_base; + a = __Pyx_PyType_GetSlot(a, tp_base, PyTypeObject*); if (a == b) return 1; } @@ -934,6 +1316,24 @@ static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { return __Pyx_InBases(a, b); } +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (cls == a || cls == b) return 1; + mro = cls->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + PyObject *base = PyTuple_GET_ITEM(mro, i); + if (base == (PyObject *)a || base == (PyObject *)b) + return 1; + } + return 0; + } + // should only get here for incompletely initialised types, i.e. never under normal usage patterns + return __Pyx_InBases(cls, a) || __Pyx_InBases(cls, b); +} + #if PY_MAJOR_VERSION == 2 static int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject* exc_type2) { @@ -964,11 +1364,11 @@ static int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc } #else static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { - int res = exc_type1 ? __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type1) : 0; - if (!res) { - res = __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + if (exc_type1) { + return __Pyx_IsAnySubtype2((PyTypeObject*)err, (PyTypeObject*)exc_type1, (PyTypeObject*)exc_type2); + } else { + return __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); } - return res; } #endif @@ -1105,12 +1505,21 @@ static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { return 0; } -static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) { +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *module, const char* from_name, const char* to_name, int allow_none) +#else +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) +#endif +{ PyObject *value = PyObject_GetAttrString(spec, from_name); int result = 0; if (likely(value)) { if (allow_none || value != Py_None) { +#if CYTHON_COMPILING_IN_LIMITED_API + result = PyModule_AddObject(module, to_name, value); +#else result = PyDict_SetItemString(moddict, to_name, value); +#endif } Py_DECREF(value); } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { @@ -1121,8 +1530,9 @@ static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject return result; } -static CYTHON_SMALL_CODE PyObject* ${pymodule_create_func_cname}(PyObject *spec, CYTHON_UNUSED PyModuleDef *def) { +static CYTHON_SMALL_CODE PyObject* ${pymodule_create_func_cname}(PyObject *spec, PyModuleDef *def) { PyObject *module = NULL, *moddict, *modname; + CYTHON_UNUSED_VAR(def); // For now, we only have exactly one module instance. if (__Pyx_check_single_interpreter()) @@ -1137,9 +1547,13 @@ static CYTHON_SMALL_CODE PyObject* ${pymodule_create_func_cname}(PyObject *spec, Py_DECREF(modname); if (unlikely(!module)) goto bad; +#if CYTHON_COMPILING_IN_LIMITED_API + moddict = module; +#else moddict = PyModule_GetDict(module); if (unlikely(!moddict)) goto bad; // moddict is a borrowed reference +#endif if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; @@ -1156,6 +1570,7 @@ bad: /////////////// CodeObjectCache.proto /////////////// +#if !CYTHON_COMPILING_IN_LIMITED_API typedef struct { PyCodeObject* code_object; int code_line; @@ -1172,11 +1587,13 @@ static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL}; static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); static PyCodeObject *__pyx_find_code_object(int code_line); static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); +#endif /////////////// CodeObjectCache /////////////// // Note that errors are simply ignored in the code below. // This is just a cache, if a lookup or insertion fails - so what? +#if !CYTHON_COMPILING_IN_LIMITED_API static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { int start = 0, mid = 0, end = count - 1; if (end >= 0 && code_line > entries[end].code_line) { @@ -1257,9 +1674,11 @@ static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { __pyx_code_cache.count++; Py_INCREF(code_object); } +#endif /////////////// CodeObjectCache.cleanup /////////////// + #if !CYTHON_COMPILING_IN_LIMITED_API if (__pyx_code_cache.entries) { __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; int i, count = __pyx_code_cache.count; @@ -1271,6 +1690,7 @@ static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { } PyMem_Free(entries); } + #endif /////////////// CheckBinaryVersion.proto /////////////// @@ -1311,7 +1731,7 @@ static int __Pyx_check_binary_version(void) { rtversion[i] = rt_from_call[i]; } PyOS_snprintf(message, sizeof(message), - "compiletime version %s of module '%.100s' " + "compile time version %s of module '%.100s' " "does not match runtime version %s", ctversion, __Pyx_MODULE_NAME, rtversion); return PyErr_WarnEx(NULL, message, 1); @@ -1343,11 +1763,11 @@ static CYTHON_INLINE int __Pyx_Is_Little_Endian(void) #if CYTHON_REFNANNY typedef struct { - void (*INCREF)(void*, PyObject*, int); - void (*DECREF)(void*, PyObject*, int); - void (*GOTREF)(void*, PyObject*, int); - void (*GIVEREF)(void*, PyObject*, int); - void* (*SetupContext)(const char*, int, const char*); + void (*INCREF)(void*, PyObject*, Py_ssize_t); + void (*DECREF)(void*, PyObject*, Py_ssize_t); + void (*GOTREF)(void*, PyObject*, Py_ssize_t); + void (*GIVEREF)(void*, PyObject*, Py_ssize_t); + void* (*SetupContext)(const char*, Py_ssize_t, const char*); void (*FinishContext)(void**); } __Pyx_RefNannyAPIStruct; static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; @@ -1357,28 +1777,40 @@ static CYTHON_INLINE int __Pyx_Is_Little_Endian(void) #define __Pyx_RefNannySetupContext(name, acquire_gil) \ if (acquire_gil) { \ PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure(); \ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__); \ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__)); \ PyGILState_Release(__pyx_gilstate_save); \ } else { \ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__); \ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__)); \ + } + #define __Pyx_RefNannyFinishContextNogil() { \ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure(); \ + __Pyx_RefNannyFinishContext(); \ + PyGILState_Release(__pyx_gilstate_save); \ } #else #define __Pyx_RefNannySetupContext(name, acquire_gil) \ - __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), __LINE__, __FILE__) + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__)) + #define __Pyx_RefNannyFinishContextNogil() __Pyx_RefNannyFinishContext() #endif + #define __Pyx_RefNannyFinishContextNogil() { \ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure(); \ + __Pyx_RefNannyFinishContext(); \ + PyGILState_Release(__pyx_gilstate_save); \ + } #define __Pyx_RefNannyFinishContext() \ __Pyx_RefNanny->FinishContext(&__pyx_refnanny) - #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), __LINE__) - #define __Pyx_XINCREF(r) do { if((r) != NULL) {__Pyx_INCREF(r); }} while(0) - #define __Pyx_XDECREF(r) do { if((r) != NULL) {__Pyx_DECREF(r); }} while(0) - #define __Pyx_XGOTREF(r) do { if((r) != NULL) {__Pyx_GOTREF(r); }} while(0) - #define __Pyx_XGIVEREF(r) do { if((r) != NULL) {__Pyx_GIVEREF(r);}} while(0) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_XINCREF(r) do { if((r) == NULL); else {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) == NULL); else {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) == NULL); else {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) == NULL); else {__Pyx_GIVEREF(r);}} while(0) #else #define __Pyx_RefNannyDeclarations #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContextNogil() #define __Pyx_RefNannyFinishContext() #define __Pyx_INCREF(r) Py_INCREF(r) #define __Pyx_DECREF(r) Py_DECREF(r) @@ -1390,6 +1822,10 @@ static CYTHON_INLINE int __Pyx_Is_Little_Endian(void) #define __Pyx_XGIVEREF(r) #endif /* CYTHON_REFNANNY */ +#define __Pyx_Py_XDECREF_SET(r, v) do { \ + PyObject *tmp = (PyObject *) r; \ + r = v; Py_XDECREF(tmp); \ + } while (0) #define __Pyx_XDECREF_SET(r, v) do { \ PyObject *tmp = (PyObject *) r; \ r = v; __Pyx_XDECREF(tmp); \ @@ -1449,7 +1885,8 @@ static int __Pyx_RegisterCleanup(void); /*proto*/ //@substitute: naming #if PY_MAJOR_VERSION < 3 || CYTHON_COMPILING_IN_PYPY -static PyObject* ${cleanup_cname}_atexit(PyObject *module, CYTHON_UNUSED PyObject *unused) { +static PyObject* ${cleanup_cname}_atexit(PyObject *module, PyObject *unused) { + CYTHON_UNUSED_VAR(unused); ${cleanup_cname}(module); Py_INCREF(Py_None); return Py_None; } @@ -1537,6 +1974,8 @@ __Pyx_FastGilFuncInit(); /////////////// FastGil.proto /////////////// //@proto_block: utility_code_proto_before_types +#if CYTHON_FAST_GIL + struct __Pyx_FastGilVtab { PyGILState_STATE (*Fast_PyGILState_Ensure)(void); void (*Fast_PyGILState_Release)(PyGILState_STATE oldstate); @@ -1571,8 +2010,15 @@ static void __Pyx_FastGilFuncInit(void); #endif #endif +#else +#define __Pyx_PyGILState_Ensure PyGILState_Ensure +#define __Pyx_PyGILState_Release PyGILState_Release +#define __Pyx_FastGIL_Remember() +#define __Pyx_FastGIL_Forget() +#define __Pyx_FastGilFuncInit() +#endif + /////////////// FastGil /////////////// -//@requires: CommonStructures.c::FetchCommonPointer // The implementations of PyGILState_Ensure/Release calls PyThread_get_key_value // several times which is turns out to be quite slow (slower in fact than // acquiring the GIL itself). Simply storing it in a thread local for the @@ -1580,15 +2026,13 @@ static void __Pyx_FastGilFuncInit(void); // To make optimal use of this thread local, we attempt to share it between // modules. -#define __Pyx_FastGIL_ABI_module "_cython_" CYTHON_ABI +#if CYTHON_FAST_GIL + +#define __Pyx_FastGIL_ABI_module __PYX_ABI_MODULE_NAME #define __Pyx_FastGIL_PyCapsuleName "FastGilFuncs" #define __Pyx_FastGIL_PyCapsule \ __Pyx_FastGIL_ABI_module "." __Pyx_FastGIL_PyCapsuleName -#if PY_VERSION_HEX < 0x02070000 - #undef CYTHON_THREAD_LOCAL -#endif - #ifdef CYTHON_THREAD_LOCAL #include "pythread.h" @@ -1676,17 +2120,12 @@ static void __Pyx_FastGilFuncInit0(void) { #else static void __Pyx_FastGilFuncInit0(void) { - CYTHON_UNUSED void* force_use = (void*)&__Pyx_FetchCommonPointer; } #endif static void __Pyx_FastGilFuncInit(void) { -#if PY_VERSION_HEX >= 0x02070000 struct __Pyx_FastGilVtab* shared = (struct __Pyx_FastGilVtab*)PyCapsule_Import(__Pyx_FastGIL_PyCapsule, 1); -#else - struct __Pyx_FastGilVtab* shared = NULL; -#endif if (shared) { __Pyx_FastGilFuncs = *shared; } else { @@ -1694,3 +2133,22 @@ static void __Pyx_FastGilFuncInit(void) { __Pyx_FastGilFuncInit0(); } } + +#endif + +///////////////////// UtilityCodePragmas ///////////////////////// + +#ifdef _MSC_VER +#pragma warning( push ) +/* Warning 4127: conditional expression is constant + * Cython uses constant conditional expressions to allow in inline functions to be optimized at + * compile-time, so this warning is not useful + */ +#pragma warning( disable : 4127 ) +#endif + +///////////////////// UtilityCodePragmasEnd ////////////////////// + +#ifdef _MSC_VER +#pragma warning( pop ) /* undo whatever Cython has done to warnings */ +#endif diff --git a/Cython/Utility/NumpyImportArray.c b/Cython/Utility/NumpyImportArray.c new file mode 100644 index 000000000..4f72d0b46 --- /dev/null +++ b/Cython/Utility/NumpyImportArray.c @@ -0,0 +1,46 @@ +///////////////////////// NumpyImportArray.init //////////////////// + +// comment below is deliberately kept in the generated C file to +// help users debug where this came from: +/* + * Cython has automatically inserted a call to _import_array since + * you didn't include one when you cimported numpy. To disable this + * add the line + * <void>numpy._import_array + */ +#ifdef NPY_FEATURE_VERSION /* This is a public define that makes us reasonably confident it's "real" Numpy */ +// NO_IMPORT_ARRAY is Numpy's mechanism for indicating that import_array is handled elsewhere +#if !NO_IMPORT_ARRAY /* https://docs.scipy.org/doc/numpy-1.17.0/reference/c-api.array.html#c.NO_IMPORT_ARRAY */ +if (unlikely(_import_array() == -1)) { + PyErr_SetString(PyExc_ImportError, "numpy.core.multiarray failed to import " + "(auto-generated because you didn't call 'numpy.import_array()' after cimporting numpy; " + "use '<void>numpy._import_array' to disable if you are certain you don't need it)."); +} +#endif +#endif + +///////////////////////// NumpyImportUFunc.init //////////////////// + +// Unlike import_array, this is generated by the @cython.ufunc decorator +// so we're confident the right headers are present and don't need to override them + +{ + // NO_IMPORT_UFUNC is Numpy's mechanism for indicating that import_umath is handled elsewhere +#if !NO_IMPORT_UFUNC /* https://numpy.org/devdocs/reference/c-api/ufunc.html#c.NO_IMPORT_UFUNC */ + if (unlikely(_import_umath() == -1)) { + PyErr_SetString(PyExc_ImportError, "numpy.core.umath failed to import " + "(auto-generated by @cython.ufunc)."); + } +#else + if ((0)) {} +#endif + // NO_IMPORT_ARRAY is Numpy's mechanism for indicating that import_array is handled elsewhere +#if !NO_IMPORT_ARRAY /* https://docs.scipy.org/doc/numpy-1.17.0/reference/c-api.array.html#c.NO_IMPORT_ARRAY */ + else if (unlikely(_import_array() == -1)) { + PyErr_SetString(PyExc_ImportError, "numpy.core.multiarray failed to import " + "(auto-generated by @cython.ufunc)."); + } +#endif +} + + diff --git a/Cython/Utility/ObjectHandling.c b/Cython/Utility/ObjectHandling.c index 02574e46e..e97569895 100644 --- a/Cython/Utility/ObjectHandling.c +++ b/Cython/Utility/ObjectHandling.c @@ -132,7 +132,7 @@ static int __Pyx_unpack_tuple2_generic(PyObject* tuple, PyObject** pvalue1, PyOb if (unlikely(!iter)) goto bad; if (decref_tuple) { Py_DECREF(tuple); tuple = NULL; } - iternext = Py_TYPE(iter)->tp_iternext; + iternext = __Pyx_PyObject_GetIterNextFunc(iter); value1 = iternext(iter); if (unlikely(!value1)) { index = 0; goto unpacking_failed; } value2 = iternext(iter); if (unlikely(!value2)) { index = 1; goto unpacking_failed; } if (!has_known_size && unlikely(__Pyx_IternextUnpackEndCheck(iternext(iter), 2))) goto bad; @@ -184,8 +184,10 @@ static PyObject *__Pyx_PyIter_Next2Default(PyObject* defval) { } static void __Pyx_PyIter_Next_ErrorNoIterator(PyObject *iterator) { + __Pyx_TypeName iterator_type_name = __Pyx_PyType_GetName(Py_TYPE(iterator)); PyErr_Format(PyExc_TypeError, - "%.200s object is not an iterator", Py_TYPE(iterator)->tp_name); + __Pyx_FMT_TYPENAME " object is not an iterator", iterator_type_name); + __Pyx_DECREF_TypeName(iterator_type_name); } // originally copied from Py3's builtin_next() @@ -198,10 +200,8 @@ static CYTHON_INLINE PyObject *__Pyx_PyIter_Next2(PyObject* iterator, PyObject* next = iternext(iterator); if (likely(next)) return next; - #if PY_VERSION_HEX >= 0x02070000 if (unlikely(iternext == &_PyObject_NextNotImplemented)) return NULL; - #endif #else // Since the slot was set, assume that PyIter_Next() will likely succeed, and properly fail otherwise. // Note: PyIter_Next() crashes in CPython if "tp_iternext" is NULL. @@ -274,24 +274,21 @@ static CYTHON_INLINE int __Pyx_IterFinish(void) { /////////////// ObjectGetItem.proto /////////////// #if CYTHON_USE_TYPE_SLOTS -static CYTHON_INLINE PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject* key);/*proto*/ +static CYTHON_INLINE PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject *key);/*proto*/ #else #define __Pyx_PyObject_GetItem(obj, key) PyObject_GetItem(obj, key) #endif /////////////// ObjectGetItem /////////////// // //@requires: GetItemInt - added in IndexNode as it uses templating. +//@requires: PyObjectGetAttrStrNoError +//@requires: PyObjectCallOneArg #if CYTHON_USE_TYPE_SLOTS -static PyObject *__Pyx_PyObject_GetIndex(PyObject *obj, PyObject* index) { +static PyObject *__Pyx_PyObject_GetIndex(PyObject *obj, PyObject *index) { + // Get element from sequence object `obj` at index `index`. PyObject *runerr = NULL; Py_ssize_t key_value; - PySequenceMethods *m = Py_TYPE(obj)->tp_as_sequence; - if (unlikely(!(m && m->sq_item))) { - PyErr_Format(PyExc_TypeError, "'%.200s' object is not subscriptable", Py_TYPE(obj)->tp_name); - return NULL; - } - key_value = __Pyx_PyIndex_AsSsize_t(index); if (likely(key_value != -1 || !(runerr = PyErr_Occurred()))) { return __Pyx_GetItemInt_Fast(obj, key_value, 0, 1, 1); @@ -299,18 +296,46 @@ static PyObject *__Pyx_PyObject_GetIndex(PyObject *obj, PyObject* index) { // Error handling code -- only manage OverflowError differently. if (PyErr_GivenExceptionMatches(runerr, PyExc_OverflowError)) { + __Pyx_TypeName index_type_name = __Pyx_PyType_GetName(Py_TYPE(index)); PyErr_Clear(); - PyErr_Format(PyExc_IndexError, "cannot fit '%.200s' into an index-sized integer", Py_TYPE(index)->tp_name); + PyErr_Format(PyExc_IndexError, + "cannot fit '" __Pyx_FMT_TYPENAME "' into an index-sized integer", index_type_name); + __Pyx_DECREF_TypeName(index_type_name); + } + return NULL; +} + +static PyObject *__Pyx_PyObject_GetItem_Slow(PyObject *obj, PyObject *key) { + __Pyx_TypeName obj_type_name; + // Handles less common slow-path checks for GetItem + if (likely(PyType_Check(obj))) { + PyObject *meth = __Pyx_PyObject_GetAttrStrNoError(obj, PYIDENT("__class_getitem__")); + if (meth) { + PyObject *result = __Pyx_PyObject_CallOneArg(meth, key); + Py_DECREF(meth); + return result; + } } + + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "'" __Pyx_FMT_TYPENAME "' object is not subscriptable", obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); return NULL; } -static PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject* key) { - PyMappingMethods *m = Py_TYPE(obj)->tp_as_mapping; - if (likely(m && m->mp_subscript)) { - return m->mp_subscript(obj, key); +static PyObject *__Pyx_PyObject_GetItem(PyObject *obj, PyObject *key) { + PyTypeObject *tp = Py_TYPE(obj); + PyMappingMethods *mm = tp->tp_as_mapping; + PySequenceMethods *sm = tp->tp_as_sequence; + + if (likely(mm && mm->mp_subscript)) { + return mm->mp_subscript(obj, key); + } + if (likely(sm && sm->sq_item)) { + return __Pyx_PyObject_GetIndex(obj, key); } - return __Pyx_PyObject_GetIndex(obj, key); + return __Pyx_PyObject_GetItem_Slow(obj, key); } #endif @@ -357,6 +382,7 @@ static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key) { #endif /////////////// GetItemInt.proto /////////////// +//@substitute: tempita #define __Pyx_GetItemInt(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck) \ (__Pyx_fits_Py_ssize_t(i, type, is_signed) ? \ @@ -379,10 +405,11 @@ static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, int wraparound, int boundscheck); /////////////// GetItemInt /////////////// +//@substitute: tempita static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { PyObject *r; - if (!j) return NULL; + if (unlikely(!j)) return NULL; r = PyObject_GetItem(o, j); Py_DECREF(j); return r; @@ -430,10 +457,18 @@ static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, } } else { // inlined PySequence_GetItem() + special cased length overflow - PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; - if (likely(m && m->sq_item)) { - if (wraparound && unlikely(i < 0) && likely(m->sq_length)) { - Py_ssize_t l = m->sq_length(o); + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if (mm && mm->mp_subscript) { + PyObject *r, *key = PyInt_FromSsize_t(i); + if (unlikely(!key)) return NULL; + r = mm->mp_subscript(o, key); + Py_DECREF(key); + return r; + } + if (likely(sm && sm->sq_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); if (likely(l >= 0)) { i += l; } else { @@ -443,7 +478,7 @@ static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, PyErr_Clear(); } } - return m->sq_item(o, i); + return sm->sq_item(o, i); } } #else @@ -470,7 +505,7 @@ static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObje static int __Pyx_SetItemInt_Generic(PyObject *o, PyObject *j, PyObject *v) { int r; - if (!j) return -1; + if (unlikely(!j)) return -1; r = PyObject_SetItem(o, j, v); Py_DECREF(j); return r; @@ -490,10 +525,19 @@ static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObje } } else { // inlined PySequence_SetItem() + special cased length overflow - PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; - if (likely(m && m->sq_ass_item)) { - if (wraparound && unlikely(i < 0) && likely(m->sq_length)) { - Py_ssize_t l = m->sq_length(o); + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if (mm && mm->mp_ass_subscript) { + int r; + PyObject *key = PyInt_FromSsize_t(i); + if (unlikely(!key)) return -1; + r = mm->mp_ass_subscript(o, key, v); + Py_DECREF(key); + return r; + } + if (likely(sm && sm->sq_ass_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); if (likely(l >= 0)) { i += l; } else { @@ -503,7 +547,7 @@ static CYTHON_INLINE int __Pyx_SetItemInt_Fast(PyObject *o, Py_ssize_t i, PyObje PyErr_Clear(); } } - return m->sq_ass_item(o, i, v); + return sm->sq_ass_item(o, i, v); } } #else @@ -536,24 +580,29 @@ static CYTHON_INLINE int __Pyx_DelItemInt_Fast(PyObject *o, Py_ssize_t i, static int __Pyx_DelItem_Generic(PyObject *o, PyObject *j) { int r; - if (!j) return -1; + if (unlikely(!j)) return -1; r = PyObject_DelItem(o, j); Py_DECREF(j); return r; } static CYTHON_INLINE int __Pyx_DelItemInt_Fast(PyObject *o, Py_ssize_t i, - CYTHON_UNUSED int is_list, CYTHON_NCP_UNUSED int wraparound) { + int is_list, CYTHON_NCP_UNUSED int wraparound) { #if !CYTHON_USE_TYPE_SLOTS if (is_list || PySequence_Check(o)) { return PySequence_DelItem(o, i); } #else // inlined PySequence_DelItem() + special cased length overflow - PySequenceMethods *m = Py_TYPE(o)->tp_as_sequence; - if (likely(m && m->sq_ass_item)) { - if (wraparound && unlikely(i < 0) && likely(m->sq_length)) { - Py_ssize_t l = m->sq_length(o); + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if ((!is_list) && mm && mm->mp_ass_subscript) { + PyObject *key = PyInt_FromSsize_t(i); + return likely(key) ? mm->mp_ass_subscript(o, key, (PyObject *)NULL) : -1; + } + if (likely(sm && sm->sq_ass_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); if (likely(l >= 0)) { i += l; } else { @@ -563,7 +612,7 @@ static CYTHON_INLINE int __Pyx_DelItemInt_Fast(PyObject *o, Py_ssize_t i, PyErr_Clear(); } } - return m->sq_ass_item(o, i, (PyObject *)NULL); + return sm->sq_ass_item(o, i, (PyObject *)NULL); } #endif return __Pyx_DelItem_Generic(o, PyInt_FromSsize_t(i)); @@ -598,7 +647,8 @@ static CYTHON_INLINE int __Pyx_PyObject_SetSlice(PyObject* obj, PyObject* value, {{endif}} Py_ssize_t cstart, Py_ssize_t cstop, PyObject** _py_start, PyObject** _py_stop, PyObject** _py_slice, - int has_cstart, int has_cstop, CYTHON_UNUSED int wraparound) { + int has_cstart, int has_cstop, int wraparound) { + __Pyx_TypeName obj_type_name; #if CYTHON_USE_TYPE_SLOTS PyMappingMethods* mp; #if PY_MAJOR_VERSION < 3 @@ -642,6 +692,8 @@ static CYTHON_INLINE int __Pyx_PyObject_SetSlice(PyObject* obj, PyObject* value, return ms->sq_ass_slice(obj, cstart, cstop, value); {{endif}} } +#else + CYTHON_UNUSED_VAR(wraparound); #endif mp = Py_TYPE(obj)->tp_as_mapping; @@ -650,6 +702,8 @@ static CYTHON_INLINE int __Pyx_PyObject_SetSlice(PyObject* obj, PyObject* value, {{else}} if (likely(mp && mp->mp_ass_subscript)) {{endif}} +#else + CYTHON_UNUSED_VAR(wraparound); #endif { {{if access == 'Get'}}PyObject*{{else}}int{{endif}} result; @@ -701,19 +755,70 @@ static CYTHON_INLINE int __Pyx_PyObject_SetSlice(PyObject* obj, PyObject* value, } return result; } + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); PyErr_Format(PyExc_TypeError, {{if access == 'Get'}} - "'%.200s' object is unsliceable", Py_TYPE(obj)->tp_name); + "'" __Pyx_FMT_TYPENAME "' object is unsliceable", obj_type_name); {{else}} - "'%.200s' object does not support slice %.10s", - Py_TYPE(obj)->tp_name, value ? "assignment" : "deletion"); + "'" __Pyx_FMT_TYPENAME "' object does not support slice %.10s", + obj_type_name, value ? "assignment" : "deletion"); {{endif}} + __Pyx_DECREF_TypeName(obj_type_name); bad: return {{if access == 'Get'}}NULL{{else}}-1{{endif}}; } +/////////////// TupleAndListFromArray.proto /////////////// + +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n); +static CYTHON_INLINE PyObject* __Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n); +#endif + +/////////////// TupleAndListFromArray /////////////// +//@substitute: naming + +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE void __Pyx_copy_object_array(PyObject *const *CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) { + PyObject *v; + Py_ssize_t i; + for (i = 0; i < length; i++) { + v = dest[i] = src[i]; + Py_INCREF(v); + } +} + +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + Py_INCREF($empty_tuple); + return $empty_tuple; + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyTupleObject*)res)->ob_item, n); + return res; +} + +static CYTHON_INLINE PyObject * +__Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return PyList_New(0); + } + res = PyList_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyListObject*)res)->ob_item, n); + return res; +} +#endif + + /////////////// SliceTupleAndList.proto /////////////// #if CYTHON_COMPILING_IN_CPYTHON @@ -725,6 +830,8 @@ static CYTHON_INLINE PyObject* __Pyx_PyTuple_GetSlice(PyObject* src, Py_ssize_t #endif /////////////// SliceTupleAndList /////////////// +//@requires: TupleAndListFromArray +//@substitute: tempita #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE void __Pyx_crop_slice(Py_ssize_t* _start, Py_ssize_t* _stop, Py_ssize_t* _length) { @@ -745,32 +852,18 @@ static CYTHON_INLINE void __Pyx_crop_slice(Py_ssize_t* _start, Py_ssize_t* _stop *_stop = stop; } -static CYTHON_INLINE void __Pyx_copy_object_array(PyObject** CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) { - PyObject *v; - Py_ssize_t i; - for (i = 0; i < length; i++) { - v = dest[i] = src[i]; - Py_INCREF(v); - } -} - {{for type in ['List', 'Tuple']}} static CYTHON_INLINE PyObject* __Pyx_Py{{type}}_GetSlice( PyObject* src, Py_ssize_t start, Py_ssize_t stop) { - PyObject* dest; Py_ssize_t length = Py{{type}}_GET_SIZE(src); __Pyx_crop_slice(&start, &stop, &length); - if (unlikely(length <= 0)) - return Py{{type}}_New(0); - - dest = Py{{type}}_New(length); - if (unlikely(!dest)) - return NULL; - __Pyx_copy_object_array( - ((Py{{type}}Object*)src)->ob_item + start, - ((Py{{type}}Object*)dest)->ob_item, - length); - return dest; +{{if type=='List'}} + if (length <= 0) { + // Avoid undefined behaviour when accessing `ob_item` of an empty list. + return PyList_New(0); + } +{{endif}} + return __Pyx_Py{{type}}_FromArray(((Py{{type}}Object*)src)->ob_item + start, length); } {{endfor}} #endif @@ -912,10 +1005,14 @@ static PyObject *__Pyx_CreateClass(PyObject *bases, PyObject *dict, PyObject *na PyObject *result; PyObject *metaclass; - if (PyDict_SetItem(dict, PYIDENT("__module__"), modname) < 0) + if (unlikely(PyDict_SetItem(dict, PYIDENT("__module__"), modname) < 0)) return NULL; - if (PyDict_SetItem(dict, PYIDENT("__qualname__"), qualname) < 0) +#if PY_VERSION_HEX >= 0x03030000 + if (unlikely(PyDict_SetItem(dict, PYIDENT("__qualname__"), qualname) < 0)) return NULL; +#else + CYTHON_MAYBE_UNUSED_VAR(qualname); +#endif /* Python2 __metaclass__ */ metaclass = __Pyx_PyDict_GetItemStr(dict, PYIDENT("__metaclass__")); @@ -936,6 +1033,94 @@ static PyObject *__Pyx_CreateClass(PyObject *bases, PyObject *dict, PyObject *na return result; } +/////////////// Py3UpdateBases.proto /////////////// + +static PyObject* __Pyx_PEP560_update_bases(PyObject *bases); /* proto */ + +/////////////// Py3UpdateBases ///////////////////// +//@requires: PyObjectCallOneArg +//@requires: PyObjectGetAttrStrNoError + +/* Shamelessly adapted from cpython/bltinmodule.c update_bases */ +static PyObject* +__Pyx_PEP560_update_bases(PyObject *bases) +{ + Py_ssize_t i, j, size_bases; + PyObject *base, *meth, *new_base, *result, *new_bases = NULL; + /*assert(PyTuple_Check(bases));*/ + + size_bases = PyTuple_GET_SIZE(bases); + for (i = 0; i < size_bases; i++) { + // original code in CPython: base = args[i]; + base = PyTuple_GET_ITEM(bases, i); + if (PyType_Check(base)) { + if (new_bases) { + // If we already have made a replacement, then we append every normal base, + // otherwise just skip it. + if (PyList_Append(new_bases, base) < 0) { + goto error; + } + } + continue; + } + // original code in CPython: + // if (_PyObject_LookupAttrId(base, &PyId___mro_entries__, &meth) < 0) { + meth = __Pyx_PyObject_GetAttrStrNoError(base, PYIDENT("__mro_entries__")); + if (!meth && PyErr_Occurred()) { + goto error; + } + if (!meth) { + if (new_bases) { + if (PyList_Append(new_bases, base) < 0) { + goto error; + } + } + continue; + } + new_base = __Pyx_PyObject_CallOneArg(meth, bases); + Py_DECREF(meth); + if (!new_base) { + goto error; + } + if (!PyTuple_Check(new_base)) { + PyErr_SetString(PyExc_TypeError, + "__mro_entries__ must return a tuple"); + Py_DECREF(new_base); + goto error; + } + if (!new_bases) { + // If this is a first successful replacement, create new_bases list and + // copy previously encountered bases. + if (!(new_bases = PyList_New(i))) { + goto error; + } + for (j = 0; j < i; j++) { + // original code in CPython: base = args[j]; + base = PyTuple_GET_ITEM(bases, j); + PyList_SET_ITEM(new_bases, j, base); + Py_INCREF(base); + } + } + j = PyList_GET_SIZE(new_bases); + if (PyList_SetSlice(new_bases, j, j, new_base) < 0) { + goto error; + } + Py_DECREF(new_base); + } + if (!new_bases) { + // unlike the CPython implementation, always return a new reference + Py_INCREF(bases); + return bases; + } + result = PyList_AsTuple(new_bases); + Py_DECREF(new_bases); + return result; + +error: + Py_XDECREF(new_bases); + return NULL; +} + /////////////// Py3ClassCreate.proto /////////////// static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, PyObject *qualname, @@ -944,27 +1129,27 @@ static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObj PyObject *mkw, int calculate_metaclass, int allow_py2_metaclass); /*proto*/ /////////////// Py3ClassCreate /////////////// -//@requires: PyObjectGetAttrStr +//@substitute: naming +//@requires: PyObjectGetAttrStrNoError //@requires: CalculateMetaclass +//@requires: PyObjectFastCall +//@requires: PyObjectCall2Args +//@requires: PyObjectLookupSpecial +// only in fallback code: +//@requires: GetBuiltinName static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, PyObject *qualname, PyObject *mkw, PyObject *modname, PyObject *doc) { PyObject *ns; if (metaclass) { - PyObject *prep = __Pyx_PyObject_GetAttrStr(metaclass, PYIDENT("__prepare__")); + PyObject *prep = __Pyx_PyObject_GetAttrStrNoError(metaclass, PYIDENT("__prepare__")); if (prep) { - PyObject *pargs = PyTuple_Pack(2, name, bases); - if (unlikely(!pargs)) { - Py_DECREF(prep); - return NULL; - } - ns = PyObject_Call(prep, pargs, mkw); + PyObject *pargs[3] = {NULL, name, bases}; + ns = __Pyx_PyObject_FastCallDict(prep, pargs+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, mkw); Py_DECREF(prep); - Py_DECREF(pargs); } else { - if (unlikely(!PyErr_ExceptionMatches(PyExc_AttributeError))) + if (unlikely(PyErr_Occurred())) return NULL; - PyErr_Clear(); ns = PyDict_New(); } } else { @@ -976,7 +1161,11 @@ static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, /* Required here to emulate assignment order */ if (unlikely(PyObject_SetItem(ns, PYIDENT("__module__"), modname) < 0)) goto bad; +#if PY_VERSION_HEX >= 0x03030000 if (unlikely(PyObject_SetItem(ns, PYIDENT("__qualname__"), qualname) < 0)) goto bad; +#else + CYTHON_MAYBE_UNUSED_VAR(qualname); +#endif if (unlikely(doc && PyObject_SetItem(ns, PYIDENT("__doc__"), doc) < 0)) goto bad; return ns; bad: @@ -984,11 +1173,164 @@ bad: return NULL; } +#if PY_VERSION_HEX < 0x030600A4 && CYTHON_PEP487_INIT_SUBCLASS +// https://www.python.org/dev/peps/pep-0487/ +static int __Pyx_SetNamesPEP487(PyObject *type_obj) { + PyTypeObject *type = (PyTypeObject*) type_obj; + PyObject *names_to_set, *key, *value, *set_name, *tmp; + Py_ssize_t i = 0; + +#if CYTHON_USE_TYPE_SLOTS + names_to_set = PyDict_Copy(type->tp_dict); +#else + { + PyObject *d = PyObject_GetAttr(type_obj, PYIDENT("__dict__")); + names_to_set = NULL; + if (likely(d)) { + // d may not be a dict, e.g. PyDictProxy in PyPy2. + PyObject *names_to_set = PyDict_New(); + int ret = likely(names_to_set) ? PyDict_Update(names_to_set, d) : -1; + Py_DECREF(d); + if (unlikely(ret < 0)) + Py_CLEAR(names_to_set); + } + } +#endif + if (unlikely(names_to_set == NULL)) + goto bad; + + while (PyDict_Next(names_to_set, &i, &key, &value)) { + set_name = __Pyx_PyObject_LookupSpecialNoError(value, PYIDENT("__set_name__")); + if (unlikely(set_name != NULL)) { + tmp = __Pyx_PyObject_Call2Args(set_name, type_obj, key); + Py_DECREF(set_name); + if (unlikely(tmp == NULL)) { + __Pyx_TypeName value_type_name = + __Pyx_PyType_GetName(Py_TYPE(value)); + __Pyx_TypeName type_name = __Pyx_PyType_GetName(type); + PyErr_Format(PyExc_RuntimeError, +#if PY_MAJOR_VERSION >= 3 + "Error calling __set_name__ on '" __Pyx_FMT_TYPENAME "' instance %R " "in '" __Pyx_FMT_TYPENAME "'", + value_type_name, key, type_name); +#else + "Error calling __set_name__ on '" __Pyx_FMT_TYPENAME "' instance %.100s in '" __Pyx_FMT_TYPENAME "'", + value_type_name, + PyString_Check(key) ? PyString_AS_STRING(key) : "?", + type_name); +#endif + goto bad; + } else { + Py_DECREF(tmp); + } + } + else if (unlikely(PyErr_Occurred())) { + goto bad; + } + } + + Py_DECREF(names_to_set); + return 0; +bad: + Py_XDECREF(names_to_set); + return -1; +} + +static PyObject *__Pyx_InitSubclassPEP487(PyObject *type_obj, PyObject *mkw) { +#if CYTHON_USE_TYPE_SLOTS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS +// Stripped-down version of "super(type_obj, type_obj).__init_subclass__(**mkw)" in CPython 3.8. + PyTypeObject *type = (PyTypeObject*) type_obj; + PyObject *mro = type->tp_mro; + Py_ssize_t i, nbases; + if (unlikely(!mro)) goto done; + + // avoid "unused" warning + (void) &__Pyx_GetBuiltinName; + + Py_INCREF(mro); + nbases = PyTuple_GET_SIZE(mro); + + // Skip over the type itself and 'object'. + assert(PyTuple_GET_ITEM(mro, 0) == type_obj); + for (i = 1; i < nbases-1; i++) { + PyObject *base, *dict, *meth; + base = PyTuple_GET_ITEM(mro, i); + dict = ((PyTypeObject *)base)->tp_dict; + meth = __Pyx_PyDict_GetItemStrWithError(dict, PYIDENT("__init_subclass__")); + if (unlikely(meth)) { + descrgetfunc f = Py_TYPE(meth)->tp_descr_get; + PyObject *res; + Py_INCREF(meth); + if (likely(f)) { + res = f(meth, NULL, type_obj); + Py_DECREF(meth); + if (unlikely(!res)) goto bad; + meth = res; + } + res = __Pyx_PyObject_FastCallDict(meth, NULL, 0, mkw); + Py_DECREF(meth); + if (unlikely(!res)) goto bad; + Py_DECREF(res); + goto done; + } else if (unlikely(PyErr_Occurred())) { + goto bad; + } + } + +done: + Py_XDECREF(mro); + return type_obj; + +bad: + Py_XDECREF(mro); + Py_DECREF(type_obj); + return NULL; + +// CYTHON_USE_TYPE_SLOTS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS +#else +// Generic fallback: "super(type_obj, type_obj).__init_subclass__(**mkw)", as used in CPython 3.8. + PyObject *super_type, *super, *func, *res; + +#if CYTHON_COMPILING_IN_PYPY && !defined(PySuper_Type) + super_type = __Pyx_GetBuiltinName(PYIDENT("super")); +#else + super_type = (PyObject*) &PySuper_Type; + // avoid "unused" warning + (void) &__Pyx_GetBuiltinName; +#endif + super = likely(super_type) ? __Pyx_PyObject_Call2Args(super_type, type_obj, type_obj) : NULL; +#if CYTHON_COMPILING_IN_PYPY && !defined(PySuper_Type) + Py_XDECREF(super_type); +#endif + if (unlikely(!super)) { + Py_CLEAR(type_obj); + goto done; + } + func = __Pyx_PyObject_GetAttrStrNoError(super, PYIDENT("__init_subclass__")); + Py_DECREF(super); + if (likely(!func)) { + if (unlikely(PyErr_Occurred())) + Py_CLEAR(type_obj); + goto done; + } + res = __Pyx_PyObject_FastCallDict(func, NULL, 0, mkw); + Py_DECREF(func); + if (unlikely(!res)) + Py_CLEAR(type_obj); + Py_XDECREF(res); +done: + return type_obj; +#endif +} + +// PY_VERSION_HEX < 0x030600A4 && CYTHON_PEP487_INIT_SUBCLASS +#endif + static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases, PyObject *dict, PyObject *mkw, int calculate_metaclass, int allow_py2_metaclass) { - PyObject *result, *margs; + PyObject *result; PyObject *owned_metaclass = NULL; + PyObject *margs[4] = {NULL, name, bases, dict}; if (allow_py2_metaclass) { /* honour Python2 __metaclass__ for backward compatibility */ owned_metaclass = PyObject_GetItem(dict, PYIDENT("__metaclass__")); @@ -1007,14 +1349,28 @@ static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObj return NULL; owned_metaclass = metaclass; } - margs = PyTuple_Pack(3, name, bases, dict); - if (unlikely(!margs)) { - result = NULL; - } else { - result = PyObject_Call(metaclass, margs, mkw); - Py_DECREF(margs); - } + result = __Pyx_PyObject_FastCallDict(metaclass, margs+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, +#if PY_VERSION_HEX < 0x030600A4 + // Before PEP-487, type(a,b,c) did not accept any keyword arguments, so guard at least against that case. + (metaclass == (PyObject*)&PyType_Type) ? NULL : mkw +#else + mkw +#endif + ); Py_XDECREF(owned_metaclass); + +#if PY_VERSION_HEX < 0x030600A4 && CYTHON_PEP487_INIT_SUBCLASS + if (likely(result) && likely(PyType_Check(result))) { + if (unlikely(__Pyx_SetNamesPEP487(result) < 0)) { + Py_CLEAR(result); + } else { + result = __Pyx_InitSubclassPEP487(result, mkw); + } + } +#else + // avoid "unused" warning + (void) &__Pyx_GetBuiltinName; +#endif return result; } @@ -1025,14 +1381,21 @@ static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type); /*pr /////////////// ExtTypeTest /////////////// static CYTHON_INLINE int __Pyx_TypeTest(PyObject *obj, PyTypeObject *type) { + __Pyx_TypeName obj_type_name; + __Pyx_TypeName type_name; if (unlikely(!type)) { PyErr_SetString(PyExc_SystemError, "Missing type object"); return 0; } if (likely(__Pyx_TypeCheck(obj, type))) return 1; - PyErr_Format(PyExc_TypeError, "Cannot convert %.200s to %.200s", - Py_TYPE(obj)->tp_name, type->tp_name); + obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + type_name = __Pyx_PyType_GetName(type); + PyErr_Format(PyExc_TypeError, + "Cannot convert " __Pyx_FMT_TYPENAME " to " __Pyx_FMT_TYPENAME, + obj_type_name, type_name); + __Pyx_DECREF_TypeName(obj_type_name); + __Pyx_DECREF_TypeName(type_name); return 0; } @@ -1100,12 +1463,12 @@ static CYTHON_INLINE PyObject* __Pyx_PyBoolOrNull_FromLong(long b) { static PyObject *__Pyx_GetBuiltinName(PyObject *name); /*proto*/ /////////////// GetBuiltinName /////////////// -//@requires: PyObjectGetAttrStr +//@requires: PyObjectGetAttrStrNoError //@substitute: naming static PyObject *__Pyx_GetBuiltinName(PyObject *name) { - PyObject* result = __Pyx_PyObject_GetAttrStr($builtins_cname, name); - if (unlikely(!result)) { + PyObject* result = __Pyx_PyObject_GetAttrStrNoError($builtins_cname, name); + if (unlikely(!result) && !PyErr_Occurred()) { PyErr_Format(PyExc_NameError, #if PY_MAJOR_VERSION >= 3 "name '%U' is not defined", name); @@ -1122,29 +1485,27 @@ static PyObject *__Pyx_GetBuiltinName(PyObject *name) { static PyObject *__Pyx__GetNameInClass(PyObject *nmspace, PyObject *name); /*proto*/ /////////////// GetNameInClass /////////////// -//@requires: PyObjectGetAttrStr //@requires: GetModuleGlobalName -//@requires: Exceptions.c::PyThreadStateGet -//@requires: Exceptions.c::PyErrFetchRestore -//@requires: Exceptions.c::PyErrExceptionMatches - -static PyObject *__Pyx_GetGlobalNameAfterAttributeLookup(PyObject *name) { - PyObject *result; - __Pyx_PyThreadState_declare - __Pyx_PyThreadState_assign - if (unlikely(!__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) - return NULL; - __Pyx_PyErr_Clear(); - __Pyx_GetModuleGlobalNameUncached(result, name); - return result; -} static PyObject *__Pyx__GetNameInClass(PyObject *nmspace, PyObject *name) { PyObject *result; - result = __Pyx_PyObject_GetAttrStr(nmspace, name); - if (!result) { - result = __Pyx_GetGlobalNameAfterAttributeLookup(name); + PyObject *dict; + assert(PyType_Check(nmspace)); +#if CYTHON_USE_TYPE_SLOTS + dict = ((PyTypeObject*)nmspace)->tp_dict; + Py_XINCREF(dict); +#else + dict = PyObject_GetAttr(nmspace, PYIDENT("__dict__")); +#endif + if (likely(dict)) { + result = PyObject_GetItem(dict, name); + Py_DECREF(dict); + if (result) { + return result; + } } + PyErr_Clear(); + __Pyx_GetModuleGlobalNameUncached(result, name); return result; } @@ -1162,6 +1523,30 @@ static PyObject *__Pyx__GetNameInClass(PyObject *nmspace, PyObject *name) { #define __Pyx_SetNameInClass(ns, name, value) PyObject_SetItem(ns, name, value) #endif +/////////////// SetNewInClass.proto /////////////// + +static int __Pyx_SetNewInClass(PyObject *ns, PyObject *name, PyObject *value); + +/////////////// SetNewInClass /////////////// +//@requires: SetNameInClass + +// Special-case setting __new__: if it's a Cython function, wrap it in a +// staticmethod. This is similar to what Python does for a Python function +// called __new__. +static int __Pyx_SetNewInClass(PyObject *ns, PyObject *name, PyObject *value) { +#ifdef __Pyx_CyFunction_USED + int ret; + if (__Pyx_CyFunction_Check(value)) { + PyObject *staticnew = PyStaticMethod_New(value); + if (unlikely(!staticnew)) return -1; + ret = __Pyx_SetNameInClass(ns, name, staticnew); + Py_DECREF(staticnew); + return ret; + } +#endif + return __Pyx_SetNameInClass(ns, name, value); +} + /////////////// GetModuleGlobalName.proto /////////////// //@requires: PyDictVersioning @@ -1199,6 +1584,7 @@ static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) #endif { PyObject *result; +// FIXME: clean up the macro guard order here: limited API first, then borrowed refs, then cpython #if !CYTHON_AVOID_BORROWED_REFS #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030500A1 // Identifier names are always interned and have a pre-calculated hash value. @@ -1209,6 +1595,14 @@ static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) } else if (unlikely(PyErr_Occurred())) { return NULL; } +#elif CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(!$module_cname)) { + return NULL; + } + result = PyObject_GetAttr($module_cname, name); + if (likely(result)) { + return result; + } #else result = PyDict_GetItem($moddict_cname, name); __PYX_UPDATE_DICT_CACHE($moddict_cname, result, *dict_cached_value, *dict_version) @@ -1246,16 +1640,31 @@ static CYTHON_INLINE PyObject *__Pyx_GetAttr(PyObject *o, PyObject *n) { return PyObject_GetAttr(o, n); } + /////////////// PyObjectLookupSpecial.proto /////////////// + +#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS +#define __Pyx_PyObject_LookupSpecialNoError(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 0) +#define __Pyx_PyObject_LookupSpecial(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 1) + +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error); /*proto*/ + +#else +#define __Pyx_PyObject_LookupSpecialNoError(o,n) __Pyx_PyObject_GetAttrStrNoError(o,n) +#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) +#endif + +/////////////// PyObjectLookupSpecial /////////////// //@requires: PyObjectGetAttrStr +//@requires: PyObjectGetAttrStrNoError #if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS -static CYTHON_INLINE PyObject* __Pyx_PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name) { +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error) { PyObject *res; PyTypeObject *tp = Py_TYPE(obj); #if PY_MAJOR_VERSION < 3 if (unlikely(PyInstance_Check(obj))) - return __Pyx_PyObject_GetAttrStr(obj, attr_name); + return with_error ? __Pyx_PyObject_GetAttrStr(obj, attr_name) : __Pyx_PyObject_GetAttrStrNoError(obj, attr_name); #endif // adapted from CPython's special_lookup() in ceval.c res = _PyType_Lookup(tp, attr_name); @@ -1266,13 +1675,11 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_LookupSpecial(PyObject* obj, PyObj } else { res = f(res, obj, (PyObject *)tp); } - } else { + } else if (with_error) { PyErr_SetObject(PyExc_AttributeError, attr_name); } return res; } -#else -#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) #endif @@ -1291,19 +1698,21 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj #if CYTHON_USE_TYPE_SLOTS && CYTHON_USE_PYTYPE_LOOKUP && PY_VERSION_HEX < 0x03070000 static PyObject *__Pyx_RaiseGenericGetAttributeError(PyTypeObject *tp, PyObject *attr_name) { + __Pyx_TypeName type_name = __Pyx_PyType_GetName(tp); PyErr_Format(PyExc_AttributeError, #if PY_MAJOR_VERSION >= 3 - "'%.50s' object has no attribute '%U'", - tp->tp_name, attr_name); + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%U'", + type_name, attr_name); #else - "'%.50s' object has no attribute '%.400s'", - tp->tp_name, PyString_AS_STRING(attr_name)); + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%.400s'", + type_name, PyString_AS_STRING(attr_name)); #endif + __Pyx_DECREF_TypeName(type_name); return NULL; } static CYTHON_INLINE PyObject* __Pyx_PyObject_GenericGetAttrNoDict(PyObject* obj, PyObject* attr_name) { - // Copied and adapted from _PyObject_GenericGetAttrWithDict() in CPython 2.6/3.7. + // Copied and adapted from _PyObject_GenericGetAttrWithDict() in CPython 3.6/3.7. // To be used in the "tp_getattro" slot of extension types that have no instance dict and cannot be subclassed. PyObject *descr; PyTypeObject *tp = Py_TYPE(obj); @@ -1455,6 +1864,7 @@ static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **me static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) { PyObject *attr; #if CYTHON_UNPACK_METHODS && CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_PYTYPE_LOOKUP + __Pyx_TypeName type_name; // Copied from _PyObject_GetMethod() in CPython 3.7 PyTypeObject *tp = Py_TYPE(obj); PyObject *descr; @@ -1475,12 +1885,14 @@ static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **me descr = _PyType_Lookup(tp, name); if (likely(descr != NULL)) { Py_INCREF(descr); +#if defined(Py_TPFLAGS_METHOD_DESCRIPTOR) && Py_TPFLAGS_METHOD_DESCRIPTOR + if (__Pyx_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)) +#elif PY_MAJOR_VERSION >= 3 // Repeating the condition below accommodates for MSVC's inability to test macros inside of macro expansions. -#if PY_MAJOR_VERSION >= 3 #ifdef __Pyx_CyFunction_USED - if (likely(PyFunction_Check(descr) || (Py_TYPE(descr) == &PyMethodDescr_Type) || __Pyx_CyFunction_Check(descr))) + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type) || __Pyx_CyFunction_Check(descr))) #else - if (likely(PyFunction_Check(descr) || (Py_TYPE(descr) == &PyMethodDescr_Type))) + if (likely(PyFunction_Check(descr) || __Pyx_IS_TYPE(descr, &PyMethodDescr_Type))) #endif #else // "PyMethodDescr_Type" is not part of the C-API in Py2. @@ -1526,19 +1938,21 @@ static int __Pyx_PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **me goto try_unpack; } - if (descr != NULL) { + if (likely(descr != NULL)) { *method = descr; return 0; } + type_name = __Pyx_PyType_GetName(tp); PyErr_Format(PyExc_AttributeError, #if PY_MAJOR_VERSION >= 3 - "'%.50s' object has no attribute '%U'", - tp->tp_name, name); + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%U'", + type_name, name); #else - "'%.50s' object has no attribute '%.400s'", - tp->tp_name, PyString_AS_STRING(name)); + "'" __Pyx_FMT_TYPENAME "' object has no attribute '%.400s'", + type_name, PyString_AS_STRING(name)); #endif + __Pyx_DECREF_TypeName(type_name); return 0; // Generic fallback implementation using normal attribute lookup. @@ -1578,23 +1992,77 @@ typedef struct { /////////////// UnpackUnboundCMethod /////////////// //@requires: PyObjectGetAttrStr +static PyObject *__Pyx_SelflessCall(PyObject *method, PyObject *args, PyObject *kwargs) { + // NOTE: possible optimization - use vectorcall + PyObject *selfless_args = PyTuple_GetSlice(args, 1, PyTuple_Size(args)); + if (unlikely(!selfless_args)) return NULL; + + PyObject *result = PyObject_Call(method, selfless_args, kwargs); + Py_DECREF(selfless_args); + return result; +} + +static PyMethodDef __Pyx_UnboundCMethod_Def = { + /* .ml_name = */ "CythonUnboundCMethod", + /* .ml_meth = */ __PYX_REINTERPRET_FUNCION(PyCFunction, __Pyx_SelflessCall), + /* .ml_flags = */ METH_VARARGS | METH_KEYWORDS, + /* .ml_doc = */ NULL +}; + static int __Pyx_TryUnpackUnboundCMethod(__Pyx_CachedCFunction* target) { PyObject *method; method = __Pyx_PyObject_GetAttrStr(target->type, *target->method_name); if (unlikely(!method)) return -1; target->method = method; +// FIXME: use functionality from CythonFunction.c/ClassMethod #if CYTHON_COMPILING_IN_CPYTHON #if PY_MAJOR_VERSION >= 3 - // method dscriptor type isn't exported in Py2.x, cannot easily check the type there if (likely(__Pyx_TypeCheck(method, &PyMethodDescr_Type))) + #else + // method descriptor type isn't exported in Py2.x, cannot easily check the type there. + // Therefore, reverse the check to the most likely alternative + // (which is returned for class methods) + if (likely(!PyCFunction_Check(method))) #endif { PyMethodDescrObject *descr = (PyMethodDescrObject*) method; target->func = descr->d_method->ml_meth; target->flag = descr->d_method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_STACKLESS); - } + } else +#endif + // bound classmethods need special treatment +#if defined(CYTHON_COMPILING_IN_PYPY) + // In PyPy functions are regular methods, so just do + // the self check +#elif PY_VERSION_HEX >= 0x03090000 + if (PyCFunction_CheckExact(method)) +#else + if (PyCFunction_Check(method)) #endif + { + PyObject *self; + int self_found; +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + self = PyObject_GetAttrString(method, "__self__"); + if (!self) { + PyErr_Clear(); + } +#else + self = PyCFunction_GET_SELF(method); +#endif + self_found = (self && self != Py_None); +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + Py_XDECREF(self); +#endif + if (self_found) { + PyObject *unbound_method = PyCFunction_New(&__Pyx_UnboundCMethod_Def, method); + if (unlikely(!unbound_method)) return -1; + // New PyCFunction will own method reference, thus decref __Pyx_PyObject_GetAttrStr + Py_DECREF(method); + target->method = unbound_method; + } + } return 0; } @@ -1666,13 +2134,13 @@ static CYTHON_INLINE PyObject* __Pyx_CallUnboundCMethod1(__Pyx_CachedCFunction* // Not using #ifdefs for PY_VERSION_HEX to avoid C compiler warnings about unused functions. if (flag == METH_O) { return (*(cfunc->func))(self, arg); - } else if (PY_VERSION_HEX >= 0x030600B1 && flag == METH_FASTCALL) { + } else if ((PY_VERSION_HEX >= 0x030600B1) && flag == METH_FASTCALL) { #if PY_VERSION_HEX >= 0x030700A0 return (*(__Pyx_PyCFunctionFast)(void*)(PyCFunction)cfunc->func)(self, &arg, 1); #else return (*(__Pyx_PyCFunctionFastWithKeywords)(void*)(PyCFunction)cfunc->func)(self, &arg, 1, NULL); #endif - } else if (PY_VERSION_HEX >= 0x030700A0 && flag == (METH_FASTCALL | METH_KEYWORDS)) { + } else if ((PY_VERSION_HEX >= 0x030700A0) && flag == (METH_FASTCALL | METH_KEYWORDS)) { return (*(__Pyx_PyCFunctionFastWithKeywords)(void*)(PyCFunction)cfunc->func)(self, &arg, 1, NULL); } } @@ -1784,6 +2252,107 @@ bad: } +/////////////// PyObjectFastCall.proto /////////////// + +#define __Pyx_PyObject_FastCall(func, args, nargs) __Pyx_PyObject_FastCallDict(func, args, (size_t)(nargs), NULL) +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject **args, size_t nargs, PyObject *kwargs); /*proto*/ + +/////////////// PyObjectFastCall /////////////// +//@requires: PyObjectCall +//@requires: PyFunctionFastCall +//@requires: PyObjectCallMethO +//@substitute: naming + +static PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject **args, size_t nargs, PyObject *kwargs) { + PyObject *argstuple; + PyObject *result; + size_t i; + + argstuple = PyTuple_New((Py_ssize_t)nargs); + if (unlikely(!argstuple)) return NULL; + for (i = 0; i < nargs; i++) { + Py_INCREF(args[i]); + PyTuple_SET_ITEM(argstuple, (Py_ssize_t)i, args[i]); + } + result = __Pyx_PyObject_Call(func, argstuple, kwargs); + Py_DECREF(argstuple); + return result; +} + +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject **args, size_t _nargs, PyObject *kwargs) { + // Special fast paths for 0 and 1 arguments + // NOTE: in many cases, this is called with a constant value for nargs + // which is known at compile-time. So the branches below will typically + // be optimized away. + Py_ssize_t nargs = __Pyx_PyVectorcall_NARGS(_nargs); +#if CYTHON_COMPILING_IN_CPYTHON + if (nargs == 0 && kwargs == NULL) { +#if defined(__Pyx_CyFunction_USED) && defined(NDEBUG) + // TODO PyCFunction_GET_FLAGS has a type-check assert that breaks with a CyFunction + // in debug mode. There is likely to be a better way of avoiding tripping this + // check that doesn't involve disabling the optimized path. + if (__Pyx_IsCyOrPyCFunction(func)) +#else + if (PyCFunction_Check(func)) +#endif + { + if (likely(PyCFunction_GET_FLAGS(func) & METH_NOARGS)) { + return __Pyx_PyObject_CallMethO(func, NULL); + } + } + } + else if (nargs == 1 && kwargs == NULL) { + if (PyCFunction_Check(func)) + { + if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { + return __Pyx_PyObject_CallMethO(func, args[0]); + } + } + } +#endif + + #if PY_VERSION_HEX < 0x030800B1 + #if CYTHON_FAST_PYCCALL + if (PyCFunction_Check(func)) { + if (kwargs) { + return _PyCFunction_FastCallDict(func, args, nargs, kwargs); + } else { + return _PyCFunction_FastCallKeywords(func, args, nargs, NULL); + } + } + #if PY_VERSION_HEX >= 0x030700A1 + if (!kwargs && __Pyx_IS_TYPE(func, &PyMethodDescr_Type)) { + return _PyMethodDescr_FastCallKeywords(func, args, nargs, NULL); + } + #endif + #endif + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs); + } + #endif + #endif + + #if CYTHON_VECTORCALL + vectorcallfunc f = _PyVectorcall_Function(func); + if (f) { + return f(func, args, (size_t)nargs, kwargs); + } + #elif defined(__Pyx_CyFunction_USED) && CYTHON_BACKPORT_VECTORCALL + // exclude fused functions for now + if (__Pyx_CyFunction_CheckExact(func)) { + __pyx_vectorcallfunc f = __Pyx_CyFunction_func_vectorcall(func); + if (f) return f(func, args, (size_t)nargs, kwargs); + } + #endif + + if (nargs == 0) { + return __Pyx_PyObject_Call(func, $empty_tuple, kwargs); + } + return __Pyx_PyObject_FastCall_fallback(func, args, (size_t)nargs, kwargs); +} + + /////////////// PyObjectCallMethod0.proto /////////////// static PyObject* __Pyx_PyObject_CallMethod0(PyObject* obj, PyObject* method_name); /*proto*/ @@ -1838,59 +2407,6 @@ static PyObject* __Pyx_PyObject_CallMethod1(PyObject* obj, PyObject* method_name } -/////////////// PyObjectCallMethod2.proto /////////////// - -static PyObject* __Pyx_PyObject_CallMethod2(PyObject* obj, PyObject* method_name, PyObject* arg1, PyObject* arg2); /*proto*/ - -/////////////// PyObjectCallMethod2 /////////////// -//@requires: PyObjectCall -//@requires: PyFunctionFastCall -//@requires: PyCFunctionFastCall -//@requires: PyObjectCall2Args - -static PyObject* __Pyx_PyObject_Call3Args(PyObject* function, PyObject* arg1, PyObject* arg2, PyObject* arg3) { - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(function)) { - PyObject *args[3] = {arg1, arg2, arg3}; - return __Pyx_PyFunction_FastCall(function, args, 3); - } - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(function)) { - PyObject *args[3] = {arg1, arg2, arg3}; - return __Pyx_PyFunction_FastCall(function, args, 3); - } - #endif - - args = PyTuple_New(3); - if (unlikely(!args)) goto done; - Py_INCREF(arg1); - PyTuple_SET_ITEM(args, 0, arg1); - Py_INCREF(arg2); - PyTuple_SET_ITEM(args, 1, arg2); - Py_INCREF(arg3); - PyTuple_SET_ITEM(args, 2, arg3); - - result = __Pyx_PyObject_Call(function, args, NULL); - Py_DECREF(args); - return result; -} - -static PyObject* __Pyx_PyObject_CallMethod2(PyObject* obj, PyObject* method_name, PyObject* arg1, PyObject* arg2) { - PyObject *args, *method = NULL, *result = NULL; - int is_method = __Pyx_PyObject_GetMethod(obj, method_name, &method); - if (likely(is_method)) { - result = __Pyx_PyObject_Call3Args(method, obj, arg1, arg2); - Py_DECREF(method); - return result; - } - if (unlikely(!method)) return NULL; - result = __Pyx_PyObject_Call2Args(method, arg1, arg2); - Py_DECREF(method); - return result; -} - - /////////////// tp_new.proto /////////////// #define __Pyx_tp_new(type_obj, args) __Pyx_tp_new_kwargs(type_obj, args, NULL) @@ -1962,14 +2478,12 @@ static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject /////////////// PyFunctionFastCall.proto /////////////// #if CYTHON_FAST_PYCALL + +#if !CYTHON_VECTORCALL #define __Pyx_PyFunction_FastCall(func, args, nargs) \ __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) -// let's assume that the non-public C-API function might still change during the 3.6 beta phase -#if 1 || PY_VERSION_HEX < 0x030600B1 static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, Py_ssize_t nargs, PyObject *kwargs); -#else -#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) #endif // Backport from Python 3 @@ -1981,7 +2495,7 @@ static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, // ((char *)(foo) \ // + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) // -// Written by Rusty Russell, public domain, http://ccodearchive.net/ +// Written by Rusty Russell, public domain, https://ccodearchive.net/ #define __Pyx_BUILD_ASSERT_EXPR(cond) \ (sizeof(char [1 - 2*!(cond)]) - 1) @@ -1990,10 +2504,8 @@ static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, #define Py_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) #endif -#if CYTHON_FAST_PYCALL - // Initialised by module init code. - static size_t __pyx_pyframe_localsplus_offset = 0; - +#if !CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03080000 #include "frameobject.h" #if PY_VERSION_HEX >= 0x030b00a6 #ifndef Py_BUILD_CORE @@ -2001,10 +2513,16 @@ static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, #endif #include "internal/pycore_frame.h" #endif + #define __Pxy_PyFrame_Initialize_Offsets() + #define __Pyx_PyFrame_GetLocalsplus(frame) ((frame)->f_localsplus) +#else + // Initialised by module init code. + static size_t __pyx_pyframe_localsplus_offset = 0; + #include "frameobject.h" // This is the long runtime version of // #define __Pyx_PyFrame_GetLocalsplus(frame) ((frame)->f_localsplus) - // offsetof(PyFrameObject, f_localsplus) differs between regular C-Python and Stackless Python. + // offsetof(PyFrameObject, f_localsplus) differs between regular C-Python and Stackless Python < 3.8. // Therefore the offset is computed at run time from PyFrame_type.tp_basicsize. That is feasible, // because f_localsplus is the last field of PyFrameObject (checked by Py_BUILD_ASSERT_EXPR below). #define __Pxy_PyFrame_Initialize_Offsets() \ @@ -2012,15 +2530,16 @@ static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, (void)(__pyx_pyframe_localsplus_offset = ((size_t)PyFrame_Type.tp_basicsize) - Py_MEMBER_SIZE(PyFrameObject, f_localsplus))) #define __Pyx_PyFrame_GetLocalsplus(frame) \ (assert(__pyx_pyframe_localsplus_offset), (PyObject **)(((char *)(frame)) + __pyx_pyframe_localsplus_offset)) -#endif // CYTHON_FAST_PYCALL #endif +#endif /* !CYTHON_VECTORCALL */ + +#endif /* CYTHON_FAST_PYCALL */ /////////////// PyFunctionFastCall /////////////// // copied from CPython 3.6 ceval.c -#if CYTHON_FAST_PYCALL - +#if CYTHON_FAST_PYCALL && !CYTHON_VECTORCALL static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, PyObject *globals) { PyFrameObject *f; @@ -2056,7 +2575,6 @@ static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args } -#if 1 || PY_VERSION_HEX < 0x030600B1 static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, Py_ssize_t nargs, PyObject *kwargs) { PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); PyObject *globals = PyFunction_GET_GLOBALS(func); @@ -2077,7 +2595,7 @@ static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, assert(kwargs == NULL || PyDict_Check(kwargs)); nk = kwargs ? PyDict_Size(kwargs) : 0; - if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { + if (unlikely(Py_EnterRecursiveCall((char*)" while calling a Python object"))) { return NULL; } @@ -2166,83 +2684,19 @@ done: Py_LeaveRecursiveCall(); return result; } -#endif /* CPython < 3.6 */ -#endif /* CYTHON_FAST_PYCALL */ - - -/////////////// PyCFunctionFastCall.proto /////////////// - -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); -#else -#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) -#endif - -/////////////// PyCFunctionFastCall /////////////// - -#if CYTHON_FAST_PYCCALL -static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { - PyCFunctionObject *func = (PyCFunctionObject*)func_obj; - PyCFunction meth = PyCFunction_GET_FUNCTION(func); - PyObject *self = PyCFunction_GET_SELF(func); - int flags = PyCFunction_GET_FLAGS(func); - - assert(PyCFunction_Check(func)); - assert(METH_FASTCALL == (flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_KEYWORDS | METH_STACKLESS))); - assert(nargs >= 0); - assert(nargs == 0 || args != NULL); - - /* _PyCFunction_FastCallDict() must not be called with an exception set, - because it may clear it (directly or indirectly) and so the - caller loses its exception */ - assert(!PyErr_Occurred()); - - if ((PY_VERSION_HEX < 0x030700A0) || unlikely(flags & METH_KEYWORDS)) { - return (*((__Pyx_PyCFunctionFastWithKeywords)(void*)meth)) (self, args, nargs, NULL); - } else { - return (*((__Pyx_PyCFunctionFast)(void*)meth)) (self, args, nargs); - } -} -#endif /* CYTHON_FAST_PYCCALL */ +#endif /* CYTHON_FAST_PYCALL && !CYTHON_VECTORCALL */ /////////////// PyObjectCall2Args.proto /////////////// -static CYTHON_UNUSED PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2); /*proto*/ +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2); /*proto*/ /////////////// PyObjectCall2Args /////////////// -//@requires: PyObjectCall -//@requires: PyFunctionFastCall -//@requires: PyCFunctionFastCall - -static CYTHON_UNUSED PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) { - PyObject *args, *result = NULL; - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(function)) { - PyObject *args[2] = {arg1, arg2}; - return __Pyx_PyFunction_FastCall(function, args, 2); - } - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(function)) { - PyObject *args[2] = {arg1, arg2}; - return __Pyx_PyCFunction_FastCall(function, args, 2); - } - #endif +//@requires: PyObjectFastCall - args = PyTuple_New(2); - if (unlikely(!args)) goto done; - Py_INCREF(arg1); - PyTuple_SET_ITEM(args, 0, arg1); - Py_INCREF(arg2); - PyTuple_SET_ITEM(args, 1, arg2); - - Py_INCREF(function); - result = __Pyx_PyObject_Call(function, args, NULL); - Py_DECREF(args); - Py_DECREF(function); -done: - return result; +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) { + PyObject *args[3] = {NULL, arg1, arg2}; + return __Pyx_PyObject_FastCall(function, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); } @@ -2251,91 +2705,97 @@ done: static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); /*proto*/ /////////////// PyObjectCallOneArg /////////////// -//@requires: PyObjectCallMethO -//@requires: PyObjectCall -//@requires: PyFunctionFastCall -//@requires: PyCFunctionFastCall - -#if CYTHON_COMPILING_IN_CPYTHON -static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_New(1); - if (unlikely(!args)) return NULL; - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 0, arg); - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; -} +//@requires: PyObjectFastCall static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, &arg, 1); - } -#endif - if (likely(PyCFunction_Check(func))) { - if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { - // fast and simple case that we are optimising for - return __Pyx_PyObject_CallMethO(func, arg); -#if CYTHON_FAST_PYCCALL - } else if (__Pyx_PyFastCFunction_Check(func)) { - return __Pyx_PyCFunction_FastCall(func, &arg, 1); -#endif - } - } - return __Pyx__PyObject_CallOneArg(func, arg); -} -#else -static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { - PyObject *result; - PyObject *args = PyTuple_Pack(1, arg); - if (unlikely(!args)) return NULL; - result = __Pyx_PyObject_Call(func, args, NULL); - Py_DECREF(args); - return result; + PyObject *args[2] = {NULL, arg}; + return __Pyx_PyObject_FastCall(func, args+1, 1 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); } -#endif /////////////// PyObjectCallNoArg.proto /////////////// -//@requires: PyObjectCall -//@substitute: naming -#if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); /*proto*/ -#else -#define __Pyx_PyObject_CallNoArg(func) __Pyx_PyObject_Call(func, $empty_tuple, NULL) -#endif /////////////// PyObjectCallNoArg /////////////// -//@requires: PyObjectCallMethO -//@requires: PyObjectCall -//@requires: PyFunctionFastCall -//@substitute: naming +//@requires: PyObjectFastCall -#if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { -#if CYTHON_FAST_PYCALL - if (PyFunction_Check(func)) { - return __Pyx_PyFunction_FastCall(func, NULL, 0); - } -#endif -#if defined(__Pyx_CyFunction_USED) && defined(NDEBUG) - // TODO PyCFunction_GET_FLAGS has a type-check assert that breaks with a CyFunction - // in debug mode. There is likely to be a better way of avoiding tripping this - // check that doesn't involve disabling the optimized path. - if (likely(PyCFunction_Check(func) || __Pyx_CyFunction_Check(func))) -#else - if (likely(PyCFunction_Check(func))) + PyObject *arg = NULL; + return __Pyx_PyObject_FastCall(func, (&arg)+1, 0 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + + +/////////////// PyVectorcallFastCallDict.proto /////////////// + +#if CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw); #endif - { - if (likely(PyCFunction_GET_FLAGS(func) & METH_NOARGS)) { - // fast and simple case that we are optimising for - return __Pyx_PyObject_CallMethO(func, NULL); - } + +/////////////// PyVectorcallFastCallDict /////////////// + +#if CYTHON_METH_FASTCALL +// Slow path when kw is non-empty +static PyObject *__Pyx_PyVectorcall_FastCallDict_kw(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + // Code based on _PyObject_FastCallDict() and _PyStack_UnpackDict() from CPython + PyObject *res = NULL; + PyObject *kwnames; + PyObject **newargs; + PyObject **kwvalues; + Py_ssize_t i, pos; + size_t j; + PyObject *key, *value; + unsigned long keys_are_strings; + Py_ssize_t nkw = PyDict_GET_SIZE(kw); + + // Copy positional arguments + newargs = (PyObject **)PyMem_Malloc((nargs + (size_t)nkw) * sizeof(args[0])); + if (unlikely(newargs == NULL)) { + PyErr_NoMemory(); + return NULL; + } + for (j = 0; j < nargs; j++) newargs[j] = args[j]; + + // Copy keyword arguments + kwnames = PyTuple_New(nkw); + if (unlikely(kwnames == NULL)) { + PyMem_Free(newargs); + return NULL; + } + kwvalues = newargs + nargs; + pos = i = 0; + keys_are_strings = Py_TPFLAGS_UNICODE_SUBCLASS; + while (PyDict_Next(kw, &pos, &key, &value)) { + keys_are_strings &= Py_TYPE(key)->tp_flags; + Py_INCREF(key); + Py_INCREF(value); + PyTuple_SET_ITEM(kwnames, i, key); + kwvalues[i] = value; + i++; + } + if (unlikely(!keys_are_strings)) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + goto cleanup; + } + + // The actual call + res = vc(func, newargs, nargs, kwnames); + +cleanup: + Py_DECREF(kwnames); + for (i = 0; i < nkw; i++) + Py_DECREF(kwvalues[i]); + PyMem_Free(newargs); + return res; +} + +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + if (likely(kw == NULL) || PyDict_GET_SIZE(kw) == 0) { + return vc(func, args, nargs, NULL); } - return __Pyx_PyObject_Call(func, $empty_tuple, NULL); + return __Pyx_PyVectorcall_FastCallDict_kw(func, vc, args, nargs, kw); } #endif @@ -2352,10 +2812,9 @@ static PyObject* __Pyx_PyNumber_InPlaceMatrixMultiply(PyObject* x, PyObject* y); #endif /////////////// MatrixMultiply /////////////// -//@requires: PyObjectGetAttrStr +//@requires: PyObjectGetAttrStrNoError //@requires: PyObjectCallOneArg -//@requires: PyFunctionFastCall -//@requires: PyCFunctionFastCall +//@requires: PyObjectCall2Args #if PY_VERSION_HEX < 0x03050000 static PyObject* __Pyx_PyObject_CallMatrixMethod(PyObject* method, PyObject* arg) { @@ -2365,34 +2824,9 @@ static PyObject* __Pyx_PyObject_CallMatrixMethod(PyObject* method, PyObject* arg if (likely(PyMethod_Check(method))) { PyObject *self = PyMethod_GET_SELF(method); if (likely(self)) { - PyObject *args; PyObject *function = PyMethod_GET_FUNCTION(method); - #if CYTHON_FAST_PYCALL - if (PyFunction_Check(function)) { - PyObject *args[2] = {self, arg}; - result = __Pyx_PyFunction_FastCall(function, args, 2); - goto done; - } - #endif - #if CYTHON_FAST_PYCCALL - if (__Pyx_PyFastCFunction_Check(function)) { - PyObject *args[2] = {self, arg}; - result = __Pyx_PyCFunction_FastCall(function, args, 2); - goto done; - } - #endif - args = PyTuple_New(2); - if (unlikely(!args)) goto done; - Py_INCREF(self); - PyTuple_SET_ITEM(args, 0, self); - Py_INCREF(arg); - PyTuple_SET_ITEM(args, 1, arg); - Py_INCREF(function); - Py_DECREF(method); method = NULL; - result = __Pyx_PyObject_Call(function, args, NULL); - Py_DECREF(args); - Py_DECREF(function); - return result; + result = __Pyx_PyObject_Call2Args(function, self, arg); + goto done; } } #endif @@ -2402,21 +2836,21 @@ done: return result; } -#define __Pyx_TryMatrixMethod(x, y, py_method_name) { \ - PyObject *func = __Pyx_PyObject_GetAttrStr(x, py_method_name); \ +#define __Pyx_TryMatrixMethod(x, y, py_method_name) { \ + PyObject *func = __Pyx_PyObject_GetAttrStrNoError(x, py_method_name); \ if (func) { \ PyObject *result = __Pyx_PyObject_CallMatrixMethod(func, y); \ if (result != Py_NotImplemented) \ return result; \ Py_DECREF(result); \ - } else { \ - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) \ - return NULL; \ - PyErr_Clear(); \ + } else if (unlikely(PyErr_Occurred())) { \ + return NULL; \ } \ } static PyObject* __Pyx__PyNumber_MatrixMultiply(PyObject* x, PyObject* y, const char* op_name) { + __Pyx_TypeName x_type_name; + __Pyx_TypeName y_type_name; int right_is_subtype = PyObject_IsSubclass((PyObject*)Py_TYPE(y), (PyObject*)Py_TYPE(x)); if (unlikely(right_is_subtype == -1)) return NULL; @@ -2429,11 +2863,13 @@ static PyObject* __Pyx__PyNumber_MatrixMultiply(PyObject* x, PyObject* y, const if (!right_is_subtype) { __Pyx_TryMatrixMethod(y, x, PYIDENT("__rmatmul__")) } + x_type_name = __Pyx_PyType_GetName(Py_TYPE(x)); + y_type_name = __Pyx_PyType_GetName(Py_TYPE(y)); PyErr_Format(PyExc_TypeError, - "unsupported operand type(s) for %.2s: '%.100s' and '%.100s'", - op_name, - Py_TYPE(x)->tp_name, - Py_TYPE(y)->tp_name); + "unsupported operand type(s) for %.2s: '" __Pyx_FMT_TYPENAME "' and '" + __Pyx_FMT_TYPENAME "'", op_name, x_type_name, y_type_name); + __Pyx_DECREF_TypeName(x_type_name); + __Pyx_DECREF_TypeName(y_type_name); return NULL; } @@ -2504,3 +2940,277 @@ static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UIN return obj_dict_version == __Pyx_get_object_dict_version(obj); } #endif + + +/////////////// PyMethodNew.proto /////////////// + +#if PY_MAJOR_VERSION >= 3 +// This should be an actual function (not a macro), such that we can put it +// directly in a tp_descr_get slot. +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + return PyMethod_New(func, self); +} +#else + #define __Pyx_PyMethod_New PyMethod_New +#endif + +///////////// PyMethodNew2Arg.proto ///////////// + +// Another wrapping of PyMethod_New that matches the Python3 signature +#if PY_MAJOR_VERSION >= 3 +#define __Pyx_PyMethod_New2Arg PyMethod_New +#else +#define __Pyx_PyMethod_New2Arg(func, self) PyMethod_New(func, self, (PyObject*)Py_TYPE(self)) +#endif + +/////////////// UnicodeConcatInPlace.proto //////////////// + +# if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +// __Pyx_PyUnicode_ConcatInPlace may modify the first argument 'left' +// However, unlike `PyUnicode_Append` it will never NULL it. +// It behaves like a regular function - returns a new reference and NULL on error + #if CYTHON_REFNANNY + #define __Pyx_PyUnicode_ConcatInPlace(left, right) __Pyx_PyUnicode_ConcatInPlaceImpl(&left, right, __pyx_refnanny) + #else + #define __Pyx_PyUnicode_ConcatInPlace(left, right) __Pyx_PyUnicode_ConcatInPlaceImpl(&left, right) + #endif + // __Pyx_PyUnicode_ConcatInPlace is slightly odd because it has the potential to modify the input + // argument (but only in cases where no user should notice). Therefore, it needs to keep Cython's + // refnanny informed. + static CYTHON_INLINE PyObject *__Pyx_PyUnicode_ConcatInPlaceImpl(PyObject **p_left, PyObject *right + #if CYTHON_REFNANNY + , void* __pyx_refnanny + #endif + ); /* proto */ +#else +#define __Pyx_PyUnicode_ConcatInPlace __Pyx_PyUnicode_Concat +#endif +#define __Pyx_PyUnicode_ConcatInPlaceSafe(left, right) ((unlikely((left) == Py_None) || unlikely((right) == Py_None)) ? \ + PyNumber_InPlaceAdd(left, right) : __Pyx_PyUnicode_ConcatInPlace(left, right)) + +/////////////// UnicodeConcatInPlace //////////////// +//@substitute: naming + +# if CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 +// copied directly from unicode_object.c "unicode_modifiable +// removing _PyUnicode_HASH since it's a macro we don't have +// - this is OK because trying PyUnicode_Resize on a non-modifyable +// object will still work, it just won't happen in place +static int +__Pyx_unicode_modifiable(PyObject *unicode) +{ + if (Py_REFCNT(unicode) != 1) + return 0; + if (!PyUnicode_CheckExact(unicode)) + return 0; + if (PyUnicode_CHECK_INTERNED(unicode)) + return 0; + return 1; +} + +static CYTHON_INLINE PyObject *__Pyx_PyUnicode_ConcatInPlaceImpl(PyObject **p_left, PyObject *right + #if CYTHON_REFNANNY + , void* __pyx_refnanny + #endif + ) { + // heavily based on PyUnicode_Append + PyObject *left = *p_left; + Py_ssize_t left_len, right_len, new_len; + + if (unlikely(__Pyx_PyUnicode_READY(left) == -1)) + return NULL; + if (unlikely(__Pyx_PyUnicode_READY(right) == -1)) + return NULL; + + // Shortcuts + left_len = PyUnicode_GET_LENGTH(left); + if (left_len == 0) { + Py_INCREF(right); + return right; + } + right_len = PyUnicode_GET_LENGTH(right); + if (right_len == 0) { + Py_INCREF(left); + return left; + } + if (unlikely(left_len > PY_SSIZE_T_MAX - right_len)) { + PyErr_SetString(PyExc_OverflowError, + "strings are too large to concat"); + return NULL; + } + new_len = left_len + right_len; + + if (__Pyx_unicode_modifiable(left) + && PyUnicode_CheckExact(right) + && PyUnicode_KIND(right) <= PyUnicode_KIND(left) + // Don't resize for ascii += latin1. Convert ascii to latin1 requires + // to change the structure size, but characters are stored just after + // the structure, and so it requires to move all characters which is + // not so different than duplicating the string. + && !(PyUnicode_IS_ASCII(left) && !PyUnicode_IS_ASCII(right))) { + + __Pyx_GIVEREF(*p_left); + if (unlikely(PyUnicode_Resize(p_left, new_len) != 0)) { + // on failure PyUnicode_Resize does not deallocate the the input + // so left will remain unchanged - simply undo the giveref + __Pyx_GOTREF(*p_left); + return NULL; + } + __Pyx_INCREF(*p_left); + + // copy 'right' into the newly allocated area of 'left' + _PyUnicode_FastCopyCharacters(*p_left, left_len, right, 0, right_len); + return *p_left; + } else { + return __Pyx_PyUnicode_Concat(left, right); + } + } +#endif + +////////////// StrConcatInPlace.proto /////////////////////// +//@requires: UnicodeConcatInPlace + +#if PY_MAJOR_VERSION >= 3 + // allow access to the more efficient versions where we know str_type is unicode + #define __Pyx_PyStr_Concat __Pyx_PyUnicode_Concat + #define __Pyx_PyStr_ConcatInPlace __Pyx_PyUnicode_ConcatInPlace +#else + #define __Pyx_PyStr_Concat PyNumber_Add + #define __Pyx_PyStr_ConcatInPlace PyNumber_InPlaceAdd +#endif +#define __Pyx_PyStr_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ? \ + PyNumber_Add(a, b) : __Pyx_PyStr_Concat(a, b)) +#define __Pyx_PyStr_ConcatInPlaceSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ? \ + PyNumber_InPlaceAdd(a, b) : __Pyx_PyStr_ConcatInPlace(a, b)) + + +/////////////// PySequenceMultiply.proto /////////////// + +#define __Pyx_PySequence_Multiply_Left(mul, seq) __Pyx_PySequence_Multiply(seq, mul) +static CYTHON_INLINE PyObject* __Pyx_PySequence_Multiply(PyObject *seq, Py_ssize_t mul); + +/////////////// PySequenceMultiply /////////////// + +static PyObject* __Pyx_PySequence_Multiply_Generic(PyObject *seq, Py_ssize_t mul) { + PyObject *result, *pymul = PyInt_FromSsize_t(mul); + if (unlikely(!pymul)) + return NULL; + result = PyNumber_Multiply(seq, pymul); + Py_DECREF(pymul); + return result; +} + +static CYTHON_INLINE PyObject* __Pyx_PySequence_Multiply(PyObject *seq, Py_ssize_t mul) { +#if CYTHON_USE_TYPE_SLOTS + PyTypeObject *type = Py_TYPE(seq); + if (likely(type->tp_as_sequence && type->tp_as_sequence->sq_repeat)) { + return type->tp_as_sequence->sq_repeat(seq, mul); + } else +#endif + { + return __Pyx_PySequence_Multiply_Generic(seq, mul); + } +} + + +/////////////// FormatTypeName.proto /////////////// + +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%U" +static __Pyx_TypeName __Pyx_PyType_GetName(PyTypeObject* tp); /*proto*/ +#define __Pyx_DECREF_TypeName(obj) Py_XDECREF(obj) +#else +typedef const char *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%.200s" +#define __Pyx_PyType_GetName(tp) ((tp)->tp_name) +#define __Pyx_DECREF_TypeName(obj) +#endif + +/////////////// FormatTypeName /////////////// + +#if CYTHON_COMPILING_IN_LIMITED_API +static __Pyx_TypeName +__Pyx_PyType_GetName(PyTypeObject* tp) +{ + PyObject *name = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + PYIDENT("__name__")); + if (unlikely(name == NULL) || unlikely(!PyUnicode_Check(name))) { + PyErr_Clear(); + Py_XSETREF(name, __Pyx_NewRef(PYIDENT("?"))); + } + return name; +} +#endif + + +/////////////// RaiseUnexpectedTypeError.proto /////////////// + +static int __Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj); /*proto*/ + +/////////////// RaiseUnexpectedTypeError /////////////// + +static int +__Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj) +{ + __Pyx_TypeName obj_type_name = __Pyx_PyType_GetName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, "Expected %s, got " __Pyx_FMT_TYPENAME, + expected, obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return 0; +} + + +/////////////// RaiseUnboundLocalError.proto /////////////// +static CYTHON_INLINE void __Pyx_RaiseUnboundLocalError(const char *varname);/*proto*/ + +/////////////// RaiseUnboundLocalError /////////////// +static CYTHON_INLINE void __Pyx_RaiseUnboundLocalError(const char *varname) { + PyErr_Format(PyExc_UnboundLocalError, "local variable '%s' referenced before assignment", varname); +} + + +/////////////// RaiseClosureNameError.proto /////////////// +static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname);/*proto*/ + +/////////////// RaiseClosureNameError /////////////// +static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname) { + PyErr_Format(PyExc_NameError, "free variable '%s' referenced before assignment in enclosing scope", varname); +} + + +/////////////// RaiseUnboundMemoryviewSliceNogil.proto /////////////// +static void __Pyx_RaiseUnboundMemoryviewSliceNogil(const char *varname);/*proto*/ + +/////////////// RaiseUnboundMemoryviewSliceNogil /////////////// +//@requires: RaiseUnboundLocalError + +// Don't inline the function, it should really never be called in production +static void __Pyx_RaiseUnboundMemoryviewSliceNogil(const char *varname) { + #ifdef WITH_THREAD + PyGILState_STATE gilstate = PyGILState_Ensure(); + #endif + __Pyx_RaiseUnboundLocalError(varname); + #ifdef WITH_THREAD + PyGILState_Release(gilstate); + #endif +} + +//////////////// RaiseCppGlobalNameError.proto /////////////////////// +static CYTHON_INLINE void __Pyx_RaiseCppGlobalNameError(const char *varname); /*proto*/ + +/////////////// RaiseCppGlobalNameError ////////////////////////////// +static CYTHON_INLINE void __Pyx_RaiseCppGlobalNameError(const char *varname) { + PyErr_Format(PyExc_NameError, "C++ global '%s' is not initialized", varname); +} + +//////////////// RaiseCppAttributeError.proto /////////////////////// +static CYTHON_INLINE void __Pyx_RaiseCppAttributeError(const char *varname); /*proto*/ + +/////////////// RaiseCppAttributeError ////////////////////////////// +static CYTHON_INLINE void __Pyx_RaiseCppAttributeError(const char *varname) { + PyErr_Format(PyExc_AttributeError, "C++ attribute '%s' is not initialized", varname); +} diff --git a/Cython/Utility/Optimize.c b/Cython/Utility/Optimize.c index 35f3a67c9..a16fb6ac9 100644 --- a/Cython/Utility/Optimize.c +++ b/Cython/Utility/Optimize.c @@ -94,7 +94,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyList_Pop(PyObject* L); /*proto*/ //@requires: ObjectHandling.c::PyObjectCallMethod0 static CYTHON_INLINE PyObject* __Pyx__PyObject_Pop(PyObject* L) { - if (Py_TYPE(L) == &PySet_Type) { + if (__Pyx_IS_TYPE(L, &PySet_Type)) { return PySet_Pop(L); } return __Pyx_PyObject_CallMethod0(L, PYIDENT("pop")); @@ -190,7 +190,7 @@ static PyObject* __Pyx_PyDict_GetItemDefault(PyObject* d, PyObject* key, PyObjec static PyObject* __Pyx_PyDict_GetItemDefault(PyObject* d, PyObject* key, PyObject* default_value) { PyObject* value; -#if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY +#if PY_MAJOR_VERSION >= 3 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000) value = PyDict_GetItemWithError(d, key); if (unlikely(!value)) { if (unlikely(PyErr_Occurred())) @@ -227,8 +227,9 @@ static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *ke /////////////// dict_setdefault /////////////// static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value, - CYTHON_UNUSED int is_safe_type) { + int is_safe_type) { PyObject* value; + CYTHON_MAYBE_UNUSED_VAR(is_safe_type); #if PY_VERSION_HEX >= 0x030400A0 // we keep the method call at the end to avoid "unused" C compiler warnings if ((1)) { @@ -238,7 +239,7 @@ static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *ke #else if (is_safe_type == 1 || (is_safe_type == -1 && /* the following builtins presumably have repeatably safe and fast hash functions */ -#if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY +#if PY_MAJOR_VERSION >= 3 && (!CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000) (PyUnicode_CheckExact(key) || PyString_CheckExact(key) || PyLong_CheckExact(key)))) { value = PyDict_GetItemWithError(d, key); if (unlikely(!value)) { @@ -444,7 +445,7 @@ static CYTHON_INLINE PyObject* __Pyx_set_iterator(PyObject* iterable, int is_set return iterable; } #else - (void)is_set; + CYTHON_UNUSED_VAR(is_set); *p_source_is_set = 0; #endif *p_orig_length = 0; @@ -460,8 +461,8 @@ static CYTHON_INLINE int __Pyx_set_iter_next( if (unlikely(!*value)) { return __Pyx_IterFinish(); } - (void)orig_length; - (void)ppos; + CYTHON_UNUSED_VAR(orig_length); + CYTHON_UNUSED_VAR(ppos); return 1; } #if CYTHON_COMPILING_IN_CPYTHON @@ -567,7 +568,12 @@ static CYTHON_INLINE int __Pyx_init_unicode_iteration( static CYTHON_INLINE int __Pyx_init_unicode_iteration( PyObject* ustring, Py_ssize_t *length, void** data, int *kind) { -#if CYTHON_PEP393_ENABLED +#if CYTHON_COMPILING_IN_LIMITED_API + // In the limited API we just point data to the unicode object + *kind = 0; + *length = PyUnicode_GetLength(ustring); + *data = (void*)ustring; +#elif CYTHON_PEP393_ENABLED if (unlikely(__Pyx_PyUnicode_READY(ustring) < 0)) return -1; *kind = PyUnicode_KIND(ustring); *length = PyUnicode_GET_LENGTH(ustring); @@ -591,51 +597,366 @@ static double __Pyx__PyObject_AsDouble(PyObject* obj); /* proto */ PyFloat_AsDouble(obj) : __Pyx__PyObject_AsDouble(obj)) #else #define __Pyx_PyObject_AsDouble(obj) \ -((likely(PyFloat_CheckExact(obj))) ? \ - PyFloat_AS_DOUBLE(obj) : __Pyx__PyObject_AsDouble(obj)) +((likely(PyFloat_CheckExact(obj))) ? PyFloat_AS_DOUBLE(obj) : \ + likely(PyLong_CheckExact(obj)) ? \ + PyLong_AsDouble(obj) : __Pyx__PyObject_AsDouble(obj)) #endif /////////////// pyobject_as_double /////////////// +//@requires: pybytes_as_double +//@requires: pyunicode_as_double +//@requires: ObjectHandling.c::PyObjectCallOneArg static double __Pyx__PyObject_AsDouble(PyObject* obj) { - PyObject* float_value; + if (PyUnicode_CheckExact(obj)) { + return __Pyx_PyUnicode_AsDouble(obj); + } else if (PyBytes_CheckExact(obj)) { + return __Pyx_PyBytes_AsDouble(obj); + } else if (PyByteArray_CheckExact(obj)) { + return __Pyx_PyByteArray_AsDouble(obj); + } else { + PyObject* float_value; #if !CYTHON_USE_TYPE_SLOTS - float_value = PyNumber_Float(obj); if ((0)) goto bad; + float_value = PyNumber_Float(obj); if ((0)) goto bad; + // avoid "unused" warnings + (void)__Pyx_PyObject_CallOneArg; #else - PyNumberMethods *nb = Py_TYPE(obj)->tp_as_number; - if (likely(nb) && likely(nb->nb_float)) { - float_value = nb->nb_float(obj); - if (likely(float_value) && unlikely(!PyFloat_Check(float_value))) { - PyErr_Format(PyExc_TypeError, - "__float__ returned non-float (type %.200s)", - Py_TYPE(float_value)->tp_name); - Py_DECREF(float_value); - goto bad; + PyNumberMethods *nb = Py_TYPE(obj)->tp_as_number; + if (likely(nb) && likely(nb->nb_float)) { + float_value = nb->nb_float(obj); + if (likely(float_value) && unlikely(!PyFloat_Check(float_value))) { + __Pyx_TypeName float_value_type_name = __Pyx_PyType_GetName(Py_TYPE(float_value)); + PyErr_Format(PyExc_TypeError, + "__float__ returned non-float (type " __Pyx_FMT_TYPENAME ")", + float_value_type_name); + __Pyx_DECREF_TypeName(float_value_type_name); + Py_DECREF(float_value); + goto bad; + } + } else { + float_value = __Pyx_PyObject_CallOneArg((PyObject*)&PyFloat_Type, obj); } - } else if (PyUnicode_CheckExact(obj) || PyBytes_CheckExact(obj)) { -#if PY_MAJOR_VERSION >= 3 - float_value = PyFloat_FromString(obj); -#else - float_value = PyFloat_FromString(obj, 0); #endif + if (likely(float_value)) { + double value = PyFloat_AS_DOUBLE(float_value); + Py_DECREF(float_value); + return value; + } + } +bad: + return (double)-1; +} + + +/////////////// pystring_as_double.proto /////////////// +//@requires: pyunicode_as_double +//@requires: pybytes_as_double + +static CYTHON_INLINE double __Pyx_PyString_AsDouble(PyObject *obj) { + #if PY_MAJOR_VERSION >= 3 + (void)__Pyx_PyBytes_AsDouble; + return __Pyx_PyUnicode_AsDouble(obj); + #else + (void)__Pyx_PyUnicode_AsDouble; + return __Pyx_PyBytes_AsDouble(obj); + #endif +} + + +/////////////// pyunicode_as_double.proto /////////////// + +static CYTHON_INLINE double __Pyx_PyUnicode_AsDouble(PyObject *obj);/*proto*/ + +/////////////// pyunicode_as_double.proto /////////////// +//@requires: pybytes_as_double + +#if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY +static const char* __Pyx__PyUnicode_AsDouble_Copy(const void* data, const int kind, char* buffer, Py_ssize_t start, Py_ssize_t end) { + int last_was_punctuation; + Py_ssize_t i; + // number must not start with punctuation + last_was_punctuation = 1; + for (i=start; i <= end; i++) { + Py_UCS4 chr = PyUnicode_READ(kind, data, i); + int is_punctuation = (chr == '_') | (chr == '.'); + *buffer = (char)chr; + // reject sequences of '_' and '.' + buffer += (chr != '_'); + if (unlikely(chr > 127)) goto parse_failure; + if (unlikely(last_was_punctuation & is_punctuation)) goto parse_failure; + last_was_punctuation = is_punctuation; + } + if (unlikely(last_was_punctuation)) goto parse_failure; + *buffer = '\0'; + return buffer; + +parse_failure: + return NULL; +} + +static double __Pyx__PyUnicode_AsDouble_inf_nan(const void* data, int kind, Py_ssize_t start, Py_ssize_t length) { + int matches = 1; + Py_UCS4 chr; + Py_UCS4 sign = PyUnicode_READ(kind, data, start); + int is_signed = (sign == '-') | (sign == '+'); + start += is_signed; + length -= is_signed; + + switch (PyUnicode_READ(kind, data, start)) { + #ifdef Py_NAN + case 'n': + case 'N': + if (unlikely(length != 3)) goto parse_failure; + chr = PyUnicode_READ(kind, data, start+1); + matches &= (chr == 'a') | (chr == 'A'); + chr = PyUnicode_READ(kind, data, start+2); + matches &= (chr == 'n') | (chr == 'N'); + if (unlikely(!matches)) goto parse_failure; + return (sign == '-') ? -Py_NAN : Py_NAN; + #endif + case 'i': + case 'I': + if (unlikely(length < 3)) goto parse_failure; + chr = PyUnicode_READ(kind, data, start+1); + matches &= (chr == 'n') | (chr == 'N'); + chr = PyUnicode_READ(kind, data, start+2); + matches &= (chr == 'f') | (chr == 'F'); + if (likely(length == 3 && matches)) + return (sign == '-') ? -Py_HUGE_VAL : Py_HUGE_VAL; + if (unlikely(length != 8)) goto parse_failure; + chr = PyUnicode_READ(kind, data, start+3); + matches &= (chr == 'i') | (chr == 'I'); + chr = PyUnicode_READ(kind, data, start+4); + matches &= (chr == 'n') | (chr == 'N'); + chr = PyUnicode_READ(kind, data, start+5); + matches &= (chr == 'i') | (chr == 'I'); + chr = PyUnicode_READ(kind, data, start+6); + matches &= (chr == 't') | (chr == 'T'); + chr = PyUnicode_READ(kind, data, start+7); + matches &= (chr == 'y') | (chr == 'Y'); + if (unlikely(!matches)) goto parse_failure; + return (sign == '-') ? -Py_HUGE_VAL : Py_HUGE_VAL; + case '.': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': + break; + default: + goto parse_failure; + } + return 0.0; +parse_failure: + return -1.0; +} + +static double __Pyx_PyUnicode_AsDouble_WithSpaces(PyObject *obj) { + double value; + const char *last; + char *end; + Py_ssize_t start, length = PyUnicode_GET_LENGTH(obj); + const int kind = PyUnicode_KIND(obj); + const void* data = PyUnicode_DATA(obj); + + // strip spaces at start and end + start = 0; + while (Py_UNICODE_ISSPACE(PyUnicode_READ(kind, data, start))) + start++; + while (start < length - 1 && Py_UNICODE_ISSPACE(PyUnicode_READ(kind, data, length - 1))) + length--; + length -= start; + if (unlikely(length <= 0)) goto fallback; + + // parse NaN / inf + value = __Pyx__PyUnicode_AsDouble_inf_nan(data, kind, start, length); + if (unlikely(value == -1.0)) goto fallback; + if (value != 0.0) return value; + + if (length < 40) { + char number[40]; + last = __Pyx__PyUnicode_AsDouble_Copy(data, kind, number, start, start + length); + if (unlikely(!last)) goto fallback; + value = PyOS_string_to_double(number, &end, NULL); } else { - PyObject* args = PyTuple_New(1); - if (unlikely(!args)) goto bad; - PyTuple_SET_ITEM(args, 0, obj); - float_value = PyObject_Call((PyObject*)&PyFloat_Type, args, 0); - PyTuple_SET_ITEM(args, 0, 0); - Py_DECREF(args); + char *number = (char*) PyMem_Malloc((length + 1) * sizeof(char)); + if (unlikely(!number)) goto fallback; + last = __Pyx__PyUnicode_AsDouble_Copy(data, kind, number, start, start + length); + if (unlikely(!last)) { + PyMem_Free(number); + goto fallback; + } + value = PyOS_string_to_double(number, &end, NULL); + PyMem_Free(number); + } + if (likely(end == last) || (value == (double)-1 && PyErr_Occurred())) { + return value; + } +fallback: + return __Pyx_SlowPyString_AsDouble(obj); +} +#endif + +static CYTHON_INLINE double __Pyx_PyUnicode_AsDouble(PyObject *obj) { + // Currently not optimised for Py2.7. +#if PY_MAJOR_VERSION >= 3 && !CYTHON_COMPILING_IN_PYPY + if (unlikely(__Pyx_PyUnicode_READY(obj) == -1)) + return (double)-1; + if (likely(PyUnicode_IS_ASCII(obj))) { + const char *s; + Py_ssize_t length; + s = PyUnicode_AsUTF8AndSize(obj, &length); + return __Pyx__PyBytes_AsDouble(obj, s, length); } + return __Pyx_PyUnicode_AsDouble_WithSpaces(obj); +#else + return __Pyx_SlowPyString_AsDouble(obj); +#endif +} + + +/////////////// pybytes_as_double.proto /////////////// + +static double __Pyx_SlowPyString_AsDouble(PyObject *obj);/*proto*/ +static double __Pyx__PyBytes_AsDouble(PyObject *obj, const char* start, Py_ssize_t length);/*proto*/ + +static CYTHON_INLINE double __Pyx_PyBytes_AsDouble(PyObject *obj) { + return __Pyx__PyBytes_AsDouble(obj, PyBytes_AS_STRING(obj), PyBytes_GET_SIZE(obj)); +} +static CYTHON_INLINE double __Pyx_PyByteArray_AsDouble(PyObject *obj) { + return __Pyx__PyBytes_AsDouble(obj, PyByteArray_AS_STRING(obj), PyByteArray_GET_SIZE(obj)); +} + + +/////////////// pybytes_as_double /////////////// + +static double __Pyx_SlowPyString_AsDouble(PyObject *obj) { + PyObject *float_value; +#if PY_MAJOR_VERSION >= 3 + float_value = PyFloat_FromString(obj); +#else + float_value = PyFloat_FromString(obj, 0); #endif if (likely(float_value)) { double value = PyFloat_AS_DOUBLE(float_value); Py_DECREF(float_value); return value; } -bad: return (double)-1; } +static const char* __Pyx__PyBytes_AsDouble_Copy(const char* start, char* buffer, Py_ssize_t length) { + // number must not start with punctuation + int last_was_punctuation = 1; + Py_ssize_t i; + for (i=0; i < length; i++) { + char chr = start[i]; + int is_punctuation = (chr == '_') | (chr == '.') | (chr == 'e') | (chr == 'E'); + *buffer = chr; + buffer += (chr != '_'); + // reject sequences of '_' and '.' + if (unlikely(last_was_punctuation & is_punctuation)) goto parse_failure; + last_was_punctuation = is_punctuation; + } + if (unlikely(last_was_punctuation)) goto parse_failure; + *buffer = '\0'; + return buffer; + +parse_failure: + return NULL; +} + +static double __Pyx__PyBytes_AsDouble_inf_nan(const char* start, Py_ssize_t length) { + int matches = 1; + char sign = start[0]; + int is_signed = (sign == '+') | (sign == '-'); + start += is_signed; + length -= is_signed; + + switch (start[0]) { + #ifdef Py_NAN + case 'n': + case 'N': + if (unlikely(length != 3)) goto parse_failure; + matches &= (start[1] == 'a' || start[1] == 'A'); + matches &= (start[2] == 'n' || start[2] == 'N'); + if (unlikely(!matches)) goto parse_failure; + return (sign == '-') ? -Py_NAN : Py_NAN; + #endif + case 'i': + case 'I': + if (unlikely(length < 3)) goto parse_failure; + matches &= (start[1] == 'n' || start[1] == 'N'); + matches &= (start[2] == 'f' || start[2] == 'F'); + if (likely(length == 3 && matches)) + return (sign == '-') ? -Py_HUGE_VAL : Py_HUGE_VAL; + if (unlikely(length != 8)) goto parse_failure; + matches &= (start[3] == 'i' || start[3] == 'I'); + matches &= (start[4] == 'n' || start[4] == 'N'); + matches &= (start[5] == 'i' || start[5] == 'I'); + matches &= (start[6] == 't' || start[6] == 'T'); + matches &= (start[7] == 'y' || start[7] == 'Y'); + if (unlikely(!matches)) goto parse_failure; + return (sign == '-') ? -Py_HUGE_VAL : Py_HUGE_VAL; + case '.': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': + break; + default: + goto parse_failure; + } + return 0.0; +parse_failure: + return -1.0; +} + +static CYTHON_INLINE int __Pyx__PyBytes_AsDouble_IsSpace(char ch) { + // see Py_ISSPACE() in CPython + // https://github.com/python/cpython/blob/master/Python/pyctype.c + return (ch == 0x20) | !((ch < 0x9) | (ch > 0xd)); +} + +CYTHON_UNUSED static double __Pyx__PyBytes_AsDouble(PyObject *obj, const char* start, Py_ssize_t length) { + double value; + Py_ssize_t i, digits; + const char *last = start + length; + char *end; + + // strip spaces at start and end + while (__Pyx__PyBytes_AsDouble_IsSpace(*start)) + start++; + while (start < last - 1 && __Pyx__PyBytes_AsDouble_IsSpace(last[-1])) + last--; + length = last - start; + if (unlikely(length <= 0)) goto fallback; + + // parse NaN / inf + value = __Pyx__PyBytes_AsDouble_inf_nan(start, length); + if (unlikely(value == -1.0)) goto fallback; + if (value != 0.0) return value; + + // look for underscores + digits = 0; + for (i=0; i < length; digits += start[i++] != '_'); + + if (likely(digits == length)) { + value = PyOS_string_to_double(start, &end, NULL); + } else if (digits < 40) { + char number[40]; + last = __Pyx__PyBytes_AsDouble_Copy(start, number, length); + if (unlikely(!last)) goto fallback; + value = PyOS_string_to_double(number, &end, NULL); + } else { + char *number = (char*) PyMem_Malloc((digits + 1) * sizeof(char)); + if (unlikely(!number)) goto fallback; + last = __Pyx__PyBytes_AsDouble_Copy(start, number, length); + if (unlikely(!last)) { + PyMem_Free(number); + goto fallback; + } + value = PyOS_string_to_double(number, &end, NULL); + PyMem_Free(number); + } + if (likely(end == last) || (value == (double)-1 && PyErr_Occurred())) { + return value; + } +fallback: + return __Pyx_SlowPyString_AsDouble(obj); +} + /////////////// PyNumberPow2.proto /////////////// @@ -648,7 +969,7 @@ static PyObject* __Pyx__PyNumber_PowerOf2(PyObject *two, PyObject *exp, PyObject static PyObject* __Pyx__PyNumber_PowerOf2(PyObject *two, PyObject *exp, PyObject *none, int inplace) { // in CPython, 1<<N is substantially faster than 2**N -// see http://bugs.python.org/issue21420 +// see https://bugs.python.org/issue21420 #if !CYTHON_COMPILING_IN_PYPY Py_ssize_t shiftby; #if PY_MAJOR_VERSION < 3 @@ -658,14 +979,12 @@ static PyObject* __Pyx__PyNumber_PowerOf2(PyObject *two, PyObject *exp, PyObject #endif if (likely(PyLong_CheckExact(exp))) { #if CYTHON_USE_PYLONG_INTERNALS - const Py_ssize_t size = Py_SIZE(exp); - // tuned to optimise branch prediction - if (likely(size == 1)) { - shiftby = ((PyLongObject*)exp)->ob_digit[0]; - } else if (size == 0) { + if (__Pyx_PyLong_IsZero(exp)) { return PyInt_FromLong(1L); - } else if (unlikely(size < 0)) { + } else if (__Pyx_PyLong_IsNeg(exp)) { goto fallback; + } else if (__Pyx_PyLong_IsCompact(exp)) { + shiftby = __Pyx_PyLong_CompactValueUnsigned(exp); } else { shiftby = PyLong_AsSsize_t(exp); } @@ -722,7 +1041,9 @@ return_compare = ( ) }} -static CYTHON_INLINE {{c_ret_type}} __Pyx_PyInt_{{'' if ret_type.is_pyobject else 'Bool'}}{{op}}{{order}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, CYTHON_UNUSED long inplace) { +static CYTHON_INLINE {{c_ret_type}} __Pyx_PyInt_{{'' if ret_type.is_pyobject else 'Bool'}}{{op}}{{order}}(PyObject *op1, PyObject *op2, long intval, long inplace) { + CYTHON_MAYBE_UNUSED_VAR(intval); + CYTHON_UNUSED_VAR(inplace); if (op1 == op2) { {{return_true if op == 'Eq' else return_false}}; } @@ -739,21 +1060,18 @@ static CYTHON_INLINE {{c_ret_type}} __Pyx_PyInt_{{'' if ret_type.is_pyobject els if (likely(PyLong_CheckExact({{pyval}}))) { int unequal; unsigned long uintval; - Py_ssize_t size = Py_SIZE({{pyval}}); - const digit* digits = ((PyLongObject*){{pyval}})->ob_digit; + Py_ssize_t size = __Pyx_PyLong_DigitCount({{pyval}}); + const digit* digits = __Pyx_PyLong_Digits({{pyval}}); if (intval == 0) { - // == 0 => Py_SIZE(pyval) == 0 - {{return_compare('size', '0', c_op)}} + {{return_compare('__Pyx_PyLong_IsZero(%s)' % pyval, '1', c_op)}} } else if (intval < 0) { - // < 0 => Py_SIZE(pyval) < 0 - if (size >= 0) + if (__Pyx_PyLong_IsNonNeg({{pyval}})) {{return_false if op == 'Eq' else return_true}}; // both are negative => can use absolute values now. intval = -intval; - size = -size; } else { // > 0 => Py_SIZE(pyval) > 0 - if (size <= 0) + if (__Pyx_PyLong_IsNeg({{pyval}})) {{return_false if op == 'Eq' else return_true}}; } // After checking that the sign is the same (and excluding 0), now compare the absolute values. @@ -776,7 +1094,11 @@ static CYTHON_INLINE {{c_ret_type}} __Pyx_PyInt_{{'' if ret_type.is_pyobject els if (PyFloat_CheckExact({{pyval}})) { const long {{'a' if order == 'CObj' else 'b'}} = intval; +#if CYTHON_COMPILING_IN_LIMITED_API + double {{ival}} = __pyx_PyFloat_AsDouble({{pyval}}); +#else double {{ival}} = PyFloat_AS_DOUBLE({{pyval}}); +#endif {{return_compare('(double)a', '(double)b', c_op)}} } @@ -807,29 +1129,27 @@ static {{c_ret_type}} __Pyx_PyInt_{{'' if ret_type.is_pyobject else 'Bool'}}{{op {{py: return_false = 'Py_RETURN_FALSE' if ret_type.is_pyobject else 'return 0'}} {{py: slot_name = {'TrueDivide': 'true_divide', 'FloorDivide': 'floor_divide'}.get(op, op.lower()) }} {{py: cfunc_name = '__Pyx_PyInt_%s%s%s' % ('' if ret_type.is_pyobject else 'Bool', op, order)}} -{{py: zerodiv_check = lambda operand, _cfunc_name=cfunc_name: '%s_ZeroDivisionError(%s)' % (_cfunc_name, operand)}} {{py: c_op = { - 'Add': '+', 'Subtract': '-', 'Remainder': '%', 'TrueDivide': '/', 'FloorDivide': '/', + 'Add': '+', 'Subtract': '-', 'Multiply': '*', 'Remainder': '%', 'TrueDivide': '/', 'FloorDivide': '/', 'Or': '|', 'Xor': '^', 'And': '&', 'Rshift': '>>', 'Lshift': '<<', 'Eq': '==', 'Ne': '!=', }[op] }} +{{py: +def zerodiv_check(operand, optype='integer', _is_mod=op == 'Remainder', _needs_check=(order == 'CObj' and c_op in '%/')): + return ((( + 'if (unlikely(zerodivision_check && ((%s) == 0))) {' + ' PyErr_SetString(PyExc_ZeroDivisionError, "%s division%s by zero");' + ' return NULL;' + '}') % (operand, optype, ' or modulo' if _is_mod else '') + ) if _needs_check else '') +}} -{{if op in ('TrueDivide', 'FloorDivide', 'Remainder')}} -#if PY_MAJOR_VERSION < 3 || CYTHON_USE_PYLONG_INTERNALS -#define {{zerodiv_check('operand')}} \ - if (unlikely(zerodivision_check && ((operand) == 0))) { \ - PyErr_SetString(PyExc_ZeroDivisionError, "integer division{{if op == 'Remainder'}} or modulo{{endif}} by zero"); \ - return NULL; \ - } -#endif -{{endif}} - -static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, int inplace, int zerodivision_check) { - // Prevent "unused" warnings. - (void)inplace; - (void)zerodivision_check; +static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, long intval, int inplace, int zerodivision_check) { + CYTHON_MAYBE_UNUSED_VAR(intval); + CYTHON_MAYBE_UNUSED_VAR(inplace); + CYTHON_UNUSED_VAR(zerodivision_check); {{if op in ('Eq', 'Ne')}} if (op1 == op2) { @@ -844,6 +1164,7 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED long x; {{endif}} long {{ival}} = PyInt_AS_LONG({{pyval}}); + {{zerodiv_check('b')}} {{if op in ('Eq', 'Ne')}} if (a {{c_op}} b) { @@ -859,21 +1180,18 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED return PyInt_FromLong(x); return PyLong_Type.tp_as_number->nb_{{slot_name}}(op1, op2); {{elif c_op == '%'}} - {{zerodiv_check('b')}} // see ExprNodes.py :: mod_int_utility_code x = a % b; x += ((x != 0) & ((x ^ b) < 0)) * b; return PyInt_FromLong(x); {{elif op == 'TrueDivide'}} - {{zerodiv_check('b')}} if (8 * sizeof(long) <= 53 || likely(labs({{ival}}) <= ((PY_LONG_LONG)1 << 53))) { return PyFloat_FromDouble((double)a / (double)b); } // let Python do the rounding return PyInt_Type.tp_as_number->nb_{{slot_name}}(op1, op2); {{elif op == 'FloorDivide'}} - // INT_MIN / -1 is the only case that overflows, b == 0 is an error case - {{zerodiv_check('b')}} + // INT_MIN / -1 is the only case that overflows if (unlikely(b == -1 && ((unsigned long)a) == 0-(unsigned long)a)) return PyInt_Type.tp_as_number->nb_{{slot_name}}(op1, op2); else { @@ -889,6 +1207,19 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED if (likely(b < (long) (sizeof(long)*8) && a == (a << b) >> b) || !a) { return PyInt_FromLong(a {{c_op}} b); } + {{elif c_op == '*'}} +#ifdef HAVE_LONG_LONG + if (sizeof(PY_LONG_LONG) > sizeof(long)) { + PY_LONG_LONG result = (PY_LONG_LONG)a {{c_op}} (PY_LONG_LONG)b; + return (result >= LONG_MIN && result <= LONG_MAX) ? + PyInt_FromLong((long)result) : PyLong_FromLongLong(result); + } +#endif +#if CYTHON_USE_TYPE_SLOTS + return PyInt_Type.tp_as_number->nb_{{slot_name}}(op1, op2); +#else + return PyNumber_{{op}}(op1, op2); +#endif {{else}} // other operations are safe, no overflow return PyInt_FromLong(a {{c_op}} b); @@ -906,26 +1237,54 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED PY_LONG_LONG ll{{ival}}, llx; #endif {{endif}} - const digit* digits = ((PyLongObject*){{pyval}})->ob_digit; - const Py_ssize_t size = Py_SIZE({{pyval}}); + {{if c_op == '&'}} + // special case for &-ing arbitrarily large numbers with known single digit operands + if ((intval & PyLong_MASK) == intval) { + long result = intval & (long) __Pyx_PyLong_CompactValue({{pyval}}); + return PyLong_FromLong(result); + } + {{endif}} + // special cases for 0: + - * % / // | ^ & >> << + if (unlikely(__Pyx_PyLong_IsZero({{pyval}}))) { + {{if order == 'CObj' and c_op in '%/'}} + // division by zero! + {{zerodiv_check('0')}} + {{elif order == 'CObj' and c_op in '+-|^>><<'}} + // x == x+0 == x-0 == x|0 == x^0 == x>>0 == x<<0 + return __Pyx_NewRef(op1); + {{elif order == 'CObj' and c_op in '*&'}} + // 0 == x*0 == x&0 + return __Pyx_NewRef(op2); + {{elif order == 'ObjC' and c_op in '+|^'}} + // x == 0+x == 0|x == 0^x + return __Pyx_NewRef(op2); + {{elif order == 'ObjC' and c_op == '-'}} + // -x == 0-x + return PyLong_FromLong(-intval); + {{elif order == 'ObjC' and (c_op in '*%&>><<' or op == 'FloorDivide')}} + // 0 == 0*x == 0%x == 0&x == 0>>x == 0<<x == 0//x + return __Pyx_NewRef(op1); + {{endif}} + } // handle most common case first to avoid indirect branch and optimise branch prediction - if (likely(__Pyx_sst_abs(size) <= 1)) { - {{ival}} = likely(size) ? digits[0] : 0; - if (size == -1) {{ival}} = -{{ival}}; + if (likely(__Pyx_PyLong_IsCompact({{pyval}}))) { + {{ival}} = __Pyx_PyLong_CompactValue({{pyval}}); } else { + const digit* digits = __Pyx_PyLong_Digits({{pyval}}); + const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount({{pyval}}); switch (size) { {{for _size in range(2, 5)}} {{for _case in (-_size, _size)}} case {{_case}}: - if (8 * sizeof(long) - 1 > {{_size}} * PyLong_SHIFT{{if op == 'TrueDivide'}} && {{_size-1}} * PyLong_SHIFT < 53{{endif}}) { + if (8 * sizeof(long) - 1 > {{_size}} * PyLong_SHIFT{{if c_op == '*'}}+30{{endif}}{{if op == 'TrueDivide'}} && {{_size-1}} * PyLong_SHIFT < 53{{endif}}) { {{ival}} = {{'-' if _case < 0 else ''}}(long) {{pylong_join(_size, 'digits')}}; break; {{if op not in ('Eq', 'Ne', 'TrueDivide')}} -#ifdef HAVE_LONG_LONG - } else if (8 * sizeof(PY_LONG_LONG) - 1 > {{_size}} * PyLong_SHIFT) { + #ifdef HAVE_LONG_LONG + } else if (8 * sizeof(PY_LONG_LONG) - 1 > {{_size}} * PyLong_SHIFT{{if c_op == '*'}}+30{{endif}}) { ll{{ival}} = {{'-' if _case < 0 else ''}}(PY_LONG_LONG) {{pylong_join(_size, 'digits', 'unsigned PY_LONG_LONG')}}; goto long_long; -#endif + #endif {{endif}} } // if size doesn't fit into a long or PY_LONG_LONG anymore, fall through to default @@ -954,20 +1313,26 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED {{return_false}}; } {{else}} - {{if c_op == '%'}} - {{zerodiv_check('b')}} + {{if c_op == '*'}} + CYTHON_UNUSED_VAR(a); + CYTHON_UNUSED_VAR(b); + #ifdef HAVE_LONG_LONG + ll{{ival}} = {{ival}}; + goto long_long; + #else + return PyLong_Type.tp_as_number->nb_{{slot_name}}(op1, op2); + #endif + {{elif c_op == '%'}} // see ExprNodes.py :: mod_int_utility_code x = a % b; x += ((x != 0) & ((x ^ b) < 0)) * b; {{elif op == 'TrueDivide'}} - {{zerodiv_check('b')}} if ((8 * sizeof(long) <= 53 || likely(labs({{ival}}) <= ((PY_LONG_LONG)1 << 53))) - || __Pyx_sst_abs(size) <= 52 / PyLong_SHIFT) { + || __Pyx_PyLong_DigitCount({{pyval}}) <= 52 / PyLong_SHIFT) { return PyFloat_FromDouble((double)a / (double)b); } return PyLong_Type.tp_as_number->nb_{{slot_name}}(op1, op2); {{elif op == 'FloorDivide'}} - {{zerodiv_check('b')}} { long q, r; // see ExprNodes.py :: div_int_utility_code @@ -1020,10 +1385,14 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED } #endif - {{if c_op in '+-' or op in ('TrueDivide', 'Eq', 'Ne')}} + {{if c_op in '+-*' or op in ('TrueDivide', 'Eq', 'Ne')}} if (PyFloat_CheckExact({{pyval}})) { const long {{'a' if order == 'CObj' else 'b'}} = intval; +#if CYTHON_COMPILING_IN_LIMITED_API + double {{ival}} = __pyx_PyFloat_AsDouble({{pyval}}); +#else double {{ival}} = PyFloat_AS_DOUBLE({{pyval}}); +#endif {{if op in ('Eq', 'Ne')}} if ((double)a {{c_op}} (double)b) { {{return_true}}; @@ -1032,12 +1401,7 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED } {{else}} double result; - {{if op == 'TrueDivide'}} - if (unlikely(zerodivision_check && b == 0)) { - PyErr_SetString(PyExc_ZeroDivisionError, "float division by zero"); - return NULL; - } - {{endif}} + {{zerodiv_check('b', 'float')}} // copied from floatobject.c in Py3.5: PyFPE_START_PROTECT("{{op.lower() if not op.endswith('Divide') else 'divide'}}", return NULL) result = ((double)a) {{c_op}} (double)b; @@ -1078,27 +1442,27 @@ static {{c_ret_type}} __Pyx_PyFloat_{{'' if ret_type.is_pyobject else 'Bool'}}{{ {{py: return_false = 'Py_RETURN_FALSE' if ret_type.is_pyobject else 'return 0'}} {{py: pyval, fval = ('op2', 'b') if order == 'CObj' else ('op1', 'a') }} {{py: cfunc_name = '__Pyx_PyFloat_%s%s%s' % ('' if ret_type.is_pyobject else 'Bool', op, order) }} -{{py: zerodiv_check = lambda operand, _cfunc_name=cfunc_name: '%s_ZeroDivisionError(%s)' % (_cfunc_name, operand)}} {{py: c_op = { 'Add': '+', 'Subtract': '-', 'TrueDivide': '/', 'Divide': '/', 'Remainder': '%', 'Eq': '==', 'Ne': '!=', }[op] }} - -{{if order == 'CObj' and c_op in '%/'}} -#define {{zerodiv_check('operand')}} if (unlikely(zerodivision_check && ((operand) == 0))) { \ - PyErr_SetString(PyExc_ZeroDivisionError, "float division{{if op == 'Remainder'}} or modulo{{endif}} by zero"); \ - return NULL; \ -} -{{endif}} +{{py: +def zerodiv_check(operand, _is_mod=op == 'Remainder', _needs_check=(order == 'CObj' and c_op in '%/')): + return ((( + 'if (unlikely(zerodivision_check && ((%s) == 0.0))) {' + ' PyErr_SetString(PyExc_ZeroDivisionError, "float division%s by zero");' + ' return NULL;' + '}') % (operand, ' or modulo' if _is_mod else '') + ) if _needs_check else '') +}} static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatval, int inplace, int zerodivision_check) { const double {{'a' if order == 'CObj' else 'b'}} = floatval; double {{fval}}{{if op not in ('Eq', 'Ne')}}, result{{endif}}; - // Prevent "unused" warnings. - (void)inplace; - (void)zerodivision_check; + CYTHON_UNUSED_VAR(inplace); + CYTHON_UNUSED_VAR(zerodivision_check); {{if op in ('Eq', 'Ne')}} if (op1 == op2) { @@ -1107,57 +1471,69 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatv {{endif}} if (likely(PyFloat_CheckExact({{pyval}}))) { +#if CYTHON_COMPILING_IN_LIMITED_API + {{fval}} = __pyx_PyFloat_AsDouble({{pyval}}); +#else {{fval}} = PyFloat_AS_DOUBLE({{pyval}}); - {{if order == 'CObj' and c_op in '%/'}}{{zerodiv_check(fval)}}{{endif}} +#endif + {{zerodiv_check(fval)}} } else #if PY_MAJOR_VERSION < 3 if (likely(PyInt_CheckExact({{pyval}}))) { {{fval}} = (double) PyInt_AS_LONG({{pyval}}); - {{if order == 'CObj' and c_op in '%/'}}{{zerodiv_check(fval)}}{{endif}} + {{zerodiv_check(fval)}} } else #endif if (likely(PyLong_CheckExact({{pyval}}))) { #if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*){{pyval}})->ob_digit; - const Py_ssize_t size = Py_SIZE({{pyval}}); - switch (size) { - case 0: {{if order == 'CObj' and c_op in '%/'}}{{zerodiv_check('0')}}{{else}}{{fval}} = 0.0;{{endif}} break; - case -1: {{fval}} = -(double) digits[0]; break; - case 1: {{fval}} = (double) digits[0]; break; - {{for _size in (2, 3, 4)}} - case -{{_size}}: - case {{_size}}: - if (8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT && ((8 * sizeof(unsigned long) < 53) || ({{_size-1}} * PyLong_SHIFT < 53))) { - {{fval}} = (double) {{pylong_join(_size, 'digits')}}; - // let CPython do its own float rounding from 2**53 on (max. consecutive integer in double float) - if ((8 * sizeof(unsigned long) < 53) || ({{_size}} * PyLong_SHIFT < 53) || ({{fval}} < (double) ((PY_LONG_LONG)1 << 53))) { - if (size == {{-_size}}) - {{fval}} = -{{fval}}; - break; + if (__Pyx_PyLong_IsZero({{pyval}})) { + {{fval}} = 0.0; + {{zerodiv_check(fval)}} + } else if (__Pyx_PyLong_IsCompact({{pyval}})) { + {{fval}} = (double) __Pyx_PyLong_CompactValue({{pyval}}); + } else { + const digit* digits = __Pyx_PyLong_Digits({{pyval}}); + const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount({{pyval}}); + switch (size) { + {{for _size in (2, 3, 4)}} + case -{{_size}}: + case {{_size}}: + if (8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT && ((8 * sizeof(unsigned long) < 53) || ({{_size-1}} * PyLong_SHIFT < 53))) { + {{fval}} = (double) {{pylong_join(_size, 'digits')}}; + // let CPython do its own float rounding from 2**53 on (max. consecutive integer in double float) + if ((8 * sizeof(unsigned long) < 53) || ({{_size}} * PyLong_SHIFT < 53) || ({{fval}} < (double) ((PY_LONG_LONG)1 << 53))) { + if (size == {{-_size}}) + {{fval}} = -{{fval}}; + break; + } } - } - // Fall through if size doesn't fit safely into a double anymore. - // It may not be obvious that this is a safe fall-through given the "fval < 2**53" - // check above. However, the number of digits that CPython uses for a given PyLong - // value is minimal, and together with the "(size-1) * SHIFT < 53" check above, - // this should make it safe. - CYTHON_FALLTHROUGH; - {{endfor}} - default: - #else - { + // Fall through if size doesn't fit safely into a double anymore. + // It may not be obvious that this is a safe fall-through given the "fval < 2**53" + // check above. However, the number of digits that CPython uses for a given PyLong + // value is minimal, and together with the "(size-1) * SHIFT < 53" check above, + // this should make it safe. + CYTHON_FALLTHROUGH; + {{endfor}} + default: #endif {{if op in ('Eq', 'Ne')}} - return {{'' if ret_type.is_pyobject else '__Pyx_PyObject_IsTrueAndDecref'}}( - PyFloat_Type.tp_richcompare({{'op1, op2' if order == 'CObj' else 'op2, op1'}}, Py_{{op.upper()}})); + return {{'' if ret_type.is_pyobject else '__Pyx_PyObject_IsTrueAndDecref'}}( + PyFloat_Type.tp_richcompare({{'op1, op2' if order == 'CObj' else 'op2, op1'}}, Py_{{op.upper()}})); {{else}} - {{fval}} = PyLong_AsDouble({{pyval}}); - if (unlikely({{fval}} == -1.0 && PyErr_Occurred())) return NULL; - {{if order == 'CObj' and c_op in '%/'}}{{zerodiv_check(fval)}}{{endif}} + {{fval}} = PyLong_AsDouble({{pyval}}); + if (unlikely({{fval}} == -1.0 && PyErr_Occurred())) return NULL; + {{if zerodiv_check(fval)}} + #if !CYTHON_USE_PYLONG_INTERNALS + {{zerodiv_check(fval)}} + #endif + {{endif}} {{endif}} + #if CYTHON_USE_PYLONG_INTERNALS + } } + #endif } else { {{if op in ('Eq', 'Ne')}} return {{'' if ret_type.is_pyobject else '__Pyx_PyObject_IsTrueAndDecref'}}( @@ -1177,7 +1553,6 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatv } {{else}} // copied from floatobject.c in Py3.5: - {{if order == 'CObj' and c_op in '%/'}}{{zerodiv_check('b')}}{{endif}} PyFPE_START_PROTECT("{{op.lower() if not op.endswith('Divide') else 'divide'}}", return NULL) {{if c_op == '%'}} result = fmod(a, b); diff --git a/Cython/Utility/Overflow.c b/Cython/Utility/Overflow.c index 0259c58f0..78e517717 100644 --- a/Cython/Utility/Overflow.c +++ b/Cython/Utility/Overflow.c @@ -20,7 +20,7 @@ TODO: Conditionally support 128-bit with intmax_t? /////////////// Common.proto /////////////// static int __Pyx_check_twos_complement(void) { - if ((-1 != ~0)) { + if ((-1) != (~0)) { PyErr_SetString(PyExc_RuntimeError, "Two's complement required for overflow checks."); return 1; } else if ((sizeof(short) == sizeof(int))) { @@ -31,7 +31,6 @@ static int __Pyx_check_twos_complement(void) { } } -#define __PYX_IS_UNSIGNED(type) ((((type) -1) > 0)) #define __PYX_SIGN_BIT(type) ((((unsigned type) 1) << (sizeof(type) * 8 - 1))) #define __PYX_HALF_MAX(type) ((((type) 1) << (sizeof(type) * 8 - 2))) #define __PYX_MIN(type) ((__PYX_IS_UNSIGNED(type) ? (type) 0 : 0 - __PYX_HALF_MAX(type) - __PYX_HALF_MAX(type))) @@ -46,6 +45,24 @@ static int __Pyx_check_twos_complement(void) { #define __Pyx_div_no_overflow(a, b, overflow) ((a) / (b)) #define __Pyx_div_const_no_overflow(a, b, overflow) ((a) / (b)) +#if defined(__has_builtin) +# if __has_builtin(__builtin_add_overflow) && !defined(__ibmxl__) +# define __PYX_HAVE_BUILTIN_OVERFLOW +# endif +#elif defined(__GNUC__) && (__GNUC__ >= 5) && (!defined(__INTEL_COMPILER) || (__INTEL_COMPILER >= 1800)) +# define __PYX_HAVE_BUILTIN_OVERFLOW +#endif + +#if defined(__GNUC__) +# define __Pyx_is_constant(x) (__builtin_constant_p(x)) +#elif defined(__has_builtin) +# if __has_builtin(__builtin_constant_p) +# define __Pyx_is_constant(x) (__builtin_constant_p(x)) +# endif +#else +# define __Pyx_is_constant(x) (0) +#endif + /////////////// Common.init /////////////// //@substitute: naming @@ -56,6 +73,8 @@ if (unlikely(__Pyx_check_twos_complement())) { /////////////// BaseCaseUnsigned.proto /////////////// +{{if UINT == "long long"}}#ifdef HAVE_LONG_LONG{{endif}} + static CYTHON_INLINE {{UINT}} __Pyx_add_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow); static CYTHON_INLINE {{UINT}} __Pyx_sub_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow); static CYTHON_INLINE {{UINT}} __Pyx_mul_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow); @@ -64,11 +83,41 @@ static CYTHON_INLINE {{UINT}} __Pyx_div_{{NAME}}_checking_overflow({{UINT}} a, { // Use these when b is known at compile time. #define __Pyx_add_const_{{NAME}}_checking_overflow __Pyx_add_{{NAME}}_checking_overflow #define __Pyx_sub_const_{{NAME}}_checking_overflow __Pyx_sub_{{NAME}}_checking_overflow +#if defined(__PYX_HAVE_BUILTIN_OVERFLOW) +#define __Pyx_mul_const_{{NAME}}_checking_overflow __Pyx_mul_{{NAME}}_checking_overflow +#else static CYTHON_INLINE {{UINT}} __Pyx_mul_const_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} constant, int *overflow); +#endif #define __Pyx_div_const_{{NAME}}_checking_overflow __Pyx_div_{{NAME}}_checking_overflow +{{if UINT == "long long"}}#endif{{endif}} + /////////////// BaseCaseUnsigned /////////////// +{{if UINT == "long long"}}#ifdef HAVE_LONG_LONG{{endif}} + +#if defined(__PYX_HAVE_BUILTIN_OVERFLOW) + +static CYTHON_INLINE {{UINT}} __Pyx_add_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { + {{UINT}} result; + *overflow |= __builtin_add_overflow(a, b, &result); + return result; +} + +static CYTHON_INLINE {{UINT}} __Pyx_sub_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { + {{UINT}} result; + *overflow |= __builtin_sub_overflow(a, b, &result); + return result; +} + +static CYTHON_INLINE {{UINT}} __Pyx_mul_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { + {{UINT}} result; + *overflow |= __builtin_mul_overflow(a, b, &result); + return result; +} + +#else + static CYTHON_INLINE {{UINT}} __Pyx_add_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { {{UINT}} r = a + b; *overflow |= r < a; @@ -82,7 +131,12 @@ static CYTHON_INLINE {{UINT}} __Pyx_sub_{{NAME}}_checking_overflow({{UINT}} a, { } static CYTHON_INLINE {{UINT}} __Pyx_mul_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { - if ((sizeof({{UINT}}) < sizeof(unsigned long))) { + // if we have a constant, use the constant version + if (__Pyx_is_constant(b)) { + return __Pyx_mul_const_{{NAME}}_checking_overflow(a, b, overflow); + } else if (__Pyx_is_constant(a)) { + return __Pyx_mul_const_{{NAME}}_checking_overflow(b, a, overflow); + } else if ((sizeof({{UINT}}) < sizeof(unsigned long))) { unsigned long big_r = ((unsigned long) a) * ((unsigned long) b); {{UINT}} r = ({{UINT}}) big_r; *overflow |= big_r != r; @@ -95,21 +149,27 @@ static CYTHON_INLINE {{UINT}} __Pyx_mul_{{NAME}}_checking_overflow({{UINT}} a, { return r; #endif } else { - {{UINT}} prod = a * b; - double dprod = ((double) a) * ((double) b); - // Overflow results in an error of at least 2^sizeof(UINT), - // whereas rounding represents an error on the order of 2^(sizeof(UINT)-53). - *overflow |= fabs(dprod - prod) > (__PYX_MAX({{UINT}}) / 2); - return prod; + return __Pyx_mul_const_{{NAME}}_checking_overflow(a, b, overflow); } } static CYTHON_INLINE {{UINT}} __Pyx_mul_const_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { - if (b > 1) { - *overflow |= a > __PYX_MAX({{UINT}}) / b; + // note that deliberately the overflow check is written such that it divides by b; this + // function is used when b is a constant thus the compiler should be able to eliminate the + // (very slow on most CPUs!) division operation + {{UINT}} prod; + if (__Pyx_is_constant(a) && !__Pyx_is_constant(b)) { + // if a is a compile-time constant and b isn't, swap them + {{UINT}} temp = b; + b = a; + a = temp; } - return a * b; + prod = a * b; + if (b != 0) + *overflow |= a > (__PYX_MAX({{UINT}}) / b); + return prod; } +#endif // __PYX_HAVE_BUILTIN_OVERFLOW static CYTHON_INLINE {{UINT}} __Pyx_div_{{NAME}}_checking_overflow({{UINT}} a, {{UINT}} b, int *overflow) { @@ -120,9 +180,13 @@ static CYTHON_INLINE {{UINT}} __Pyx_div_{{NAME}}_checking_overflow({{UINT}} a, { return a / b; } +{{if UINT == "long long"}}#endif{{endif}} + /////////////// BaseCaseSigned.proto /////////////// +{{if INT == "long long"}}#ifdef HAVE_LONG_LONG{{endif}} + static CYTHON_INLINE {{INT}} __Pyx_add_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow); static CYTHON_INLINE {{INT}} __Pyx_sub_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow); static CYTHON_INLINE {{INT}} __Pyx_mul_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow); @@ -130,13 +194,43 @@ static CYTHON_INLINE {{INT}} __Pyx_div_{{NAME}}_checking_overflow({{INT}} a, {{I // Use when b is known at compile time. -static CYTHON_INLINE {{INT}} __Pyx_add_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow); -static CYTHON_INLINE {{INT}} __Pyx_sub_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow); +#define __Pyx_add_const_{{NAME}}_checking_overflow __Pyx_add_{{NAME}}_checking_overflow +#define __Pyx_sub_const_{{NAME}}_checking_overflow __Pyx_sub_{{NAME}}_checking_overflow +#if defined(__PYX_HAVE_BUILTIN_OVERFLOW) +#define __Pyx_mul_const_{{NAME}}_checking_overflow __Pyx_mul_{{NAME}}_checking_overflow +#else static CYTHON_INLINE {{INT}} __Pyx_mul_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} constant, int *overflow); +#endif #define __Pyx_div_const_{{NAME}}_checking_overflow __Pyx_div_{{NAME}}_checking_overflow +{{if INT == "long long"}}#endif{{endif}} + /////////////// BaseCaseSigned /////////////// +{{if INT == "long long"}}#ifdef HAVE_LONG_LONG{{endif}} + +#if defined(__PYX_HAVE_BUILTIN_OVERFLOW) + +static CYTHON_INLINE {{INT}} __Pyx_add_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { + {{INT}} result; + *overflow |= __builtin_add_overflow(a, b, &result); + return result; +} + +static CYTHON_INLINE {{INT}} __Pyx_sub_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { + {{INT}} result; + *overflow |= __builtin_sub_overflow(a, b, &result); + return result; +} + +static CYTHON_INLINE {{INT}} __Pyx_mul_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { + {{INT}} result; + *overflow |= __builtin_mul_overflow(a, b, &result); + return result; +} + +#else + static CYTHON_INLINE {{INT}} __Pyx_add_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { if ((sizeof({{INT}}) < sizeof(long))) { long big_r = ((long) a) + ((long) b); @@ -151,40 +245,33 @@ static CYTHON_INLINE {{INT}} __Pyx_add_{{NAME}}_checking_overflow({{INT}} a, {{I return r; #endif } else { - // Signed overflow undefined, but unsigned overflow is well defined. - {{INT}} r = ({{INT}}) ((unsigned {{INT}}) a + (unsigned {{INT}}) b); + // Signed overflow undefined, but unsigned overflow is well defined. Casting is + // implementation-defined, but we assume two's complement (see __Pyx_check_twos_complement + // above), and arithmetic in two's-complement is the same as unsigned arithmetic. + unsigned {{INT}} r = (unsigned {{INT}}) a + (unsigned {{INT}}) b; // Overflow happened if the operands have the same sign, but the result // has opposite sign. - // sign(a) == sign(b) != sign(r) - {{INT}} sign_a = __PYX_SIGN_BIT({{INT}}) & a; - {{INT}} sign_b = __PYX_SIGN_BIT({{INT}}) & b; - {{INT}} sign_r = __PYX_SIGN_BIT({{INT}}) & r; - *overflow |= (sign_a == sign_b) & (sign_a != sign_r); - return r; - } -} - -static CYTHON_INLINE {{INT}} __Pyx_add_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { - if (b > 0) { - *overflow |= a > __PYX_MAX({{INT}}) - b; - } else if (b < 0) { - *overflow |= a < __PYX_MIN({{INT}}) - b; + *overflow |= (((unsigned {{INT}})a ^ r) & ((unsigned {{INT}})b ^ r)) >> (8 * sizeof({{INT}}) - 1); + return ({{INT}}) r; } - return a + b; } static CYTHON_INLINE {{INT}} __Pyx_sub_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { - *overflow |= b == __PYX_MIN({{INT}}); - return __Pyx_add_{{NAME}}_checking_overflow(a, -b, overflow); -} - -static CYTHON_INLINE {{INT}} __Pyx_sub_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { - *overflow |= b == __PYX_MIN({{INT}}); - return __Pyx_add_const_{{NAME}}_checking_overflow(a, -b, overflow); + // Compilers don't handle widening as well in the subtraction case, so don't bother + unsigned {{INT}} r = (unsigned {{INT}}) a - (unsigned {{INT}}) b; + // Overflow happened if the operands differing signs, and the result + // has opposite sign to a. + *overflow |= (((unsigned {{INT}})a ^ (unsigned {{INT}})b) & ((unsigned {{INT}})a ^ r)) >> (8 * sizeof({{INT}}) - 1); + return ({{INT}}) r; } static CYTHON_INLINE {{INT}} __Pyx_mul_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { - if ((sizeof({{INT}}) < sizeof(long))) { + // if we have a constant, use the constant version + if (__Pyx_is_constant(b)) { + return __Pyx_mul_const_{{NAME}}_checking_overflow(a, b, overflow); + } else if (__Pyx_is_constant(a)) { + return __Pyx_mul_const_{{NAME}}_checking_overflow(b, a, overflow); + } else if ((sizeof({{INT}}) < sizeof(long))) { long big_r = ((long) a) * ((long) b); {{INT}} r = ({{INT}}) big_r; *overflow |= big_r != r; @@ -197,16 +284,20 @@ static CYTHON_INLINE {{INT}} __Pyx_mul_{{NAME}}_checking_overflow({{INT}} a, {{I return ({{INT}}) r; #endif } else { - {{INT}} prod = a * b; - double dprod = ((double) a) * ((double) b); - // Overflow results in an error of at least 2^sizeof(INT), - // whereas rounding represents an error on the order of 2^(sizeof(INT)-53). - *overflow |= fabs(dprod - prod) > (__PYX_MAX({{INT}}) / 2); - return prod; + return __Pyx_mul_const_{{NAME}}_checking_overflow(a, b, overflow); } } static CYTHON_INLINE {{INT}} __Pyx_mul_const_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { + // note that deliberately all these comparisons are written such that they divide by b; this + // function is used when b is a constant thus the compiler should be able to eliminate the + // (very slow on most CPUs!) division operations + if (__Pyx_is_constant(a) && !__Pyx_is_constant(b)) { + // if a is a compile-time constant and b isn't, swap them + {{INT}} temp = b; + b = a; + a = temp; + } if (b > 1) { *overflow |= a > __PYX_MAX({{INT}}) / b; *overflow |= a < __PYX_MIN({{INT}}) / b; @@ -216,18 +307,21 @@ static CYTHON_INLINE {{INT}} __Pyx_mul_const_{{NAME}}_checking_overflow({{INT}} *overflow |= a > __PYX_MIN({{INT}}) / b; *overflow |= a < __PYX_MAX({{INT}}) / b; } - return a * b; + return ({{INT}}) (((unsigned {{INT}})a) * ((unsigned {{INT}}) b)); } +#endif // defined(__PYX_HAVE_BUILTIN_OVERFLOW) static CYTHON_INLINE {{INT}} __Pyx_div_{{NAME}}_checking_overflow({{INT}} a, {{INT}} b, int *overflow) { if (b == 0) { *overflow |= 1; return 0; } - *overflow |= (a == __PYX_MIN({{INT}})) & (b == -1); - return a / b; + *overflow |= a == __PYX_MIN({{INT}}) && b == -1; + return ({{INT}}) ((unsigned {{INT}}) a / (unsigned {{INT}}) b); } +{{if INT == "long long"}}#endif{{endif}} + /////////////// SizeCheck.init /////////////// //@substitute: naming @@ -293,19 +387,24 @@ static CYTHON_INLINE {{TYPE}} __Pyx_{{BINOP}}_{{NAME}}_checking_overflow({{TYPE} /////////////// LeftShift.proto /////////////// static CYTHON_INLINE {{TYPE}} __Pyx_lshift_{{NAME}}_checking_overflow({{TYPE}} a, {{TYPE}} b, int *overflow) { - *overflow |= + int overflow_check = #if {{SIGNED}} - (b < 0) | + (a < 0) || (b < 0) || #endif - (b > ({{TYPE}}) (8 * sizeof({{TYPE}}))) | (a > (__PYX_MAX({{TYPE}}) >> b)); - return a << b; + // the following must be a _logical_ OR as the RHS is undefined if the LHS is true + (b >= ({{TYPE}}) (8 * sizeof({{TYPE}}))) || (a > (__PYX_MAX({{TYPE}}) >> b)); + if (overflow_check) { + *overflow |= 1; + return 0; + } else { + return a << b; + } } #define __Pyx_lshift_const_{{NAME}}_checking_overflow __Pyx_lshift_{{NAME}}_checking_overflow /////////////// UnaryNegOverflows.proto /////////////// -//FIXME: shouldn't the macro name be prefixed by "__Pyx_" ? Too late now, I guess... // from intobject.c -#define UNARY_NEG_WOULD_OVERFLOW(x) \ +#define __Pyx_UNARY_NEG_WOULD_OVERFLOW(x) \ (((x) < 0) & ((unsigned long)(x) == 0-(unsigned long)(x))) diff --git a/Cython/Utility/Profile.c b/Cython/Utility/Profile.c index a0ab1fa98..20b599e79 100644 --- a/Cython/Utility/Profile.c +++ b/Cython/Utility/Profile.c @@ -6,7 +6,7 @@ // but maybe some other profilers don't. #ifndef CYTHON_PROFILE -#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_PYSTON +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY #define CYTHON_PROFILE 0 #else #define CYTHON_PROFILE 1 @@ -239,12 +239,12 @@ if (CYTHON_TRACE_NOGIL) { \ int ret = 0; \ PyThreadState *tstate; \ - PyGILState_STATE state = PyGILState_Ensure(); \ + PyGILState_STATE state = __Pyx_PyGILState_Ensure(); \ tstate = __Pyx_PyThreadState_Current; \ if (__Pyx_IsTracing(tstate, 0, 0) && tstate->c_tracefunc && $frame_cname->f_trace) { \ ret = __Pyx_call_line_trace_func(tstate, $frame_cname, lineno); \ } \ - PyGILState_Release(state); \ + __Pyx_PyGILState_Release(state); \ if (unlikely(ret)) goto_error; \ } \ } else { \ diff --git a/Cython/Utility/StringTools.c b/Cython/Utility/StringTools.c index 98b5e260e..553585987 100644 --- a/Cython/Utility/StringTools.c +++ b/Cython/Utility/StringTools.c @@ -7,15 +7,73 @@ #include <string> + +//////////////////// ssize_strlen.proto //////////////////// + +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s);/*proto*/ + +//////////////////// ssize_strlen //////////////////// +//@requires: IncludeStringH + +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s) { + size_t len = strlen(s); + if (unlikely(len > PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "byte string is too long"); + return -1; + } + return (Py_ssize_t) len; +} + + +//////////////////// ssize_pyunicode_strlen.proto //////////////////// + +static CYTHON_INLINE Py_ssize_t __Pyx_Py_UNICODE_ssize_strlen(const Py_UNICODE *u);/*proto*/ + +//////////////////// ssize_pyunicode_strlen //////////////////// + +static CYTHON_INLINE Py_ssize_t __Pyx_Py_UNICODE_ssize_strlen(const Py_UNICODE *u) { + size_t len = __Pyx_Py_UNICODE_strlen(u); + if (unlikely(len > PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "Py_UNICODE string is too long"); + return -1; + } + return (Py_ssize_t) len; +} + + //////////////////// InitStrings.proto //////////////////// static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ //////////////////// InitStrings //////////////////// +#if PY_MAJOR_VERSION >= 3 +static int __Pyx_InitString(__Pyx_StringTabEntry t, PyObject **str) { + if (t.is_unicode | t.is_str) { + if (t.intern) { + *str = PyUnicode_InternFromString(t.s); + } else if (t.encoding) { + *str = PyUnicode_Decode(t.s, t.n - 1, t.encoding, NULL); + } else { + *str = PyUnicode_FromStringAndSize(t.s, t.n - 1); + } + } else { + *str = PyBytes_FromStringAndSize(t.s, t.n - 1); + } + if (!*str) + return -1; + // initialise cached hash value + if (PyObject_Hash(*str) == -1) + return -1; + return 0; +} +#endif + static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { while (t->p) { - #if PY_MAJOR_VERSION < 3 + #if PY_MAJOR_VERSION >= 3 /* Python 3+ has unicode identifiers */ + __Pyx_InitString(*t, t->p); + #else if (t->is_unicode) { *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); } else if (t->intern) { @@ -23,24 +81,12 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { } else { *t->p = PyString_FromStringAndSize(t->s, t->n - 1); } - #else /* Python 3+ has unicode identifiers */ - if (t->is_unicode | t->is_str) { - if (t->intern) { - *t->p = PyUnicode_InternFromString(t->s); - } else if (t->encoding) { - *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); - } else { - *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); - } - } else { - *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); - } - #endif if (!*t->p) return -1; // initialise cached hash value if (PyObject_Hash(*t->p) == -1) return -1; + #endif ++t; } return 0; @@ -183,7 +229,7 @@ static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int //@requires: BytesEquals static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) { -#if CYTHON_COMPILING_IN_PYPY +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API return PyObject_RichCompareBool(s1, s2, equals); #else #if PY_MAJOR_VERSION < 3 @@ -294,7 +340,7 @@ static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int eq //@requires: IncludeStringH static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) { -#if CYTHON_COMPILING_IN_PYPY +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API return PyObject_RichCompareBool(s1, s2, equals); #else if (s1 == s2) { @@ -591,6 +637,8 @@ static CYTHON_INLINE PyObject* __Pyx_PyUnicode_Substring( stop = length; if (stop <= start) return __Pyx_NewRef($empty_unicode); + if (start == 0 && stop == length) + return __Pyx_NewRef(text); #if CYTHON_PEP393_ENABLED return PyUnicode_FromKindAndData(PyUnicode_KIND(text), PyUnicode_1BYTE_DATA(text) + start*PyUnicode_KIND(text), stop-start); @@ -835,25 +883,29 @@ static PyObject* __Pyx_PyUnicode_Join(PyObject* value_tuple, Py_ssize_t value_co //@substitute: naming static PyObject* __Pyx_PyUnicode_Join(PyObject* value_tuple, Py_ssize_t value_count, Py_ssize_t result_ulength, - CYTHON_UNUSED Py_UCS4 max_char) { + Py_UCS4 max_char) { #if CYTHON_USE_UNICODE_INTERNALS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS PyObject *result_uval; - int result_ukind; + int result_ukind, kind_shift; Py_ssize_t i, char_pos; void *result_udata; + CYTHON_MAYBE_UNUSED_VAR(max_char); #if CYTHON_PEP393_ENABLED // Py 3.3+ (post PEP-393) result_uval = PyUnicode_New(result_ulength, max_char); if (unlikely(!result_uval)) return NULL; result_ukind = (max_char <= 255) ? PyUnicode_1BYTE_KIND : (max_char <= 65535) ? PyUnicode_2BYTE_KIND : PyUnicode_4BYTE_KIND; + kind_shift = (result_ukind == PyUnicode_4BYTE_KIND) ? 2 : result_ukind - 1; result_udata = PyUnicode_DATA(result_uval); #else // Py 2.x/3.2 (pre PEP-393) result_uval = PyUnicode_FromUnicode(NULL, result_ulength); if (unlikely(!result_uval)) return NULL; result_ukind = sizeof(Py_UNICODE); + kind_shift = (result_ukind == 4) ? 2 : result_ukind - 1; result_udata = PyUnicode_AS_UNICODE(result_uval); #endif + assert(kind_shift == 2 || kind_shift == 1 || kind_shift == 0); char_pos = 0; for (i=0; i < value_count; i++) { @@ -866,12 +918,12 @@ static PyObject* __Pyx_PyUnicode_Join(PyObject* value_tuple, Py_ssize_t value_co ulength = __Pyx_PyUnicode_GET_LENGTH(uval); if (unlikely(!ulength)) continue; - if (unlikely(char_pos + ulength < 0)) + if (unlikely((PY_SSIZE_T_MAX >> kind_shift) - ulength < char_pos)) goto overflow; ukind = __Pyx_PyUnicode_KIND(uval); udata = __Pyx_PyUnicode_DATA(uval); if (!CYTHON_PEP393_ENABLED || ukind == result_ukind) { - memcpy((char *)result_udata + char_pos * result_ukind, udata, (size_t) (ulength * result_ukind)); + memcpy((char *)result_udata + (char_pos << kind_shift), udata, (size_t) (ulength << kind_shift)); } else { #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030300F0 || defined(_PyUnicode_FastCopyCharacters) _PyUnicode_FastCopyCharacters(result_uval, char_pos, uval, 0, ulength); @@ -893,8 +945,9 @@ bad: return NULL; #else // non-CPython fallback - result_ulength++; - value_count++; + CYTHON_UNUSED_VAR(max_char); + CYTHON_UNUSED_VAR(result_ulength); + CYTHON_UNUSED_VAR(value_count); return PyUnicode_Join($empty_unicode, value_tuple); #endif } @@ -1008,11 +1061,11 @@ static CYTHON_INLINE int __Pyx_PyByteArray_AppendObject(PyObject* bytearray, PyO } else #endif #if CYTHON_USE_PYLONG_INTERNALS - if (likely(PyLong_CheckExact(value)) && likely(Py_SIZE(value) == 1 || Py_SIZE(value) == 0)) { - if (Py_SIZE(value) == 0) { + if (likely(PyLong_CheckExact(value)) && likely(__Pyx_PyLong_IsCompact(value))) { + if (__Pyx_PyLong_IsZero(value)) { ival = 0; } else { - ival = ((PyLongObject*)value)->ob_digit[0]; + ival = __Pyx_PyLong_CompactValue(value); if (unlikely(ival > 255)) goto bad_range; } } else @@ -1130,11 +1183,12 @@ static PyObject* __Pyx_PyObject_Format(PyObject* obj, PyObject* format_spec) { likely(PyString_CheckExact(s)) ? PyUnicode_FromEncodedObject(s, NULL, "strict") : \ PyObject_Format(s, f)) #elif CYTHON_USE_TYPE_SLOTS - // Py3 nicely returns unicode strings from str() which makes this quite efficient for builtin types + // Py3 nicely returns unicode strings from str() and repr(), which makes this quite efficient for builtin types. + // In Py3.8+, tp_str() delegates to tp_repr(), so we call tp_repr() directly here. #define __Pyx_PyObject_FormatSimple(s, f) ( \ likely(PyUnicode_CheckExact(s)) ? (Py_INCREF(s), s) : \ - likely(PyLong_CheckExact(s)) ? PyLong_Type.tp_str(s) : \ - likely(PyFloat_CheckExact(s)) ? PyFloat_Type.tp_str(s) : \ + likely(PyLong_CheckExact(s)) ? PyLong_Type.tp_repr(s) : \ + likely(PyFloat_CheckExact(s)) ? PyFloat_Type.tp_repr(s) : \ PyObject_Format(s, f)) #else #define __Pyx_PyObject_FormatSimple(s, f) ( \ @@ -1193,3 +1247,22 @@ static CYTHON_INLINE PyObject* __Pyx_PyUnicode_Unicode(PyObject *obj) { #define __Pyx_PyObject_Unicode(obj) \ (likely(PyUnicode_CheckExact(obj)) ? __Pyx_NewRef(obj) : PyObject_Unicode(obj)) #endif + + +//////////////////// PyStr_Str.proto //////////////////// + +static CYTHON_INLINE PyObject* __Pyx_PyStr_Str(PyObject *obj);/*proto*/ + +//////////////////// PyStr_Str //////////////////// + +static CYTHON_INLINE PyObject* __Pyx_PyStr_Str(PyObject *obj) { + if (unlikely(obj == Py_None)) + obj = PYIDENT("None"); + return __Pyx_NewRef(obj); +} + + +//////////////////// PyObject_Str.proto //////////////////// + +#define __Pyx_PyObject_Str(obj) \ + (likely(PyString_CheckExact(obj)) ? __Pyx_NewRef(obj) : PyObject_Str(obj)) diff --git a/Cython/Utility/TestCythonScope.pyx b/Cython/Utility/TestCythonScope.pyx index f585be298..7da7665c3 100644 --- a/Cython/Utility/TestCythonScope.pyx +++ b/Cython/Utility/TestCythonScope.pyx @@ -1,6 +1,11 @@ ########## TestClass ########## # These utilities are for testing purposes +# The "cythonscope" test calls METH_O functions with their (self, arg) signature. +# cython: always_allow_keywords=False + +from __future__ import print_function + cdef extern from *: cdef object __pyx_test_dep(object) @@ -12,32 +17,32 @@ cdef class TestClass(object): self.value = value def __str__(self): - return 'TestClass(%d)' % self.value + return f'TestClass({self.value})' cdef cdef_method(self, int value): - print 'Hello from cdef_method', value + print('Hello from cdef_method', value) cpdef cpdef_method(self, int value): - print 'Hello from cpdef_method', value + print('Hello from cpdef_method', value) def def_method(self, int value): - print 'Hello from def_method', value + print('Hello from def_method', value) @cname('cdef_cname') cdef cdef_cname_method(self, int value): - print "Hello from cdef_cname_method", value + print("Hello from cdef_cname_method", value) @cname('cpdef_cname') cpdef cpdef_cname_method(self, int value): - print "Hello from cpdef_cname_method", value + print("Hello from cpdef_cname_method", value) @cname('def_cname') def def_cname_method(self, int value): - print "Hello from def_cname_method", value + print("Hello from def_cname_method", value) @cname('__pyx_test_call_other_cy_util') cdef test_call(obj): - print 'test_call' + print('test_call') __pyx_test_dep(obj) @cname('__pyx_TestClass_New') @@ -46,19 +51,20 @@ cdef _testclass_new(int value): ########### TestDep ########## +from __future__ import print_function + @cname('__pyx_test_dep') cdef test_dep(obj): - print 'test_dep', obj + print('test_dep', obj) ########## TestScope ########## @cname('__pyx_testscope') cdef object _testscope(int value): - return "hello from cython scope, value=%d" % value + return f"hello from cython scope, value={value}" ########## View.TestScope ########## @cname('__pyx_view_testscope') cdef object _testscope(int value): - return "hello from cython.view scope, value=%d" % value - + return f"hello from cython.view scope, value={value}" diff --git a/Cython/Utility/TypeConversion.c b/Cython/Utility/TypeConversion.c index 404814907..3e730e0fb 100644 --- a/Cython/Utility/TypeConversion.c +++ b/Cython/Utility/TypeConversion.c @@ -68,9 +68,9 @@ static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) -#define __Pyx_PyObject_AsWritableString(s) ((char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_AsWritableSString(s) ((signed char*) __Pyx_PyObject_AsString(s)) -#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableString(s) ((char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) #define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) #define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) #define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) @@ -80,12 +80,23 @@ static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); #define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) // There used to be a Py_UNICODE_strlen() in CPython 3.x, but it is deprecated since Py3.3. -static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) { +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const wchar_t *u) +{ + const wchar_t *u_end = u; + while (*u_end++) ; + return (size_t)(u_end - u - 1); +} +#else +static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) +{ const Py_UNICODE *u_end = u; while (*u_end++) ; return (size_t)(u_end - u - 1); } +#endif +#define __Pyx_PyUnicode_FromOrdinal(o) PyUnicode_FromOrdinal((int)o) #define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) #define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode #define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode @@ -116,7 +127,49 @@ static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*); #else #define __Pyx_PyNumber_Int(x) (PyInt_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Int(x)) #endif -#define __Pyx_PyNumber_Float(x) (PyFloat_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Float(x)) +// __Pyx_PyNumber_Float is now in its own section since it has dependencies (needed to make +// string conversion work the same in all circumstances). + +#if CYTHON_USE_PYLONG_INTERNALS + #if PY_VERSION_HEX >= 0x030C00A7 + #define __Pyx_PyLong_Sign(x) (((PyLongObject*)x)->long_value.lv_tag & 3) + #define __Pyx_PyLong_IsNeg(x) ((__Pyx_PyLong_Sign(x) & 2) != 0) + #define __Pyx_PyLong_IsNonNeg(x) (!__Pyx_PyLong_IsNeg(x)) + #define __Pyx_PyLong_IsZero(x) (__Pyx_PyLong_Sign(x) & 1) + #define __Pyx_PyLong_IsPos(x) (__Pyx_PyLong_Sign(x) == 0) + #define __Pyx_PyLong_IsCompact(x) (((PyLongObject*)x)->long_value.lv_tag < (2 << 3)) // (2 << NON_SIZE_BITS) + #define __Pyx_PyLong_CompactValue(x) ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * (Py_ssize_t) __Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_CompactValueUnsigned(x) (__Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) (((PyLongObject*)x)->long_value.lv_tag >> 3) // (>> NON_SIZE_BITS) + #define __Pyx_PyLong_SignedDigitCount(x) \ + ((1 - (Py_ssize_t) (((PyLongObject*)x)->long_value.lv_tag & 3)) * (Py_ssize_t) (((PyLongObject*)x)->long_value.lv_tag >> 3)) // (>> NON_SIZE_BITS) + + // CPython 3.12 requires C99 + typedef Py_ssize_t __Pyx_compact_pylong; + typedef size_t __Pyx_compact_upylong; + + #else // Py < 3.12 + #define __Pyx_PyLong_IsNeg(x) (Py_SIZE(x) < 0) + #define __Pyx_PyLong_IsNonNeg(x) (Py_SIZE(x) >= 0) + #define __Pyx_PyLong_IsZero(x) (Py_SIZE(x) == 0) + #define __Pyx_PyLong_IsPos(x) (Py_SIZE(x) > 0) + #define __Pyx_PyLong_IsCompact(x) (Py_SIZE(x) == 0 || Py_SIZE(x) == 1 || Py_SIZE(x) == -1) + #define __Pyx_PyLong_CompactValue(x) \ + ((Py_SIZE(x) == 0) ? (sdigit) 0 : ((Py_SIZE(x) < 0) ? -(sdigit)__Pyx_PyLong_Digits(x)[0] : (sdigit)__Pyx_PyLong_Digits(x)[0])) + #define __Pyx_PyLong_CompactValueUnsigned(x) ((Py_SIZE(x) == 0) ? 0 : __Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) __Pyx_sst_abs(Py_SIZE(x)) + #define __Pyx_PyLong_SignedDigitCount(x) Py_SIZE(x) + + typedef sdigit __Pyx_compact_pylong; + typedef digit __Pyx_compact_upylong; + #endif + + #if PY_VERSION_HEX >= 0x030C00A5 + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->long_value.ob_digit) + #else + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->ob_digit) + #endif +#endif #if PY_MAJOR_VERSION < 3 && __PYX_DEFAULT_STRING_ENCODING_IS_ASCII static int __Pyx_sys_getdefaultencoding_not_ascii; @@ -271,7 +324,7 @@ static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ } else #endif /* __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT */ -#if (!CYTHON_COMPILING_IN_PYPY) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) +#if (!CYTHON_COMPILING_IN_PYPY && !CYTHON_COMPILING_IN_LIMITED_API) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) if (PyByteArray_Check(o)) { *length = PyByteArray_GET_SIZE(o); return PyByteArray_AS_STRING(o); @@ -304,23 +357,27 @@ static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) { } static PyObject* __Pyx_PyNumber_IntOrLongWrongResultType(PyObject* result, const char* type_name) { + __Pyx_TypeName result_type_name = __Pyx_PyType_GetName(Py_TYPE(result)); #if PY_MAJOR_VERSION >= 3 if (PyLong_Check(result)) { // CPython issue #17576: warn if 'result' not of exact type int. if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "__int__ returned non-int (type %.200s). " - "The ability to return an instance of a strict subclass of int " - "is deprecated, and may be removed in a future version of Python.", - Py_TYPE(result)->tp_name)) { + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME "). " + "The ability to return an instance of a strict subclass of int is deprecated, " + "and may be removed in a future version of Python.", + result_type_name)) { + __Pyx_DECREF_TypeName(result_type_name); Py_DECREF(result); return NULL; } + __Pyx_DECREF_TypeName(result_type_name); return result; } #endif PyErr_Format(PyExc_TypeError, - "__%.4s__ returned non-%.4s (type %.200s)", - type_name, type_name, Py_TYPE(result)->tp_name); + "__%.4s__ returned non-%.4s (type " __Pyx_FMT_TYPENAME ")", + type_name, type_name, result_type_name); + __Pyx_DECREF_TypeName(result_type_name); Py_DECREF(result); return NULL; } @@ -390,14 +447,12 @@ static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { #endif if (likely(PyLong_CheckExact(b))) { #if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)b)->ob_digit; - const Py_ssize_t size = Py_SIZE(b); // handle most common case first to avoid indirect branch and optimise branch prediction - if (likely(__Pyx_sst_abs(size) <= 1)) { - ival = likely(size) ? digits[0] : 0; - if (size == -1) ival = -ival; - return ival; + if (likely(__Pyx_PyLong_IsCompact(b))) { + return __Pyx_PyLong_CompactValue(b); } else { + const digit* digits = __Pyx_PyLong_Digits(b); + const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount(b); switch (size) { {{for _size in (2, 3, 4)}} {{for _case in (_size, -_size)}} @@ -449,12 +504,50 @@ static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t ival) { return PyInt_FromSize_t(ival); } +/////////////// pynumber_float.proto /////////////// + +static CYTHON_INLINE PyObject* __Pyx__PyNumber_Float(PyObject* obj); /* proto */ +#define __Pyx_PyNumber_Float(x) (PyFloat_CheckExact(x) ? __Pyx_NewRef(x) : __Pyx__PyNumber_Float(x)) + +/////////////// pynumber_float /////////////// +//@requires: Optimize.c::pybytes_as_double +//@requires: Optimize.c::pyunicode_as_double + +static CYTHON_INLINE PyObject* __Pyx__PyNumber_Float(PyObject* obj) { + // 'obj is PyFloat' is handled in the calling macro + double val; + if (PyLong_CheckExact(obj)) { +#if CYTHON_USE_PYLONG_INTERNALS + if (likely(__Pyx_PyLong_IsCompact(obj))) { + val = (double) __Pyx_PyLong_CompactValue(obj); + goto no_error; + } +#endif + val = PyLong_AsDouble(obj); + } else if (PyUnicode_CheckExact(obj)) { + val = __Pyx_PyUnicode_AsDouble(obj); + } else if (PyBytes_CheckExact(obj)) { + val = __Pyx_PyBytes_AsDouble(obj); + } else if (PyByteArray_CheckExact(obj)) { + val = __Pyx_PyByteArray_AsDouble(obj); + } else { + return PyNumber_Float(obj); + } + + if (unlikely(val == -1 && PyErr_Occurred())) { + return NULL; + } +#if CYTHON_USE_PYLONG_INTERNALS +no_error: +#endif + return PyFloat_FromDouble(val); +} /////////////// GCCDiagnostics.proto /////////////// // GCC diagnostic pragmas were introduced in GCC 4.6 // Used to silence conversion warnings that are ok but cannot be avoided. -#if defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#if !defined(__INTEL_COMPILER) && defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) #define __Pyx_HAS_GCC_DIAGNOSTIC #endif @@ -484,24 +577,41 @@ bad: /////////////// FromPyCTupleUtility.proto /////////////// -static {{struct_type_decl}} {{funcname}}(PyObject *); +static CYTHON_INLINE {{struct_type_decl}} {{funcname}}(PyObject *); /////////////// FromPyCTupleUtility /////////////// -static {{struct_type_decl}} {{funcname}}(PyObject * o) { - {{struct_type_decl}} result; - - if (!PyTuple_Check(o) || PyTuple_GET_SIZE(o) != {{size}}) { - PyErr_Format(PyExc_TypeError, "Expected %.16s of size %d, got %.200s", "a tuple", {{size}}, Py_TYPE(o)->tp_name); - goto bad; - } #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS +static {{struct_type_decl}} __Pyx_tuple_{{funcname}}(PyObject * o) { + {{struct_type_decl}} result; + {{for ix, component in enumerate(components):}} {{py:attr = "result.f%s" % ix}} {{attr}} = {{component.from_py_function}}(PyTuple_GET_ITEM(o, {{ix}})); if ({{component.error_condition(attr)}}) goto bad; {{endfor}} -#else + + return result; +bad: + return result; +} +#endif + +static {{struct_type_decl}} __Pyx_seq_{{funcname}}(PyObject * o) { + {{struct_type_decl}} result; + + if (unlikely(!PySequence_Check(o))) { + __Pyx_TypeName o_type_name = __Pyx_PyType_GetName(Py_TYPE(o)); + PyErr_Format(PyExc_TypeError, + "Expected a sequence of size %zd, got " __Pyx_FMT_TYPENAME, (Py_ssize_t) {{size}}, o_type_name); + __Pyx_DECREF_TypeName(o_type_name); + goto bad; + } else if (unlikely(PySequence_Length(o) != {{size}})) { + PyErr_Format(PyExc_TypeError, + "Expected a sequence of size %zd, got size %zd", (Py_ssize_t) {{size}}, PySequence_Length(o)); + goto bad; + } + { PyObject *item; {{for ix, component in enumerate(components):}} @@ -512,13 +622,22 @@ static {{struct_type_decl}} {{funcname}}(PyObject * o) { if ({{component.error_condition(attr)}}) goto bad; {{endfor}} } -#endif return result; bad: return result; } +static CYTHON_INLINE {{struct_type_decl}} {{funcname}}(PyObject * o) { + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + if (likely(PyTuple_Check(o) && PyTuple_GET_SIZE(o) == {{size}})) { + return __Pyx_tuple_{{funcname}}(o); + } + #endif + + return __Pyx_seq_{{funcname}}(o); +} + /////////////// UnicodeAsUCS4.proto /////////////// @@ -784,10 +903,10 @@ static CYTHON_INLINE PyObject* {{TO_PY_FUNCTION}}({{TYPE}} value, Py_ssize_t wid } } while (unlikely(remaining != 0)); - if (last_one_off) { - assert(*dpos == '0'); - dpos++; - } + // Correct dpos by 1 if we read an excess digit. + assert(!last_one_off || *dpos == '0'); + dpos += last_one_off; + length = end - dpos; ulength = length; prepend_sign = 0; @@ -886,7 +1005,7 @@ static CYTHON_INLINE {{TYPE}} {{FROM_PY_FUNCTION}}(PyObject *x) { const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 if (likely(PyInt_Check(x))) { - if (sizeof({{TYPE}}) < sizeof(long)) { + if ((sizeof({{TYPE}}) < sizeof(long))) { __PYX_VERIFY_RETURN_INT({{TYPE}}, long, PyInt_AS_LONG(x)) } else { long val = PyInt_AS_LONG(x); @@ -900,21 +1019,28 @@ static CYTHON_INLINE {{TYPE}} {{FROM_PY_FUNCTION}}(PyObject *x) { if (likely(PyLong_Check(x))) { if (is_unsigned) { #if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return ({{TYPE}}) 0; - case 1: __PYX_VERIFY_RETURN_INT({{TYPE}}, digit, digits[0]) - {{for _size in (2, 3, 4)}} - case {{_size}}: - if (8 * sizeof({{TYPE}}) > {{_size-1}} * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT({{TYPE}}, unsigned long, {{pylong_join(_size, 'digits')}}) - } else if (8 * sizeof({{TYPE}}) >= {{_size}} * PyLong_SHIFT) { - return ({{TYPE}}) {{pylong_join(_size, 'digits', TYPE)}}; + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + //} else if (__Pyx_PyLong_IsZero(x)) { + // return ({{TYPE}}) 0; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT({{TYPE}}, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + {{for _size in (2, 3, 4)}} + case {{_size}}: + if ((8 * sizeof({{TYPE}}) > {{_size-1}} * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT({{TYPE}}, unsigned long, {{pylong_join(_size, 'digits')}}) + } else if ((8 * sizeof({{TYPE}}) >= {{_size}} * PyLong_SHIFT)) { + return ({{TYPE}}) {{pylong_join(_size, 'digits', TYPE)}}; + } } - } - break; - {{endfor}} + break; + {{endfor}} + } } #endif #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 @@ -931,71 +1057,170 @@ static CYTHON_INLINE {{TYPE}} {{FROM_PY_FUNCTION}}(PyObject *x) { goto raise_neg_overflow; } #endif - if (sizeof({{TYPE}}) <= sizeof(unsigned long)) { + if ((sizeof({{TYPE}}) <= sizeof(unsigned long))) { __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, unsigned long, PyLong_AsUnsignedLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof({{TYPE}}) <= sizeof(unsigned PY_LONG_LONG)) { + } else if ((sizeof({{TYPE}}) <= sizeof(unsigned PY_LONG_LONG))) { __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) #endif } } else { // signed #if CYTHON_USE_PYLONG_INTERNALS - const digit* digits = ((PyLongObject*)x)->ob_digit; - switch (Py_SIZE(x)) { - case 0: return ({{TYPE}}) 0; - case -1: __PYX_VERIFY_RETURN_INT({{TYPE}}, sdigit, (sdigit) (-(sdigit)digits[0])) - case 1: __PYX_VERIFY_RETURN_INT({{TYPE}}, digit, +digits[0]) - {{for _size in (2, 3, 4)}} - {{for _case in (-_size, _size)}} - case {{_case}}: - if (8 * sizeof({{TYPE}}){{' - 1' if _case < 0 else ''}} > {{_size-1}} * PyLong_SHIFT) { - if (8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT) { - __PYX_VERIFY_RETURN_INT({{TYPE}}, {{'long' if _case < 0 else 'unsigned long'}}, {{'-(long) ' if _case < 0 else ''}}{{pylong_join(_size, 'digits')}}) - } else if (8 * sizeof({{TYPE}}) - 1 > {{_size}} * PyLong_SHIFT) { - return ({{TYPE}}) ({{'((%s)-1)*' % TYPE if _case < 0 else ''}}{{pylong_join(_size, 'digits', TYPE)}}); + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT({{TYPE}}, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + {{for _size in (2, 3, 4)}} + {{for _case in (-_size, _size)}} + case {{_case}}: + if ((8 * sizeof({{TYPE}}){{' - 1' if _case < 0 else ''}} > {{_size-1}} * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > {{_size}} * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT({{TYPE}}, {{'long' if _case < 0 else 'unsigned long'}}, {{'-(long) ' if _case < 0 else ''}}{{pylong_join(_size, 'digits')}}) + } else if ((8 * sizeof({{TYPE}}) - 1 > {{_size}} * PyLong_SHIFT)) { + return ({{TYPE}}) ({{'((%s)-1)*' % TYPE if _case < 0 else ''}}{{pylong_join(_size, 'digits', TYPE)}}); + } } - } - break; - {{endfor}} - {{endfor}} + break; + {{endfor}} + {{endfor}} + } } #endif - if (sizeof({{TYPE}}) <= sizeof(long)) { + if ((sizeof({{TYPE}}) <= sizeof(long))) { __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, long, PyLong_AsLong(x)) #ifdef HAVE_LONG_LONG - } else if (sizeof({{TYPE}}) <= sizeof(PY_LONG_LONG)) { + } else if ((sizeof({{TYPE}}) <= sizeof(PY_LONG_LONG))) { __PYX_VERIFY_RETURN_INT_EXC({{TYPE}}, PY_LONG_LONG, PyLong_AsLongLong(x)) #endif } } + + {{if IS_ENUM}} + PyErr_SetString(PyExc_RuntimeError, + "_PyLong_AsByteArray() not available, cannot convert large enums"); + return ({{TYPE}}) -1; + {{else}} + // large integer type and no access to PyLong internals => allow for a more expensive conversion { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) - PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); -#else {{TYPE}} val; PyObject *v = __Pyx_PyNumber_IntOrLong(x); - #if PY_MAJOR_VERSION < 3 +#if PY_MAJOR_VERSION < 3 if (likely(v) && !PyLong_Check(v)) { PyObject *tmp = v; v = PyNumber_Long(tmp); Py_DECREF(tmp); } - #endif +#endif if (likely(v)) { + int ret = -1; +#if !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) int one = 1; int is_little = (int)*(unsigned char *)&one; unsigned char *bytes = (unsigned char *)&val; - int ret = _PyLong_AsByteArray((PyLongObject *)v, - bytes, sizeof(val), - is_little, !is_unsigned); + ret = _PyLong_AsByteArray((PyLongObject *)v, + bytes, sizeof(val), + is_little, !is_unsigned); +#else +// Inefficient copy of bit chunks through the C-API. Probably still better than a "cannot do this" exception. + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + long idigit; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + + // use exact PyLong to prevent user defined &&/<</etc. implementations + if (unlikely(!PyLong_CheckExact(v))) { + PyObject *tmp = v; + v = PyNumber_Long(v); + assert(PyLong_CheckExact(v)); + Py_DECREF(tmp); + if (unlikely(!v)) return ({{TYPE}}) -1; + } + +#if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030B0000 + if (Py_SIZE(x) == 0) + return ({{TYPE}}) 0; + is_negative = Py_SIZE(x) < 0; +#else + { + // misuse Py_False as a quick way to compare to a '0' int object + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return ({{TYPE}}) -1; + is_negative = result == 1; + } +#endif + + if (is_unsigned && unlikely(is_negative)) { + goto raise_neg_overflow; + } else if (is_negative) { + // bit-invert to make sure we can safely convert it + stepval = PyNumber_Invert(v); + if (unlikely(!stepval)) + return ({{TYPE}}) -1; + } else { + stepval = __Pyx_NewRef(v); + } + + // unpack full chunks of bits + val = ({{TYPE}}) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof({{TYPE}}) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + + val |= (({{TYPE}}) idigit) << bits; + + #if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030B0000 + if (Py_SIZE(stepval) == 0) + goto unpacking_done; + #endif + } + + // detect overflow when adding the last bits + idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof({{TYPE}}) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= (({{TYPE}}) idigit) << bits; + + #if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030B0000 + unpacking_done: + #endif + // handle sign and overflow into sign bit + if (!is_unsigned) { + // gcc warns about unsigned (val < 0) => test sign bit instead + if (unlikely(val & ((({{TYPE}}) 1) << (sizeof({{TYPE}}) * 8 - 1)))) + goto raise_overflow; + // undo the PyNumber_Invert() above + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif Py_DECREF(v); if (likely(!ret)) return val; } -#endif return ({{TYPE}}) -1; } + {{endif}} } else { {{TYPE}} val; PyObject *tmp = __Pyx_PyNumber_IntOrLong(x); diff --git a/Cython/Utility/UFuncs.pyx b/Cython/Utility/UFuncs.pyx new file mode 100644 index 000000000..c2043e8cc --- /dev/null +++ b/Cython/Utility/UFuncs.pyx @@ -0,0 +1,51 @@ +##################### UFuncDefinition ###################### + +cdef extern from *: + ctypedef int npy_intp + struct PyObject + PyObject* __Pyx_NewRef(object) + {{inline_func_declaration}} + +# variable names have to come from tempita to avoid duplication +@cname("{{func_cname}}") +cdef void {{func_cname}}(char **args, const npy_intp *dimensions, const npy_intp* steps, void* data) except * {{"nogil" if will_be_called_without_gil else ""}}: + cdef npy_intp i + cdef npy_intp n = dimensions[0] + {{for idx, tp in enumerate(in_types)}} + cdef char* in_{{idx}} = args[{{idx}}] + cdef {{tp.empty_declaration_code(pyrex=True)}} cast_in_{{idx}} + {{endfor}} + {{for idx, tp in enumerate(out_types)}} + cdef char* out_{{idx}} = args[{{idx+len(in_types)}}] + cdef {{tp.empty_declaration_code(pyrex=True)}} cast_out_{{idx}} + {{endfor}} + {{for idx in range(len(out_types)+len(in_types))}} + cdef npy_intp step_{{idx}} = steps[{{idx}}] + {{endfor}} + + {{"with gil" if (not nogil and will_be_called_without_gil) else "if True"}}: + for i in range(n): + {{for idx, tp in enumerate(in_types)}} + {{if tp.is_pyobject}} + cast_in_{{idx}} = (<{{tp.empty_declaration_code(pyrex=True)}}>(<void**>in_{{idx}})[0]) + {{else}} + cast_in_{{idx}} = (<{{tp.empty_declaration_code(pyrex=True)}}*>in_{{idx}})[0] + {{endif}} + {{endfor}} + + {{", ".join("cast_out_{}".format(idx) for idx in range(len(out_types)))}} = \ + {{inline_func_call}}({{", ".join("cast_in_{}".format(idx) for idx in range(len(in_types)))}}) + + {{for idx, tp in enumerate(out_types)}} + {{if tp.is_pyobject}} + (<void**>out_{{idx}})[0] = <void*>__Pyx_NewRef(cast_out_{{idx}}) + {{else}} + (<{{tp.empty_declaration_code(pyrex=True)}}*>out_{{idx}})[0] = cast_out_{{idx}} + {{endif}} + {{endfor}} + {{for idx in range(len(in_types))}} + in_{{idx}} += step_{{idx}} + {{endfor}} + {{for idx in range(len(out_types))}} + out_{{idx}} += step_{{idx+len(in_types)}} + {{endfor}} diff --git a/Cython/Utility/UFuncs_C.c b/Cython/Utility/UFuncs_C.c new file mode 100644 index 000000000..e7ce8812e --- /dev/null +++ b/Cython/Utility/UFuncs_C.c @@ -0,0 +1,42 @@ +///////////////////////// UFuncsInit.proto ///////////////////////// +//@proto_block: utility_code_proto_before_types + +#include <numpy/arrayobject.h> +#include <numpy/ufuncobject.h> + +// account for change in type of arguments to PyUFuncGenericFunction in Numpy 1.19.x +// Unfortunately we can only test against Numpy version 1.20.x since it wasn't marked +// as an API break. Therefore, I'm "solving" the issue by casting function pointer types +// on lower Numpy versions. +#if NPY_API_VERSION >= 0x0000000e // Numpy 1.20.x +#define __PYX_PYUFUNCGENERICFUNCTION_CAST(x) x +#else +#define __PYX_PYUFUNCGENERICFUNCTION_CAST(x) (PyUFuncGenericFunction)x +#endif + +/////////////////////// UFuncConsts.proto //////////////////// + +// getter functions because we can't forward-declare arrays +static PyUFuncGenericFunction* {{ufunc_funcs_name}}(void); /* proto */ +static char* {{ufunc_types_name}}(void); /* proto */ +static void* {{ufunc_data_name}}[] = {NULL}; // always null + +/////////////////////// UFuncConsts ///////////////////////// + +static PyUFuncGenericFunction* {{ufunc_funcs_name}}(void) { + static PyUFuncGenericFunction arr[] = { + {{for loop, cname in looper(func_cnames)}} + __PYX_PYUFUNCGENERICFUNCTION_CAST(&{{cname}}){{if not loop.last}},{{endif}} + {{endfor}} + }; + return arr; +} + +static char* {{ufunc_types_name}}(void) { + static char arr[] = { + {{for loop, tp in looper(type_constants)}} + {{tp}}{{if not loop.last}},{{endif}} + {{endfor}} + }; + return arr; +} diff --git a/Cython/Utils.pxd b/Cython/Utils.pxd new file mode 100644 index 000000000..3b64c610e --- /dev/null +++ b/Cython/Utils.pxd @@ -0,0 +1,3 @@ + +cdef class _TryFinallyGeneratorContextManager: + cdef object _gen diff --git a/Cython/Utils.py b/Cython/Utils.py index 13f83fb75..ffcee9dc3 100644 --- a/Cython/Utils.py +++ b/Cython/Utils.py @@ -1,10 +1,18 @@ -# -# Cython -- Things that don't belong -# anywhere else in particular -# +""" +Cython -- Things that don't belong anywhere else in particular +""" from __future__ import absolute_import +import cython + +cython.declare( + basestring=object, + os=object, sys=object, re=object, io=object, codecs=object, glob=object, shutil=object, tempfile=object, + cython_version=object, + _function_caches=list, _parse_file_version=object, _match_file_encoding=object, +) + try: from __builtin__ import basestring except ImportError: @@ -20,31 +28,97 @@ import sys import re import io import codecs +import glob import shutil import tempfile -from contextlib import contextmanager +from functools import wraps + +from . import __version__ as cython_version + +PACKAGE_FILES = ("__init__.py", "__init__.pyc", "__init__.pyx", "__init__.pxd") + +_build_cache_name = "__{0}_cache".format +_CACHE_NAME_PATTERN = re.compile(r"^__(.+)_cache$") modification_time = os.path.getmtime +GENERATED_BY_MARKER = "/* Generated by Cython %s */" % cython_version +GENERATED_BY_MARKER_BYTES = GENERATED_BY_MARKER.encode('us-ascii') + + +class _TryFinallyGeneratorContextManager(object): + """ + Fast, bare minimum @contextmanager, only for try-finally, not for exception handling. + """ + def __init__(self, gen): + self._gen = gen + + def __enter__(self): + return next(self._gen) + + def __exit__(self, exc_type, exc_val, exc_tb): + try: + next(self._gen) + except (StopIteration, GeneratorExit): + pass + + +def try_finally_contextmanager(gen_func): + @wraps(gen_func) + def make_gen(*args, **kwargs): + return _TryFinallyGeneratorContextManager(gen_func(*args, **kwargs)) + return make_gen + + _function_caches = [] + + def clear_function_caches(): for cache in _function_caches: cache.clear() + def cached_function(f): cache = {} _function_caches.append(cache) uncomputed = object() + + @wraps(f) def wrapper(*args): res = cache.get(args, uncomputed) if res is uncomputed: res = cache[args] = f(*args) return res + wrapper.uncached = f return wrapper + +def _find_cache_attributes(obj): + """The function iterates over the attributes of the object and, + if it finds the name of the cache, it returns it and the corresponding method name. + The method may not be present in the object. + """ + for attr_name in dir(obj): + match = _CACHE_NAME_PATTERN.match(attr_name) + if match is not None: + yield attr_name, match.group(1) + + +def clear_method_caches(obj): + """Removes every cache found in the object, + if a corresponding method exists for that cache. + """ + for cache_name, method_name in _find_cache_attributes(obj): + if hasattr(obj, method_name): + delattr(obj, cache_name) + # if there is no corresponding method, then we assume + # that this attribute was not created by our cached method + + def cached_method(f): - cache_name = '__%s_cache' % f.__name__ + cache_name = _build_cache_name(f.__name__) + def wrapper(self, *args): cache = getattr(self, cache_name, None) if cache is None: @@ -54,8 +128,10 @@ def cached_method(f): return cache[args] res = cache[args] = f(self, *args) return res + return wrapper + def replace_suffix(path, newsuf): base, _ = os.path.splitext(path) return base + newsuf @@ -81,6 +157,9 @@ def castrate_file(path, st): # failed compilation. # Also sets access and modification times back to # those specified by st (a stat struct). + if not is_cython_generated_file(path, allow_failed=True, if_not_found=False): + return + try: f = open_new_file(path) except EnvironmentError: @@ -92,6 +171,42 @@ def castrate_file(path, st): if st: os.utime(path, (st.st_atime, st.st_mtime-1)) + +def is_cython_generated_file(path, allow_failed=False, if_not_found=True): + failure_marker = b"#error Do not use this file, it is the result of a failed Cython compilation." + file_content = None + if os.path.exists(path): + try: + with open(path, "rb") as f: + file_content = f.read(len(failure_marker)) + except (OSError, IOError): + pass # Probably just doesn't exist any more + + if file_content is None: + # file does not exist (yet) + return if_not_found + + return ( + # Cython C file? + file_content.startswith(b"/* Generated by Cython ") or + # Cython output file after previous failures? + (allow_failed and file_content == failure_marker) or + # Let's allow overwriting empty files as well. They might have resulted from previous failures. + not file_content + ) + + +def file_generated_by_this_cython(path): + file_content = b'' + if os.path.exists(path): + try: + with open(path, "rb") as f: + file_content = f.read(len(GENERATED_BY_MARKER_BYTES)) + except (OSError, IOError): + pass # Probably just doesn't exist any more + return file_content and file_content.startswith(GENERATED_BY_MARKER_BYTES) + + def file_newer_than(path, time): ftime = modification_time(path) return ftime > time @@ -134,24 +249,31 @@ def find_root_package_dir(file_path): else: return dir + @cached_function -def check_package_dir(dir, package_names): +def check_package_dir(dir_path, package_names): + namespace = True for dirname in package_names: - dir = os.path.join(dir, dirname) - if not is_package_dir(dir): - return None - return dir + dir_path = os.path.join(dir_path, dirname) + has_init = contains_init(dir_path) + if has_init: + namespace = False + return dir_path, namespace + @cached_function -def is_package_dir(dir_path): - for filename in ("__init__.py", - "__init__.pyc", - "__init__.pyx", - "__init__.pxd"): +def contains_init(dir_path): + for filename in PACKAGE_FILES: path = os.path.join(dir_path, filename) if path_exists(path): return 1 + +def is_package_dir(dir_path): + if contains_init(dir_path): + return 1 + + @cached_function def path_exists(path): # try on the filesystem first @@ -176,6 +298,40 @@ def path_exists(path): pass return False + +_parse_file_version = re.compile(r".*[.]cython-([0-9]+)[.][^./\\]+$").findall + + +@cached_function +def find_versioned_file(directory, filename, suffix, + _current_version=int(re.sub(r"^([0-9]+)[.]([0-9]+).*", r"\1\2", cython_version))): + """ + Search a directory for versioned pxd files, e.g. "lib.cython-30.pxd" for a Cython 3.0+ version. + + @param directory: the directory to search + @param filename: the filename without suffix + @param suffix: the filename extension including the dot, e.g. ".pxd" + @return: the file path if found, or None + """ + assert not suffix or suffix[:1] == '.' + path_prefix = os.path.join(directory, filename) + + matching_files = glob.glob(path_prefix + ".cython-*" + suffix) + path = path_prefix + suffix + if not os.path.exists(path): + path = None + best_match = (-1, path) # last resort, if we do not have versioned .pxd files + + for path in matching_files: + versions = _parse_file_version(path) + if versions: + int_version = int(versions[0]) + # Let's assume no duplicates. + if best_match[0] < int_version <= _current_version: + best_match = (int_version, path) + return best_match[1] + + # file name encodings def decode_filename(filename): @@ -189,12 +345,13 @@ def decode_filename(filename): pass return filename + # support for source file encoding detection _match_file_encoding = re.compile(br"(\w*coding)[:=]\s*([-\w.]+)").search -def detect_opened_file_encoding(f): +def detect_opened_file_encoding(f, default='UTF-8'): # PEPs 263 and 3120 # Most of the time the first two lines fall in the first couple of hundred chars, # and this bulk read/split is much faster. @@ -206,6 +363,7 @@ def detect_opened_file_encoding(f): lines = start.split(b"\n") if not data: break + m = _match_file_encoding(lines[0]) if m and m.group(1) != b'c_string_encoding': return m.group(2).decode('iso8859-1') @@ -213,7 +371,7 @@ def detect_opened_file_encoding(f): m = _match_file_encoding(lines[1]) if m: return m.group(2).decode('iso8859-1') - return "UTF-8" + return default def skip_bom(f): @@ -333,7 +491,7 @@ def get_cython_cache_dir(): return os.path.expanduser(os.path.join('~', '.cython')) -@contextmanager +@try_finally_contextmanager def captured_fd(stream=2, encoding=None): orig_stream = os.dup(stream) # keep copy of original stream try: @@ -345,19 +503,49 @@ def captured_fd(stream=2, encoding=None): return _output[0] os.dup2(temp_file.fileno(), stream) # replace stream by copy of pipe - try: - def get_output(): - result = read_output() - return result.decode(encoding) if encoding else result - - yield get_output - finally: - os.dup2(orig_stream, stream) # restore original stream - read_output() # keep the output in case it's used after closing the context manager + def get_output(): + result = read_output() + return result.decode(encoding) if encoding else result + + yield get_output + # note: @contextlib.contextmanager requires try-finally here + os.dup2(orig_stream, stream) # restore original stream + read_output() # keep the output in case it's used after closing the context manager finally: os.close(orig_stream) +def get_encoding_candidates(): + candidates = [sys.getdefaultencoding()] + for stream in (sys.stdout, sys.stdin, sys.__stdout__, sys.__stdin__): + encoding = getattr(stream, 'encoding', None) + # encoding might be None (e.g. somebody redirects stdout): + if encoding is not None and encoding not in candidates: + candidates.append(encoding) + return candidates + + +def prepare_captured(captured): + captured_bytes = captured.strip() + if not captured_bytes: + return None + for encoding in get_encoding_candidates(): + try: + return captured_bytes.decode(encoding) + except UnicodeDecodeError: + pass + # last resort: print at least the readable ascii parts correctly. + return captured_bytes.decode('latin-1') + + +def print_captured(captured, output, header_line=None): + captured = prepare_captured(captured) + if captured: + if header_line: + output.write(header_line) + output.write(captured) + + def print_bytes(s, header_text=None, end=b'\n', file=sys.stdout, flush=True): if header_text: file.write(header_text) # note: text! => file.write() instead of out.write() @@ -372,33 +560,29 @@ def print_bytes(s, header_text=None, end=b'\n', file=sys.stdout, flush=True): if flush: out.flush() -class LazyStr: - def __init__(self, callback): - self.callback = callback - def __str__(self): - return self.callback() - def __repr__(self): - return self.callback() - def __add__(self, right): - return self.callback() + right - def __radd__(self, left): - return left + self.callback() - class OrderedSet(object): - def __init__(self, elements=()): - self._list = [] - self._set = set() - self.update(elements) - def __iter__(self): - return iter(self._list) - def update(self, elements): - for e in elements: - self.add(e) - def add(self, e): - if e not in self._set: - self._list.append(e) - self._set.add(e) + def __init__(self, elements=()): + self._list = [] + self._set = set() + self.update(elements) + + def __iter__(self): + return iter(self._list) + + def update(self, elements): + for e in elements: + self.add(e) + + def add(self, e): + if e not in self._set: + self._list.append(e) + self._set.add(e) + + def __bool__(self): + return bool(self._set) + + __nonzero__ = __bool__ # Class decorator that adds a metaclass and recreates the class with it. @@ -420,24 +604,30 @@ def add_metaclass(metaclass): def raise_error_if_module_name_forbidden(full_module_name): - #it is bad idea to call the pyx-file cython.pyx, so fail early + # it is bad idea to call the pyx-file cython.pyx, so fail early if full_module_name == 'cython' or full_module_name.startswith('cython.'): raise ValueError('cython is a special module, cannot be used as a module name') def build_hex_version(version_string): """ - Parse and translate '4.3a1' into the readable hex representation '0x040300A1' (like PY_VERSION_HEX). + Parse and translate public version identifier like '4.3a1' into the readable hex representation '0x040300A1' (like PY_VERSION_HEX). + + SEE: https://peps.python.org/pep-0440/#public-version-identifiers """ - # First, parse '4.12a1' into [4, 12, 0, 0xA01]. + # Parse '4.12a1' into [4, 12, 0, 0xA01] + # And ignore .dev, .pre and .post segments digits = [] release_status = 0xF0 - for digit in re.split('([.abrc]+)', version_string): - if digit in ('a', 'b', 'rc'): - release_status = {'a': 0xA0, 'b': 0xB0, 'rc': 0xC0}[digit] + for segment in re.split(r'(\D+)', version_string): + if segment in ('a', 'b', 'rc'): + release_status = {'a': 0xA0, 'b': 0xB0, 'rc': 0xC0}[segment] digits = (digits + [0, 0])[:3] # 1.2a1 -> 1.2.0a1 - elif digit != '.': - digits.append(int(digit)) + elif segment in ('.dev', '.pre', '.post'): + break # break since those are the last segments + elif segment != '.': + digits.append(int(segment)) + digits = (digits + [0] * 3)[:4] digits[3] += release_status @@ -457,15 +647,14 @@ def write_depfile(target, source, dependencies): # paths below the base_dir are relative, otherwise absolute paths = [] for fname in dependencies: - fname = os.path.abspath(fname) if fname.startswith(src_base_dir): try: newpath = os.path.relpath(fname, cwd) except ValueError: # if they are on different Windows drives, absolute is fine - newpath = fname + newpath = os.path.abspath(fname) else: - newpath = fname + newpath = os.path.abspath(fname) paths.append(newpath) depline = os.path.relpath(target, cwd) + ": \\\n " |