Issue #14197: merge

1 files changed, 263 insertions, 179 deletions

diff --git a/Makefile.pre.in b/Makefile.pre.in
index e5c2a3f301..adffa03865 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -34,7 +34,7 @@ MAINCC=		@MAINCC@
 LINKCC=		@LINKCC@
 AR=		@AR@
 RANLIB=		@RANLIB@
-SVNVERSION=	@SVNVERSION@
+READELF=	@READELF@
 SOABI=		@SOABI@
 LDVERSION=	@LDVERSION@
 HGVERSION=	@HGVERSION@
@@ -59,6 +59,8 @@ INSTALL_DATA=	@INSTALL_DATA@
 # Also, making them read-only seems to be a good idea...
 INSTALL_SHARED= ${INSTALL} -m 555
 
+MKDIR_P=	@MKDIR_P@
+
 MAKESETUP=      $(srcdir)/Modules/makesetup
 
 # Compiler options
@@ -74,7 +76,7 @@ PY_CFLAGS=	$(BASECFLAGS) $(OPT) $(CONFIGURE_CFLAGS) $(CFLAGS) $(EXTRA_CFLAGS)
 # Both CPPFLAGS and LDFLAGS need to contain the shell's value for setup.py to
 # be able to build extension modules using the directories specified in the
 # environment variables
-PY_CPPFLAGS=	-I. -IInclude -I$(srcdir)/Include $(CONFIGURE_CPPFLAGS) $(CPPFLAGS)
+PY_CPPFLAGS=	-I. -I$(srcdir)/Include $(CONFIGURE_CPPFLAGS) $(CPPFLAGS)
 PY_LDFLAGS=	$(CONFIGURE_LDFLAGS) $(LDFLAGS)
 NO_AS_NEEDED=	@NO_AS_NEEDED@
 LDLAST=		@LDLAST@
@@ -192,6 +194,10 @@ LIBOBJS=	@LIBOBJS@
 PYTHON=		python$(EXE)
 BUILDPYTHON=	python$(BUILDEXE)
 
+PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@
+_PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@
+HOST_GNU_TYPE=  @host@
+
 # The task to run while instrument when building the profile-opt target
 PROFILE_TASK=	$(srcdir)/Tools/pybench/pybench.py -n 2 --with-gc --with-syscheck
 #PROFILE_TASK=	$(srcdir)/Lib/test/regrtest.py
@@ -234,20 +240,41 @@ LIBFFI_INCLUDEDIR=	@LIBFFI_INCLUDEDIR@
 # Parser
 PGEN=		Parser/pgen$(EXE)
 
+PSRCS=		\
+		Parser/acceler.c \
+		Parser/grammar1.c \
+		Parser/listnode.c \
+		Parser/node.c \
+		Parser/parser.c \
+		Parser/bitset.c \
+		Parser/metagrammar.c \
+		Parser/firstsets.c \
+		Parser/grammar.c \
+		Parser/pgen.c
+
 POBJS=		\
 		Parser/acceler.o \
 		Parser/grammar1.o \
 		Parser/listnode.o \
 		Parser/node.o \
 		Parser/parser.o \
-		Parser/parsetok.o \
 		Parser/bitset.o \
 		Parser/metagrammar.o \
 		Parser/firstsets.o \
 		Parser/grammar.o \
 		Parser/pgen.o
 
-PARSER_OBJS=	$(POBJS) Parser/myreadline.o Parser/tokenizer.o
+PARSER_OBJS=	$(POBJS) Parser/myreadline.o Parser/parsetok.o Parser/tokenizer.o
+
+PGSRCS=		\
+		Objects/obmalloc.c \
+		Python/dynamic_annotations.c \
+		Python/mysnprintf.c \
+		Python/pyctype.c \
+		Parser/tokenizer_pgen.c \
+		Parser/printgrammar.c \
+		Parser/parsetok_pgen.c \
+		Parser/pgenmain.c
 
 PGOBJS=		\
 		Objects/obmalloc.o \
@@ -256,13 +283,16 @@ PGOBJS=		\
 		Python/pyctype.o \
 		Parser/tokenizer_pgen.o \
 		Parser/printgrammar.o \
+		Parser/parsetok_pgen.o \
 		Parser/pgenmain.o
 
 PARSER_HEADERS= \
-		Parser/parser.h \
-		Parser/tokenizer.h
+		$(srcdir)/Parser/parser.h \
+		$(srcdir)/Include/parsetok.h \
+		$(srcdir)/Parser/tokenizer.h
 
-PGENOBJS=	$(PGENMAIN) $(POBJS) $(PGOBJS)
+PGENSRCS=	$(PSRCS) $(PGSRCS)
+PGENOBJS=	$(POBJS) $(PGOBJS)
 
 ##########################################################################
 # AST
@@ -299,7 +329,6 @@ PYTHON_OBJS=	\
 		Python/codecs.o \
 		Python/dynamic_annotations.o \
 		Python/errors.o \
-		Python/frozen.o \
 		Python/frozenmain.o \
 		Python/future.o \
 		Python/getargs.o \
@@ -367,11 +396,12 @@ OBJECT_OBJS=	\
 		Objects/memoryobject.o \
 		Objects/methodobject.o \
 		Objects/moduleobject.o \
+		Objects/namespaceobject.o \
 		Objects/object.o \
 		Objects/obmalloc.o \
 		Objects/capsule.o \
 		Objects/rangeobject.o \
-                Objects/setobject.o \
+		Objects/setobject.o \
 		Objects/sliceobject.o \
 		Objects/structseq.o \
 		Objects/tupleobject.o \
@@ -380,10 +410,11 @@ OBJECT_OBJS=	\
 		Objects/unicodectype.o \
 		Objects/weakrefobject.o
 
+SYSCONFIGDATA=$(srcdir)/Lib/_sysconfigdata.py
 
 ##########################################################################
 # objects that get linked into the Python library
-LIBRARY_OBJS=	\
+LIBRARY_OBJS_OMIT_FROZEN=	\
 		Modules/getbuildinfo.o \
 		$(PARSER_OBJS) \
 		$(OBJECT_OBJS) \
@@ -392,12 +423,16 @@ LIBRARY_OBJS=	\
 		$(SIGNAL_OBJS) \
 		$(MODOBJS)
 
+LIBRARY_OBJS=	\
+		$(LIBRARY_OBJS_OMIT_FROZEN) \
+		Python/frozen.o
+
 #########################################################################
 # Rules
 
 # Default target
 all:		build_all
-build_all:	$(BUILDPYTHON) oldsharedmods sharedmods gdbhooks Modules/_testembed
+build_all:	$(BUILDPYTHON) $(SYSCONFIGDATA) oldsharedmods sharedmods gdbhooks Modules/_testembed
 
 # Compile a binary with gcc profile guided optimization.
 profile-opt:
@@ -415,10 +450,11 @@ build_all_generate_profile:
 	$(MAKE) all CFLAGS="$(CFLAGS) -fprofile-generate" LIBS="$(LIBS) -lgcov"
 
 run_profile_task:
-	./$(BUILDPYTHON) $(PROFILE_TASK)
+	: # FIXME: can't run for a cross build
+	$(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK)
 
 build_all_use_profile:
-	$(MAKE) all CFLAGS="$(CFLAGS) -fprofile-use"
+	$(MAKE) all CFLAGS="$(CFLAGS) -fprofile-use -fprofile-correction"
 
 coverage:
 	@echo "Building with support for coverage checking:"
@@ -430,16 +466,18 @@ coverage:
 $(BUILDPYTHON):	Modules/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY)
 	$(LINKCC) $(PY_LDFLAGS) $(LINKFORSHARED) -o $@ Modules/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST)
 
-platform: $(BUILDPYTHON)
-	$(RUNSHARED) ./$(BUILDPYTHON) -E -c 'import sys ; from sysconfig import get_platform ; print(get_platform()+"-"+sys.version[0:3])' >platform
+platform: $(BUILDPYTHON) $(SYSCONFIGDATA)
+	$(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print(get_platform()+"-"+sys.version[0:3])' >platform
 
+# Generate the sysconfig build-time data
+$(SYSCONFIGDATA): $(BUILDPYTHON)
+	$(RUNSHARED) $(PYTHON_FOR_BUILD) -S -m sysconfig --generate-posix-vars
 
 # Build the shared modules
-sharedmods: $(BUILDPYTHON)
-	@case $$MAKEFLAGS in \
-	*s*) $(RUNSHARED) CC='$(CC)' LDSHARED='$(BLDSHARED)' OPT='$(OPT)' ./$(BUILDPYTHON) -E $(srcdir)/setup.py -q build;; \
-	*) $(RUNSHARED) CC='$(CC)' LDSHARED='$(BLDSHARED)' OPT='$(OPT)' ./$(BUILDPYTHON) -E $(srcdir)/setup.py build;; \
-	esac
+sharedmods: $(BUILDPYTHON) $(SYSCONFIGDATA)
+	case $$MAKEFLAGS in *s*) quiet=-q; esac; \
+	$(RUNSHARED) CC='$(CC)' LDSHARED='$(BLDSHARED)' OPT='$(OPT)' \
+		$(PYTHON_FOR_BUILD) $(srcdir)/setup.py $$quiet build
 
 # Build static library
 # avoid long command lines, same as LIBRARY_OBJS
@@ -448,7 +486,7 @@ $(LIBRARY): $(LIBRARY_OBJS)
 	$(AR) $(ARFLAGS) $@ Modules/getbuildinfo.o
 	$(AR) $(ARFLAGS) $@ $(PARSER_OBJS)
 	$(AR) $(ARFLAGS) $@ $(OBJECT_OBJS)
-	$(AR) $(ARFLAGS) $@ $(PYTHON_OBJS)
+	$(AR) $(ARFLAGS) $@ $(PYTHON_OBJS) Python/frozen.o
 	$(AR) $(ARFLAGS) $@ $(MODULE_OBJS) $(SIGNAL_OBJS)
 	$(AR) $(ARFLAGS) $@ $(MODOBJS)
 	$(RANLIB) $@
@@ -495,7 +533,7 @@ $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK): \
 		-install_name $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK) \
 		-compatibility_version $(VERSION) \
 		-current_version $(VERSION) \
-		-framework CoreFoundation $(LIBS); 
+		-framework CoreFoundation $(LIBS);
 	$(INSTALL) -d -m $(DIRMODE)  \
 		$(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/Resources/English.lproj
 	$(INSTALL_DATA) $(RESSRCDIR)/Info.plist \
@@ -546,6 +584,18 @@ Modules/_testembed: Modules/_testembed.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY)
 	$(LINKCC) $(PY_LDFLAGS) $(LINKFORSHARED) -o $@ Modules/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST)
 
 ############################################################################
+# Importlib
+
+Modules/_freeze_importlib: Modules/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN)
+	$(LINKCC) $(PY_LDFLAGS) -o $@ Modules/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST)
+
+Python/importlib.h: $(srcdir)/Lib/importlib/_bootstrap.py Modules/_freeze_importlib.c
+	$(MAKE) Modules/_freeze_importlib
+	./Modules/_freeze_importlib \
+		$(srcdir)/Lib/importlib/_bootstrap.py Python/importlib.h
+
+
+############################################################################
 # Special rules for object files
 
 Modules/getbuildinfo.o: $(PARSER_OBJS) \
@@ -556,7 +606,6 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \
 		$(MODOBJS) \
 		$(srcdir)/Modules/getbuildinfo.c
 	$(CC) -c $(PY_CORE_CFLAGS) \
-	      -DSVNVERSION="\"`LC_ALL=C $(SVNVERSION)`\"" \
 	      -DHGVERSION="\"`LC_ALL=C $(HGVERSION)`\"" \
 	      -DHGTAG="\"`LC_ALL=C $(HGTAG)`\"" \
 	      -DHGBRANCH="\"`LC_ALL=C $(HGBRANCH)`\"" \
@@ -573,6 +622,9 @@ Modules/getpath.o: $(srcdir)/Modules/getpath.c Makefile
 Modules/python.o: $(srcdir)/Modules/python.c
 	$(MAINCC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Modules/python.c
 
+Modules/_testembed.o: $(srcdir)/Modules/_testembed.c
+	$(MAINCC) -c $(PY_CORE_CFLAGS) -o $@ $(srcdir)/Modules/_testembed.c
+
 Python/dynload_shlib.o: $(srcdir)/Python/dynload_shlib.c Makefile
 	$(CC) -c $(PY_CORE_CFLAGS) \
 		-DSOABI='"$(SOABI)"' \
@@ -585,12 +637,13 @@ Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile
 
 $(IO_OBJS): $(IO_H)
 
-# Use a stamp file to prevent make -j invoking pgen twice
-$(GRAMMAR_H) $(GRAMMAR_C): Parser/pgen.stamp
-Parser/pgen.stamp: $(PGEN) $(GRAMMAR_INPUT)
-		-@$(INSTALL) -d Include
+$(GRAMMAR_H): $(GRAMMAR_INPUT) $(PGENSRCS)
+		@$(MKDIR_P) Include
+		$(MAKE) $(PGEN)
 		$(PGEN) $(GRAMMAR_INPUT) $(GRAMMAR_H) $(GRAMMAR_C)
-		-touch Parser/pgen.stamp
+$(GRAMMAR_C): $(GRAMMAR_H) $(GRAMMAR_INPUT) $(PGENSRCS)
+		$(MAKE) $(GRAMMAR_H)
+		touch $(GRAMMAR_C)
 
 $(PGEN):	$(PGENOBJS)
 		$(CC) $(OPT) $(PY_LDFLAGS) $(PGENOBJS) $(LIBS) -o $(PGEN)
@@ -601,13 +654,15 @@ Parser/grammar.o:	$(srcdir)/Parser/grammar.c \
 Parser/metagrammar.o:	$(srcdir)/Parser/metagrammar.c
 
 Parser/tokenizer_pgen.o:	$(srcdir)/Parser/tokenizer.c
+Parser/parsetok_pgen.o:	$(srcdir)/Parser/parsetok.c
+Parser/printgrammar.o: $(srcdir)/Parser/printgrammar.c
 
 Parser/pgenmain.o:	$(srcdir)/Include/parsetok.h
 
 $(AST_H): $(AST_ASDL) $(ASDLGEN_FILES)
 	$(ASDLGEN) -h $(AST_H_DIR) $(AST_ASDL)
 
-$(AST_C): $(AST_ASDL) $(ASDLGEN_FILES)
+$(AST_C): $(AST_H) $(AST_ASDL) $(ASDLGEN_FILES)
 	$(ASDLGEN) -c $(AST_C_DIR) $(AST_ASDL)
 
 Python/compile.o Python/symtable.o Python/ast.o: $(GRAMMAR_H) $(AST_H)
@@ -628,20 +683,29 @@ BYTESTR_DEPS = \
 		$(srcdir)/Objects/stringlib/eq.h \
 		$(srcdir)/Objects/stringlib/fastsearch.h \
 		$(srcdir)/Objects/stringlib/find.h \
+		$(srcdir)/Objects/stringlib/find_max_char.h \
 		$(srcdir)/Objects/stringlib/partition.h \
 		$(srcdir)/Objects/stringlib/split.h \
 		$(srcdir)/Objects/stringlib/stringdefs.h \
-		$(srcdir)/Objects/stringlib/string_format.h \
 		$(srcdir)/Objects/stringlib/transmogrify.h \
 		$(srcdir)/Objects/stringlib/unicodedefs.h \
-		$(srcdir)/Objects/stringlib/localeutil.h
+		$(srcdir)/Objects/stringlib/localeutil.h \
+		$(srcdir)/Objects/stringlib/undef.h
+
+UNICODE_DEPS = $(BYTESTR_DEPS) \
+		$(srcdir)/Objects/stringlib/asciilib.h \
+		$(srcdir)/Objects/stringlib/codecs.h \
+		$(srcdir)/Objects/stringlib/ucs1lib.h \
+		$(srcdir)/Objects/stringlib/ucs2lib.h \
+		$(srcdir)/Objects/stringlib/ucs4lib.h \
+		$(srcdir)/Objects/stringlib/unicode_format.h \
+		$(srcdir)/Objects/stringlib/unicodedefs.h
 
 Objects/bytesobject.o: $(srcdir)/Objects/bytesobject.c $(BYTESTR_DEPS)
 
 Objects/bytearrayobject.o: $(srcdir)/Objects/bytearrayobject.c $(BYTESTR_DEPS)
 
-Objects/unicodeobject.o: $(srcdir)/Objects/unicodeobject.c \
-				$(BYTESTR_DEPS)
+Objects/unicodeobject.o: $(srcdir)/Objects/unicodeobject.c $(UNICODE_DEPS)
 
 Objects/dictobject.o: $(srcdir)/Objects/stringlib/eq.h
 Objects/setobject.o: $(srcdir)/Objects/stringlib/eq.h
@@ -649,11 +713,12 @@ Objects/setobject.o: $(srcdir)/Objects/stringlib/eq.h
 $(OPCODETARGETS_H): $(OPCODETARGETGEN_FILES)
 	$(OPCODETARGETGEN) $(OPCODETARGETS_H)
 
-Python/ceval.o: $(OPCODETARGETS_H) Python/ceval_gil.h
+Python/ceval.o: $(OPCODETARGETS_H) $(srcdir)/Python/ceval_gil.h
 
 Python/formatter_unicode.o: $(srcdir)/Python/formatter_unicode.c \
-				$(BYTESTR_DEPS) \
-				$(srcdir)/Objects/stringlib/formatter.h
+				$(BYTESTR_DEPS)
+
+Python/frozen.o: $(srcdir)/Python/importlib.h
 
 Objects/typeobject.o: $(srcdir)/Objects/typeslots.inc
 $(srcdir)/Objects/typeslots.inc: $(srcdir)/Include/typeslots.h $(srcdir)/Objects/typeslots.py
@@ -663,89 +728,89 @@ $(srcdir)/Objects/typeslots.inc: $(srcdir)/Include/typeslots.h $(srcdir)/Objects
 # Header files
 
 PYTHON_HEADERS= \
-		Include/Python-ast.h \
-		Include/Python.h \
-		Include/abstract.h \
-		Include/accu.h \
-		Include/asdl.h \
-		Include/ast.h \
-                Include/bltinmodule.h \
-		Include/bitset.h \
-		Include/boolobject.h \
-		Include/bytes_methods.h \
-		Include/bytearrayobject.h \
-		Include/bytesobject.h \
-		Include/cellobject.h \
-		Include/ceval.h \
-		Include/classobject.h \
-		Include/code.h \
-		Include/codecs.h \
-		Include/compile.h \
-		Include/complexobject.h \
-		Include/descrobject.h \
-		Include/dictobject.h \
-		Include/dtoa.h \
-		Include/dynamic_annotations.h \
-		Include/enumobject.h \
-		Include/errcode.h \
-		Include/eval.h \
-		Include/fileobject.h \
-		Include/fileutils.h \
-		Include/floatobject.h \
-		Include/frameobject.h \
-		Include/funcobject.h \
-		Include/genobject.h \
-		Include/import.h \
-		Include/intrcheck.h \
-		Include/iterobject.h \
-		Include/listobject.h \
-		Include/longintrepr.h \
-		Include/longobject.h \
-		Include/marshal.h \
-		Include/memoryobject.h \
-		Include/metagrammar.h \
-		Include/methodobject.h \
-		Include/modsupport.h \
-		Include/moduleobject.h \
-		Include/node.h \
-		Include/object.h \
-		Include/objimpl.h \
-		Include/opcode.h \
-		Include/osdefs.h \
-		Include/parsetok.h \
-		Include/patchlevel.h \
-		Include/pgen.h \
-		Include/pgenheaders.h \
-		Include/pyarena.h \
-		Include/pyatomic.h \
-		Include/pycapsule.h \
-		Include/pyctype.h \
-		Include/pydebug.h \
-		Include/pyerrors.h \
-		Include/pyfpe.h \
-		Include/pymath.h \
-		Include/pygetopt.h \
-		Include/pymem.h \
-		Include/pyport.h \
-		Include/pystate.h \
-		Include/pystrcmp.h \
-		Include/pystrtod.h \
-		Include/pythonrun.h \
-		Include/pythread.h \
-		Include/pytime.h \
-		Include/rangeobject.h \
-		Include/setobject.h \
-		Include/sliceobject.h \
-		Include/structmember.h \
-		Include/structseq.h \
-		Include/symtable.h \
-		Include/sysmodule.h \
-		Include/traceback.h \
-		Include/tupleobject.h \
-		Include/ucnhash.h \
-		Include/unicodeobject.h \
-		Include/warnings.h \
-		Include/weakrefobject.h \
+		$(srcdir)/Include/Python.h \
+		$(srcdir)/Include/abstract.h \
+		$(srcdir)/Include/accu.h \
+		$(srcdir)/Include/asdl.h \
+		$(srcdir)/Include/ast.h \
+		$(srcdir)/Include/bltinmodule.h \
+		$(srcdir)/Include/bitset.h \
+		$(srcdir)/Include/boolobject.h \
+		$(srcdir)/Include/bytes_methods.h \
+		$(srcdir)/Include/bytearrayobject.h \
+		$(srcdir)/Include/bytesobject.h \
+		$(srcdir)/Include/cellobject.h \
+		$(srcdir)/Include/ceval.h \
+		$(srcdir)/Include/classobject.h \
+		$(srcdir)/Include/code.h \
+		$(srcdir)/Include/codecs.h \
+		$(srcdir)/Include/compile.h \
+		$(srcdir)/Include/complexobject.h \
+		$(srcdir)/Include/descrobject.h \
+		$(srcdir)/Include/dictobject.h \
+		$(srcdir)/Include/dtoa.h \
+		$(srcdir)/Include/dynamic_annotations.h \
+		$(srcdir)/Include/enumobject.h \
+		$(srcdir)/Include/errcode.h \
+		$(srcdir)/Include/eval.h \
+		$(srcdir)/Include/fileobject.h \
+		$(srcdir)/Include/fileutils.h \
+		$(srcdir)/Include/floatobject.h \
+		$(srcdir)/Include/frameobject.h \
+		$(srcdir)/Include/funcobject.h \
+		$(srcdir)/Include/genobject.h \
+		$(srcdir)/Include/import.h \
+		$(srcdir)/Include/intrcheck.h \
+		$(srcdir)/Include/iterobject.h \
+		$(srcdir)/Include/listobject.h \
+		$(srcdir)/Include/longintrepr.h \
+		$(srcdir)/Include/longobject.h \
+		$(srcdir)/Include/marshal.h \
+		$(srcdir)/Include/memoryobject.h \
+		$(srcdir)/Include/metagrammar.h \
+		$(srcdir)/Include/methodobject.h \
+		$(srcdir)/Include/modsupport.h \
+		$(srcdir)/Include/moduleobject.h \
+		$(srcdir)/Include/namespaceobject.h \
+		$(srcdir)/Include/node.h \
+		$(srcdir)/Include/object.h \
+		$(srcdir)/Include/objimpl.h \
+		$(srcdir)/Include/opcode.h \
+		$(srcdir)/Include/osdefs.h \
+		$(srcdir)/Include/patchlevel.h \
+		$(srcdir)/Include/pgen.h \
+		$(srcdir)/Include/pgenheaders.h \
+		$(srcdir)/Include/pyarena.h \
+		$(srcdir)/Include/pyatomic.h \
+		$(srcdir)/Include/pycapsule.h \
+		$(srcdir)/Include/pyctype.h \
+		$(srcdir)/Include/pydebug.h \
+		$(srcdir)/Include/pyerrors.h \
+		$(srcdir)/Include/pyfpe.h \
+		$(srcdir)/Include/pymath.h \
+		$(srcdir)/Include/pygetopt.h \
+		$(srcdir)/Include/pymacro.h \
+		$(srcdir)/Include/pymem.h \
+		$(srcdir)/Include/pyport.h \
+		$(srcdir)/Include/pystate.h \
+		$(srcdir)/Include/pystrcmp.h \
+		$(srcdir)/Include/pystrtod.h \
+		$(srcdir)/Include/pythonrun.h \
+		$(srcdir)/Include/pythread.h \
+		$(srcdir)/Include/pytime.h \
+		$(srcdir)/Include/rangeobject.h \
+		$(srcdir)/Include/setobject.h \
+		$(srcdir)/Include/sliceobject.h \
+		$(srcdir)/Include/structmember.h \
+		$(srcdir)/Include/structseq.h \
+		$(srcdir)/Include/symtable.h \
+		$(srcdir)/Include/sysmodule.h \
+		$(srcdir)/Include/traceback.h \
+		$(srcdir)/Include/tupleobject.h \
+		$(srcdir)/Include/ucnhash.h \
+		$(srcdir)/Include/unicodeobject.h \
+		$(srcdir)/Include/warnings.h \
+		$(srcdir)/Include/weakrefobject.h \
 		pyconfig.h \
 		$(PARSER_HEADERS)
 
@@ -754,65 +819,56 @@ $(LIBRARY_OBJS) $(MODOBJS) Modules/python.o: $(PYTHON_HEADERS)
 
 ######################################################################
 
-# Test the interpreter (twice, once without .pyc files, once with)
+TESTOPTS=	$(EXTRATESTOPTS)
+TESTPYTHON=	$(RUNSHARED) ./$(BUILDPYTHON) $(TESTPYTHONOPTS)
+TESTRUNNER=	$(TESTPYTHON) $(srcdir)/Tools/scripts/run_tests.py
+TESTTIMEOUT=	3600
+
+# Run a basic set of regression tests.
+# This excludes some tests that are particularly resource-intensive.
+test:		all platform
+		$(TESTRUNNER) $(TESTOPTS)
+
+# Run the full test suite twice - once without .pyc files, and once with.
 # In the past, we've had problems where bugs in the marshalling or
 # elsewhere caused bytecode read from .pyc files to behave differently
 # than bytecode generated directly from a .py source file.  Sometimes
-# the bytecode read from a .pyc file had the bug, somtimes the directly
+# the bytecode read from a .pyc file had the bug, sometimes the directly
 # generated bytecode.  This is sometimes a very shy bug needing a lot of
 # sample data.
-
-TESTOPTS=	-l $(EXTRATESTOPTS)
-TESTPROG=	$(srcdir)/Lib/test/regrtest.py
-TESTPYTHON=	$(RUNSHARED) ./$(BUILDPYTHON) -Wd -E -R -bb $(TESTPYTHONOPTS)
-test:		all platform
-		-find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f
-		-$(TESTPYTHON) $(TESTPROG) $(TESTOPTS)
-		$(TESTPYTHON) $(TESTPROG) $(TESTOPTS)
-
 testall:	all platform
 		-find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f
-		$(TESTPYTHON) $(srcdir)/Lib/compileall.py
+		$(TESTPYTHON) -E $(srcdir)/Lib/compileall.py
 		-find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f
-		-$(TESTPYTHON) $(TESTPROG) -uall $(TESTOPTS)
-		$(TESTPYTHON) $(TESTPROG) -uall $(TESTOPTS)
+		-$(TESTRUNNER) -u all $(TESTOPTS)
+		$(TESTRUNNER) -u all $(TESTOPTS)
 
-#  Run the unitests for both architectures in a Universal build on OSX
-#  Must be run on an Intel box.
+# Run the test suite for both architectures in a Universal build on OSX.
+# Must be run on an Intel box.
 testuniversal:	all platform
 		if [ `arch` != 'i386' ];then \
 			echo "This can only be used on OSX/i386" ;\
 			exit 1 ;\
 		fi
-		-find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f
-		-$(TESTPYTHON) $(TESTPROG) -uall $(TESTOPTS)
-		$(TESTPYTHON) $(TESTPROG) -uall $(TESTOPTS)
-		$(RUNSHARED) /usr/libexec/oah/translate ./$(BUILDPYTHON) -E $(TESTPROG) -uall $(TESTOPTS)
+		$(TESTRUNNER) -u all $(TESTOPTS)
+		$(RUNSHARED) /usr/libexec/oah/translate \
+			./$(BUILDPYTHON) -E -m test -j 0 -u all $(TESTOPTS)
 
-
-# Like testall, but with a single pass only
-# run an optional script to include some information about the build environment
+# Like testall, but with only one pass and without multiple processes.
+# Run an optional script to include information about the build environment.
 buildbottest:	all platform
 		-@if which pybuildbot.identify >/dev/null 2>&1; then \
 			pybuildbot.identify "CC='$(CC)'" "CXX='$(CXX)'"; \
 		fi
-		$(TESTPYTHON) $(TESTPROG) -uall -rwW $(TESTOPTS)
+		$(TESTRUNNER) -j 1 -u all -W --timeout=$(TESTTIMEOUT) $(TESTOPTS)
 
 QUICKTESTOPTS=	$(TESTOPTS) -x test_subprocess test_io test_lib2to3 \
 		test_multibytecodec test_urllib2_localnet test_itertools \
 		test_multiprocessing test_mailbox test_socket test_poll \
-		test_select test_zipfile
+		test_select test_zipfile test_concurrent_futures
 quicktest:	all platform
-		-find $(srcdir)/Lib -name '*.py[co]' -print | xargs rm -f
-		-$(TESTPYTHON) $(TESTPROG) $(QUICKTESTOPTS)
-		$(TESTPYTHON) $(TESTPROG) $(QUICKTESTOPTS)
+		$(TESTRUNNER) $(QUICKTESTOPTS)
 
-MEMTESTOPTS=    $(QUICKTESTOPTS) -x test_dl test___all__ test_fork1 \
-		test_longexp
-memtest:	all platform
-		-rm -f $(srcdir)/Lib/test/*.py[co]
-		-$(TESTPYTHON) $(TESTPROG) $(MEMTESTOPTS)
-		$(TESTPYTHON) $(TESTPROG) $(MEMTESTOPTS)
 
 install: altinstall bininstall
 
@@ -853,7 +909,7 @@ altbininstall: $(BUILDPYTHON)
 	done
 	$(INSTALL_PROGRAM) $(BUILDPYTHON) $(DESTDIR)$(BINDIR)/python$(LDVERSION)$(EXE)
 	-if test "$(VERSION)" != "$(LDVERSION)"; then \
-		if test -f $(DESTDIR)$(BINDIR)/$(PYTHON)$(VERSION)$(EXE) -o -h $(DESTDIR)$(BINDIR)/$(PYTHON)$(VERSION)$(EXE); \
+		if test -f $(DESTDIR)$(BINDIR)/python$(VERSION)$(EXE) -o -h $(DESTDIR)$(BINDIR)/python$(VERSION)$(EXE); \
 		then rm -f $(DESTDIR)$(BINDIR)/python$(VERSION)$(EXE); \
 		fi; \
 		(cd $(DESTDIR)$(BINDIR); $(LN) python$(LDVERSION)$(EXE) python$(VERSION)$(EXE)); \
@@ -874,11 +930,11 @@ altbininstall: $(BUILDPYTHON)
 	fi
 
 bininstall: altbininstall
-	-if test -f $(DESTDIR)$(BINDIR)/$(PYTHON)3$(EXE) -o -h $(DESTDIR)$(BINDIR)/$(PYTHON)3$(EXE); \
-	then rm -f $(DESTDIR)$(BINDIR)/$(PYTHON)3$(EXE); \
+	-if test -f $(DESTDIR)$(BINDIR)/python3$(EXE) -o -h $(DESTDIR)$(BINDIR)/python3$(EXE); \
+	then rm -f $(DESTDIR)$(BINDIR)/python3$(EXE); \
 	else true; \
 	fi
-	(cd $(DESTDIR)$(BINDIR); $(LN) python$(VERSION)$(EXE) $(PYTHON)3$(EXE))
+	(cd $(DESTDIR)$(BINDIR); $(LN) -s python$(VERSION)$(EXE) python3$(EXE))
 	-if test "$(VERSION)" != "$(LDVERSION)"; then \
 		rm -f $(DESTDIR)$(BINDIR)/python$(VERSION)-config; \
 		(cd $(DESTDIR)$(BINDIR); $(LN) -s python$(LDVERSION)-config python$(VERSION)-config); \
@@ -895,6 +951,8 @@ bininstall: altbininstall
 	(cd $(DESTDIR)$(BINDIR); $(LN) -s pydoc$(VERSION) pydoc3)
 	-rm -f $(DESTDIR)$(BINDIR)/2to3
 	(cd $(DESTDIR)$(BINDIR); $(LN) -s 2to3-$(VERSION) 2to3)
+	-rm -f $(DESTDIR)$(BINDIR)/pyvenv
+	(cd $(DESTDIR)$(BINDIR); $(LN) -s pyvenv-$(VERSION) pyvenv)
 
 # Install the manual page
 maninstall:
@@ -920,21 +978,43 @@ LIBSUBDIRS=	tkinter tkinter/test tkinter/test/test_tkinter \
 		test/cjkencodings test/decimaltestdata test/xmltestdata \
 		test/subprocessdata test/sndhdrdata \
 		test/tracedmodules test/encoded_modules \
-		concurrent concurrent/futures encodings \
-		email email/mime email/test email/test/data \
+		test/namespace_pkgs \
+		test/namespace_pkgs/both_portions \
+		test/namespace_pkgs/both_portions/foo \
+		test/namespace_pkgs/not_a_namespace_pkg \
+		test/namespace_pkgs/not_a_namespace_pkg/foo \
+		test/namespace_pkgs/portion1 \
+		test/namespace_pkgs/portion1/foo \
+		test/namespace_pkgs/portion2 \
+		test/namespace_pkgs/portion2/foo \
+		test/namespace_pkgs/project1 \
+		test/namespace_pkgs/project1/parent \
+		test/namespace_pkgs/project1/parent/child \
+		test/namespace_pkgs/project2 \
+		test/namespace_pkgs/project2/parent \
+		test/namespace_pkgs/project2/parent/child \
+		test/namespace_pkgs/project3 \
+		test/namespace_pkgs/project3/parent \
+		test/namespace_pkgs/project3/parent/child \
+                test/namespace_pkgs/module_and_namespace_package \
+                test/namespace_pkgs/module_and_namespace_package/a_test \
+		collections concurrent concurrent/futures encodings \
+		email email/mime test/test_email test/test_email/data \
 		html json test/json_tests http dbm xmlrpc \
 		sqlite3 sqlite3/test \
 		logging csv wsgiref urllib \
 		lib2to3 lib2to3/fixes lib2to3/pgen2 lib2to3/tests \
-		lib2to3/tests/data lib2to3/tests/data/fixers lib2to3/tests/data/fixers/myfixes \
+		lib2to3/tests/data lib2to3/tests/data/fixers \
+		lib2to3/tests/data/fixers/myfixes \
 		ctypes ctypes/test ctypes/macholib idlelib idlelib/Icons \
 		distutils distutils/command distutils/tests $(XMLLIBSUBDIRS) \
-		importlib importlib/test importlib/test/builtin \
-		importlib/test/extension importlib/test/frozen \
-		importlib/test/import_ importlib/test/source \
+		importlib test/test_importlib test/test_importlib/builtin \
+		test/test_importlib/extension test/test_importlib/frozen \
+		test/test_importlib/import_ test/test_importlib/source \
 		turtledemo \
 		multiprocessing multiprocessing/dummy \
-		unittest unittest/test \
+		unittest unittest/test unittest/test/testmock \
+		venv venv/scripts venv/scripts/posix \
 		curses pydoc_data $(MACHDEPS)
 libinstall:	build_all $(srcdir)/Lib/$(PLATDIR) $(srcdir)/Modules/xxmodule.c
 	@for i in $(SCRIPTDIR) $(LIBDEST); \
@@ -956,7 +1036,7 @@ libinstall:	build_all $(srcdir)/Lib/$(PLATDIR) $(srcdir)/Modules/xxmodule.c
 		else	true; \
 		fi; \
 	done
-	@for i in $(srcdir)/Lib/*.py $(srcdir)/Lib/*.egg-info ; \
+	@for i in $(srcdir)/Lib/*.py ; \
 	do \
 		if test -x $$i; then \
 			$(INSTALL_SCRIPT) $$i $(DESTDIR)$(LIBDEST); \
@@ -997,25 +1077,25 @@ libinstall:	build_all $(srcdir)/Lib/$(PLATDIR) $(srcdir)/Modules/xxmodule.c
 			$(DESTDIR)$(LIBDEST)/distutils/tests ; \
 	fi
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST)  $(RUNSHARED) \
-		./$(BUILDPYTHON) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \
+		$(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \
 		-d $(LIBDEST) -f \
 		-x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \
 		$(DESTDIR)$(LIBDEST)
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
-		./$(BUILDPYTHON) -Wi -O $(DESTDIR)$(LIBDEST)/compileall.py \
+		$(PYTHON_FOR_BUILD) -Wi -O $(DESTDIR)$(LIBDEST)/compileall.py \
 		-d $(LIBDEST) -f \
 		-x 'bad_coding|badsyntax|site-packages|lib2to3/tests/data' \
 		$(DESTDIR)$(LIBDEST)
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST)  $(RUNSHARED) \
-		./$(BUILDPYTHON) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \
+		$(PYTHON_FOR_BUILD) -Wi $(DESTDIR)$(LIBDEST)/compileall.py \
 		-d $(LIBDEST)/site-packages -f \
 		-x badsyntax $(DESTDIR)$(LIBDEST)/site-packages
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
-		./$(BUILDPYTHON) -Wi -O $(DESTDIR)$(LIBDEST)/compileall.py \
+		$(PYTHON_FOR_BUILD) -Wi -O $(DESTDIR)$(LIBDEST)/compileall.py \
 		-d $(LIBDEST)/site-packages -f \
 		-x badsyntax $(DESTDIR)$(LIBDEST)/site-packages
 	-PYTHONPATH=$(DESTDIR)$(LIBDEST) $(RUNSHARED) \
-		./$(BUILDPYTHON) -Wi -c "import lib2to3.pygram, lib2to3.patcomp;lib2to3.patcomp.PatternCompiler()"
+		$(PYTHON_FOR_BUILD) -Wi -c "import lib2to3.pygram, lib2to3.patcomp;lib2to3.patcomp.PatternCompiler()"
 
 # Create the PLATDIR source directory, if one wasn't distributed..
 $(srcdir)/Lib/$(PLATDIR):
@@ -1109,7 +1189,7 @@ libainstall:	all python-config
 # Install the dynamically loadable modules
 # This goes into $(exec_prefix)
 sharedinstall: sharedmods
-	$(RUNSHARED) ./$(BUILDPYTHON) -E $(srcdir)/setup.py install \
+	$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/setup.py install \
 	   	--prefix=$(prefix) \
 		--install-scripts=$(BINDIR) \
 		--install-platlib=$(DESTSHARED) \
@@ -1184,13 +1264,13 @@ frameworkinstallextras:
 # This installs a few of the useful scripts in Tools/scripts
 scriptsinstall:
 	SRCDIR=$(srcdir) $(RUNSHARED) \
-	./$(BUILDPYTHON) $(srcdir)/Tools/scripts/setup.py install \
+	$(PYTHON_FOR_BUILD) $(srcdir)/Tools/scripts/setup.py install \
 	--prefix=$(prefix) \
 	--install-scripts=$(BINDIR) \
 	--root=$(DESTDIR)/
 
 # Build the toplevel Makefile
-Makefile.pre: Makefile.pre.in config.status
+Makefile.pre: $(srcdir)/Makefile.pre.in config.status
 	CONFIG_FILES=Makefile.pre CONFIG_HEADERS= $(SHELL) config.status
 	$(MAKE) -f Makefile.pre Makefile
 
@@ -1233,6 +1313,10 @@ TAGS::
 	etags Include/*.h; \
 	for i in $(SRCDIRS); do etags -a $$i/*.[ch]; done
 
+# Touch generated files
+touch:
+	hg --config extensions.touch=Tools/hg/hgtouch.py touch -v
+
 # Sanitation targets -- clean leaves libraries, executables and tags
 # files, which clobber removes as well
 pycremoval:
@@ -1257,14 +1341,15 @@ clean: pycremoval
 	find build -name 'fficonfig.h' -exec rm -f {} ';' || true
 	find build -name 'fficonfig.py' -exec rm -f {} ';' || true
 	-rm -f Lib/lib2to3/*Grammar*.pickle
-	-rm -f Modules/_testembed
+	-rm -f $(SYSCONFIGDATA)
+	-rm -f Modules/_testembed Modules/_freeze_importlib
 
 profile-removal:
 	find . -name '*.gc??' -exec rm -f {} ';'
 
 clobber: clean profile-removal
 	-rm -f $(BUILDPYTHON) $(PGEN) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY) \
-		tags TAGS Parser/pgen.stamp \
+		tags TAGS \
 		config.cache config.log pyconfig.h Modules/config.c
 	-rm -rf build platform
 	-rm -rf $(PYTHONFRAMEWORKDIR)
@@ -1295,8 +1380,7 @@ smelly: all
 # Find files with funny names
 funny:
 	find $(SUBDIRS) $(SUBDIRSTOO) \
-		-name .svn -prune \
-		-o -type d \
+		-type d \
 		-o -name '*.[chs]' \
 		-o -name '*.py' \
 		-o -name '*.pyw' \
@@ -1335,13 +1419,13 @@ patchcheck:
 Python/thread.o: @THREADHEADERS@
 
 # Declare targets that aren't real files
-.PHONY: all build_all sharedmods oldsharedmods test quicktest memtest
+.PHONY: all build_all sharedmods oldsharedmods test quicktest
 .PHONY: install altinstall oldsharedinstall bininstall altbininstall
 .PHONY: maninstall libinstall inclinstall libainstall sharedinstall
 .PHONY: frameworkinstall frameworkinstallframework frameworkinstallstructure
 .PHONY: frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools
 .PHONY: frameworkaltinstallunixtools recheck autoconf clean clobber distclean
-.PHONY: smelly funny patchcheck
+.PHONY: smelly funny patchcheck touch
 .PHONY: gdbhooks
 
 # IF YOU PUT ANYTHING HERE IT WILL GO AWAY