From 004869681a2b618d5b64fd42b8f7812f65c7dc88 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 15 Nov 2012 21:39:36 -0500 Subject: Issue #15894: Document why we don't worry about re-acquiring the global import lock after forking. --- Python/import.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 2f71b97bc8..6882b57060 100644 --- a/Python/import.c +++ b/Python/import.c @@ -202,8 +202,11 @@ _PyImport_ReInitLock(void) if (import_lock_level > 1) { /* Forked as a side effect of import */ long me = PyThread_get_thread_ident(); - PyThread_acquire_lock(import_lock, 0); - /* XXX: can the previous line fail? */ + /* The following could fail if the lock is already held, but forking as + a side-effect of an import is a) rare, b) nuts, and c) difficult to + do thanks to the lock only being held when doing individual module + locks per import. */ + PyThread_acquire_lock(import_lock, NOWAIT_LOCK); import_lock_thread = me; import_lock_level--; } else { -- cgit v1.2.1 From 838169869f9d3474c58ed662b1a51541de235c34 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 14 Dec 2012 17:04:59 +0200 Subject: Issue #16421: allow to load multiple modules from the same shared object. Patch by V?clav ?milauer. --- Python/import.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 6882b57060..8e7482ac4e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -452,12 +452,12 @@ PyImport_GetMagicTag(void) /* Magic for extension modules (built-in as well as dynamically loaded). To prevent initializing an extension module more than - once, we keep a static dictionary 'extensions' keyed by module name - (for built-in modules) or by filename (for dynamically loaded - modules), containing these modules. A copy of the module's - dictionary is stored by calling _PyImport_FixupExtensionObject() - immediately after the module initialization function succeeds. A - copy can be retrieved from there by calling + once, we keep a static dictionary 'extensions' keyed by the tuple + (module name, module name) (for built-in modules) or by + (filename, module name) (for dynamically loaded modules), containing these + modules. A copy of the module's dictionary is stored by calling + _PyImport_FixupExtensionObject() immediately after the module initialization + function succeeds. A copy can be retrieved from there by calling _PyImport_FindExtensionObject(). Modules which do support multiple initialization set their m_size @@ -470,7 +470,7 @@ int _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, PyObject *filename) { - PyObject *modules, *dict; + PyObject *modules, *dict, *filename_name; struct PyModuleDef *def; if (extensions == NULL) { extensions = PyDict_New(); @@ -508,7 +508,11 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, if (def->m_base.m_copy == NULL) return -1; } - PyDict_SetItem(extensions, filename, (PyObject*)def); + filename_name = PyTuple_Pack(2,filename, name); + if (filename_name == NULL) + return -1; + if (PyDict_SetItem(extensions, filename_name, (PyObject*)def) < 0) + return -1; return 0; } @@ -528,11 +532,14 @@ _PyImport_FixupBuiltin(PyObject *mod, char *name) PyObject * _PyImport_FindExtensionObject(PyObject *name, PyObject *filename) { - PyObject *mod, *mdict; + PyObject *mod, *mdict, *filename_name; PyModuleDef* def; if (extensions == NULL) return NULL; - def = (PyModuleDef*)PyDict_GetItem(extensions, filename); + filename_name = PyTuple_Pack(2,filename, name); + if (filename_name == NULL) + return NULL; + def = (PyModuleDef*)PyDict_GetItem(extensions, filename_name); if (def == NULL) return NULL; if (def->m_size == -1) { -- cgit v1.2.1 From a0d1572bb8860a41834b2bca05748df2a28ffb5d Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 15 Dec 2012 00:05:16 -0500 Subject: cleanup and fix refleaks --- Python/import.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 8e7482ac4e..50267895c5 100644 --- a/Python/import.c +++ b/Python/import.c @@ -470,8 +470,9 @@ int _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, PyObject *filename) { - PyObject *modules, *dict, *filename_name; + PyObject *modules, *dict, *key; struct PyModuleDef *def; + int res; if (extensions == NULL) { extensions = PyDict_New(); if (extensions == NULL) @@ -508,10 +509,12 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, if (def->m_base.m_copy == NULL) return -1; } - filename_name = PyTuple_Pack(2,filename, name); - if (filename_name == NULL) + key = PyTuple_Pack(2, filename, name); + if (key == NULL) return -1; - if (PyDict_SetItem(extensions, filename_name, (PyObject*)def) < 0) + res = PyDict_SetItem(extensions, key, (PyObject *)def); + Py_DECREF(key); + if (res < 0) return -1; return 0; } @@ -532,14 +535,15 @@ _PyImport_FixupBuiltin(PyObject *mod, char *name) PyObject * _PyImport_FindExtensionObject(PyObject *name, PyObject *filename) { - PyObject *mod, *mdict, *filename_name; + PyObject *mod, *mdict, *key; PyModuleDef* def; if (extensions == NULL) return NULL; - filename_name = PyTuple_Pack(2,filename, name); - if (filename_name == NULL) + key = PyTuple_Pack(2, filename, name); + if (key == NULL) return NULL; - def = (PyModuleDef*)PyDict_GetItem(extensions, filename_name); + def = (PyModuleDef *)PyDict_GetItem(extensions, key); + Py_DECREF(key); if (def == NULL) return NULL; if (def->m_size == -1) { -- cgit v1.2.1 From 63c6c6e958c8f9749e69ad62e653147ff08680c1 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Mon, 18 Mar 2013 23:13:31 -0700 Subject: sprinkle const --- Python/import.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index a50ef05b07..8773be46ad 100644 --- a/Python/import.c +++ b/Python/import.c @@ -846,7 +846,7 @@ imp_fix_co_filename(PyObject *self, PyObject *args) /* Forward */ -static struct _frozen * find_frozen(PyObject *); +static const struct _frozen * find_frozen(PyObject *); /* Helper to test for built-in module */ @@ -983,10 +983,10 @@ init_builtin(PyObject *name) /* Frozen modules */ -static struct _frozen * +static const struct _frozen * find_frozen(PyObject *name) { - struct _frozen *p; + const struct _frozen *p; if (name == NULL) return NULL; @@ -1003,7 +1003,7 @@ find_frozen(PyObject *name) static PyObject * get_frozen_object(PyObject *name) { - struct _frozen *p = find_frozen(name); + const struct _frozen *p = find_frozen(name); int size; if (p == NULL) { @@ -1027,7 +1027,7 @@ get_frozen_object(PyObject *name) static PyObject * is_frozen_package(PyObject *name) { - struct _frozen *p = find_frozen(name); + const struct _frozen *p = find_frozen(name); int size; if (p == NULL) { @@ -1054,7 +1054,7 @@ is_frozen_package(PyObject *name) int PyImport_ImportFrozenModuleObject(PyObject *name) { - struct _frozen *p; + const struct _frozen *p; PyObject *co, *m, *path; int ispackage; int size; @@ -1781,7 +1781,7 @@ static PyObject * imp_is_frozen(PyObject *self, PyObject *args) { PyObject *name; - struct _frozen *p; + const struct _frozen *p; if (!PyArg_ParseTuple(args, "U:is_frozen", &name)) return NULL; p = find_frozen(name); -- cgit v1.2.1 From 05af1e7fca1463b6a0fc7a790313d93a3e68c222 Mon Sep 17 00:00:00 2001 From: Alexandre Vassalotti Date: Thu, 2 May 2013 10:44:04 -0700 Subject: Closes #17892: Fix the name of _PyObject_CallMethodObjIdArgs --- Python/import.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 805cdd7e2a..b77eda106d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -704,7 +704,7 @@ PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, "no interpreter!"); } - pathobj = _PyObject_CallMethodObjIdArgs(interp->importlib, + pathobj = _PyObject_CallMethodIdObjArgs(interp->importlib, &PyId__get_sourcefile, cpathobj, NULL); if (pathobj == NULL) @@ -1441,7 +1441,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, } if (initializing > 0) { /* _bootstrap._lock_unlock_module() releases the import lock */ - value = _PyObject_CallMethodObjIdArgs(interp->importlib, + value = _PyObject_CallMethodIdObjArgs(interp->importlib, &PyId__lock_unlock_module, abs_name, NULL); if (value == NULL) @@ -1459,7 +1459,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, } else { /* _bootstrap._find_and_load() releases the import lock */ - mod = _PyObject_CallMethodObjIdArgs(interp->importlib, + mod = _PyObject_CallMethodIdObjArgs(interp->importlib, &PyId__find_and_load, abs_name, builtins_import, NULL); if (mod == NULL) { @@ -1528,7 +1528,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, } } else { - final_mod = _PyObject_CallMethodObjIdArgs(interp->importlib, + final_mod = _PyObject_CallMethodIdObjArgs(interp->importlib, &PyId__handle_fromlist, mod, fromlist, builtins_import, NULL); -- cgit v1.2.1 From 5756d7dace379752116766907b63ac77ffc4bc98 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 6 May 2013 21:15:57 +0200 Subject: Issue #1545463: Global variables caught in reference cycles are now garbage-collected at shutdown. --- Python/import.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index b77eda106d..cd4fb788a1 100644 --- a/Python/import.c +++ b/Python/import.c @@ -403,6 +403,14 @@ PyImport_Cleanup(void) } } + /* Collect garbage remaining after deleting the modules. Mostly + reference cycles created by classes. */ + PyGC_Collect(); + + /* Dump GC stats before it's too late, since it uses the warnings + machinery. */ + _PyGC_DumpShutdownStats(); + /* Next, delete sys and builtins (in that order) */ value = PyDict_GetItemString(modules, "sys"); if (value != NULL && PyModule_Check(value)) { -- cgit v1.2.1 From 63d05b117ac621279bea27b1a7903d3085d18240 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 8 May 2013 13:23:25 +0200 Subject: Issue #1545463: At shutdown, defer finalization of codec modules so that stderr remains usable. (should fix Windows buildbot failures on test_gc) --- Python/import.c | 55 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 20 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index cd4fb788a1..1fbafecc1b 100644 --- a/Python/import.c +++ b/Python/import.c @@ -295,6 +295,30 @@ static char* sys_files[] = { NULL }; +static int +is_essential_module(PyObject *name) +{ + Py_ssize_t name_len; + char *name_str = PyUnicode_AsUTF8AndSize(name, &name_len); + + if (name_str == NULL) { + PyErr_Clear(); + return 0; + } + if (strcmp(name_str, "builtins") == 0) + return 1; + if (strcmp(name_str, "sys") == 0) + return 1; + /* These are all needed for stderr to still function */ + if (strcmp(name_str, "codecs") == 0) + return 1; + if (strcmp(name_str, "_codecs") == 0) + return 1; + if (strncmp(name_str, "encodings.", 10) == 0) + return 1; + return 0; +} + /* Un-initialize things, as good as we can */ @@ -374,9 +398,7 @@ PyImport_Cleanup(void) if (value->ob_refcnt != 1) continue; if (PyUnicode_Check(key) && PyModule_Check(value)) { - if (PyUnicode_CompareWithASCIIString(key, "builtins") == 0) - continue; - if (PyUnicode_CompareWithASCIIString(key, "sys") == 0) + if (is_essential_module(key)) continue; if (Py_VerboseFlag) PySys_FormatStderr( @@ -392,9 +414,7 @@ PyImport_Cleanup(void) pos = 0; while (PyDict_Next(modules, &pos, &key, &value)) { if (PyUnicode_Check(key) && PyModule_Check(value)) { - if (PyUnicode_CompareWithASCIIString(key, "builtins") == 0) - continue; - if (PyUnicode_CompareWithASCIIString(key, "sys") == 0) + if (is_essential_module(key)) continue; if (Py_VerboseFlag) PySys_FormatStderr("# cleanup[2] %U\n", key); @@ -411,20 +431,15 @@ PyImport_Cleanup(void) machinery. */ _PyGC_DumpShutdownStats(); - /* Next, delete sys and builtins (in that order) */ - value = PyDict_GetItemString(modules, "sys"); - if (value != NULL && PyModule_Check(value)) { - if (Py_VerboseFlag) - PySys_WriteStderr("# cleanup sys\n"); - _PyModule_Clear(value); - PyDict_SetItemString(modules, "sys", Py_None); - } - value = PyDict_GetItemString(modules, "builtins"); - if (value != NULL && PyModule_Check(value)) { - if (Py_VerboseFlag) - PySys_WriteStderr("# cleanup builtins\n"); - _PyModule_Clear(value); - PyDict_SetItemString(modules, "builtins", Py_None); + /* Next, delete all remaining modules */ + pos = 0; + while (PyDict_Next(modules, &pos, &key, &value)) { + if (PyUnicode_Check(key) && PyModule_Check(value)) { + if (Py_VerboseFlag) + PySys_FormatStderr("# cleanup[3] %U\n", key); + _PyModule_Clear(value); + PyDict_SetItem(modules, key, Py_None); + } } /* Finally, clear and delete the modules directory */ -- cgit v1.2.1 From 4bb13f5d6801f4857651f13a800a964cb8b47223 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sun, 19 May 2013 01:11:58 +0200 Subject: Issue #17937: Try harder to collect cyclic garbage at shutdown. --- Python/import.c | 1 + 1 file changed, 1 insertion(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 1fbafecc1b..a42b0f89b0 100644 --- a/Python/import.c +++ b/Python/import.c @@ -444,6 +444,7 @@ PyImport_Cleanup(void) /* Finally, clear and delete the modules directory */ PyDict_Clear(modules); + _PyGC_CollectNoFail(); interp->modules = NULL; Py_DECREF(modules); } -- cgit v1.2.1 From 41c36b5a56ae88660fd66b03ade69aecf4c2de86 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 31 May 2013 23:18:39 -0400 Subject: Issue #18065: For frozen packages set __path__ to []. Previously __path__ was set to [__name__], but that could lead to bad results if someone managed to circumvent the frozen importer and somehow ended up with a finder that thought __name__ was a legit directory/location. --- Python/import.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index a42b0f89b0..0bb46d2e1e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1107,19 +1107,17 @@ PyImport_ImportFrozenModuleObject(PyObject *name) goto err_return; } if (ispackage) { - /* Set __path__ to the package name */ + /* Set __path__ to the empty list */ PyObject *d, *l; int err; m = PyImport_AddModuleObject(name); if (m == NULL) goto err_return; d = PyModule_GetDict(m); - l = PyList_New(1); + l = PyList_New(0); if (l == NULL) { goto err_return; } - Py_INCREF(name); - PyList_SET_ITEM(l, 0, name); err = PyDict_SetItemString(d, "__path__", l); Py_DECREF(l); if (err != 0) -- cgit v1.2.1 From 21d4e9534aceac9c2753b70bffe4f75ff38ec525 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 12 Jun 2013 23:29:18 -0400 Subject: Issue #15767: Touch up ModuleNotFoundError usage by import. Forgot to raise ModuleNotFoundError when None is found in sys.modules. This led to introducing the C function PyErr_SetImportErrorSubclass() to make setting ModuleNotFoundError easier. Also updated the reference docs to mention ModuleNotFoundError appropriately. Updated the docs for ModuleNotFoundError to mention the None in sys.modules case. Lastly, it was noticed that PyErr_SetImportError() was not setting an exception when returning None in one case. That issue is now fixed. --- Python/import.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 0bb46d2e1e..fad54e66ce 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1436,7 +1436,8 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, PyObject *msg = PyUnicode_FromFormat("import of %R halted; " "None in sys.modules", abs_name); if (msg != NULL) { - PyErr_SetImportError(msg, abs_name, NULL); + PyErr_SetImportErrorSubclass(PyExc_ModuleNotFoundError, msg, + abs_name, NULL); Py_DECREF(msg); } mod = NULL; -- cgit v1.2.1 From 23c2e70efd68fb16de7b6b3ad0e469784c966ace Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sun, 23 Jun 2013 15:53:09 +0200 Subject: import.c does neither need mode_t nor _mkdir() anymore --- Python/import.c | 8 -------- 1 file changed, 8 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index fad54e66ce..b9c5924887 100644 --- a/Python/import.c +++ b/Python/import.c @@ -19,14 +19,6 @@ extern "C" { #endif -#ifdef MS_WINDOWS -/* for stat.st_mode */ -typedef unsigned short mode_t; -/* for _mkdir */ -#include -#endif - - #define CACHEDIR "__pycache__" /* See _PyImport_FixupExtensionObject() below */ -- cgit v1.2.1 From af5e4f4219618da2bfcb02b52ac7c7ac54b7542a Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 4 Jul 2013 17:48:16 -0400 Subject: Issue #15767: Revert 3a50025f1900 for ModuleNotFoundError --- Python/import.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index b9c5924887..2e5d205973 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1428,8 +1428,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, PyObject *msg = PyUnicode_FromFormat("import of %R halted; " "None in sys.modules", abs_name); if (msg != NULL) { - PyErr_SetImportErrorSubclass(PyExc_ModuleNotFoundError, msg, - abs_name, NULL); + PyErr_SetImportError(msg, abs_name, NULL); Py_DECREF(msg); } mod = NULL; -- cgit v1.2.1 From 556ce6a1ab262d2aa56c505aecf6173382ed0e3c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 16 Jul 2013 22:26:05 +0200 Subject: Issue #18408: handle PySys_GetObject() failure, raise a RuntimeError --- Python/import.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 2e5d205973..a1c0aee540 100644 --- a/Python/import.c +++ b/Python/import.c @@ -85,8 +85,10 @@ _PyImportZip_Init(void) int err = 0; path_hooks = PySys_GetObject("path_hooks"); - if (path_hooks == NULL) + if (path_hooks == NULL) { + PyErr_SetString(PyExc_RuntimeError, "unable to get sys.path_hooks"); goto error; + } if (Py_VerboseFlag) PySys_WriteStderr("# installing zipimport hook\n"); @@ -944,11 +946,11 @@ PyAPI_FUNC(PyObject *) PyImport_GetImporter(PyObject *path) { PyObject *importer=NULL, *path_importer_cache=NULL, *path_hooks=NULL; - if ((path_importer_cache = PySys_GetObject("path_importer_cache"))) { - if ((path_hooks = PySys_GetObject("path_hooks"))) { - importer = get_path_importer(path_importer_cache, - path_hooks, path); - } + path_importer_cache = PySys_GetObject("path_importer_cache"); + path_hooks = PySys_GetObject("path_hooks"); + if (path_importer_cache != NULL && path_hooks != NULL) { + importer = get_path_importer(path_importer_cache, + path_hooks, path); } Py_XINCREF(importer); /* get_path_importer returns a borrowed reference */ return importer; -- cgit v1.2.1