From 004869681a2b618d5b64fd42b8f7812f65c7dc88 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 15 Nov 2012 21:39:36 -0500 Subject: Issue #15894: Document why we don't worry about re-acquiring the global import lock after forking. --- Python/import.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 2f71b97bc8..6882b57060 100644 --- a/Python/import.c +++ b/Python/import.c @@ -202,8 +202,11 @@ _PyImport_ReInitLock(void) if (import_lock_level > 1) { /* Forked as a side effect of import */ long me = PyThread_get_thread_ident(); - PyThread_acquire_lock(import_lock, 0); - /* XXX: can the previous line fail? */ + /* The following could fail if the lock is already held, but forking as + a side-effect of an import is a) rare, b) nuts, and c) difficult to + do thanks to the lock only being held when doing individual module + locks per import. */ + PyThread_acquire_lock(import_lock, NOWAIT_LOCK); import_lock_thread = me; import_lock_level--; } else { -- cgit v1.2.1 From 838169869f9d3474c58ed662b1a51541de235c34 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 14 Dec 2012 17:04:59 +0200 Subject: Issue #16421: allow to load multiple modules from the same shared object. Patch by V?clav ?milauer. --- Python/import.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 6882b57060..8e7482ac4e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -452,12 +452,12 @@ PyImport_GetMagicTag(void) /* Magic for extension modules (built-in as well as dynamically loaded). To prevent initializing an extension module more than - once, we keep a static dictionary 'extensions' keyed by module name - (for built-in modules) or by filename (for dynamically loaded - modules), containing these modules. A copy of the module's - dictionary is stored by calling _PyImport_FixupExtensionObject() - immediately after the module initialization function succeeds. A - copy can be retrieved from there by calling + once, we keep a static dictionary 'extensions' keyed by the tuple + (module name, module name) (for built-in modules) or by + (filename, module name) (for dynamically loaded modules), containing these + modules. A copy of the module's dictionary is stored by calling + _PyImport_FixupExtensionObject() immediately after the module initialization + function succeeds. A copy can be retrieved from there by calling _PyImport_FindExtensionObject(). Modules which do support multiple initialization set their m_size @@ -470,7 +470,7 @@ int _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, PyObject *filename) { - PyObject *modules, *dict; + PyObject *modules, *dict, *filename_name; struct PyModuleDef *def; if (extensions == NULL) { extensions = PyDict_New(); @@ -508,7 +508,11 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, if (def->m_base.m_copy == NULL) return -1; } - PyDict_SetItem(extensions, filename, (PyObject*)def); + filename_name = PyTuple_Pack(2,filename, name); + if (filename_name == NULL) + return -1; + if (PyDict_SetItem(extensions, filename_name, (PyObject*)def) < 0) + return -1; return 0; } @@ -528,11 +532,14 @@ _PyImport_FixupBuiltin(PyObject *mod, char *name) PyObject * _PyImport_FindExtensionObject(PyObject *name, PyObject *filename) { - PyObject *mod, *mdict; + PyObject *mod, *mdict, *filename_name; PyModuleDef* def; if (extensions == NULL) return NULL; - def = (PyModuleDef*)PyDict_GetItem(extensions, filename); + filename_name = PyTuple_Pack(2,filename, name); + if (filename_name == NULL) + return NULL; + def = (PyModuleDef*)PyDict_GetItem(extensions, filename_name); if (def == NULL) return NULL; if (def->m_size == -1) { -- cgit v1.2.1 From a0d1572bb8860a41834b2bca05748df2a28ffb5d Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 15 Dec 2012 00:05:16 -0500 Subject: cleanup and fix refleaks --- Python/import.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 8e7482ac4e..50267895c5 100644 --- a/Python/import.c +++ b/Python/import.c @@ -470,8 +470,9 @@ int _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, PyObject *filename) { - PyObject *modules, *dict, *filename_name; + PyObject *modules, *dict, *key; struct PyModuleDef *def; + int res; if (extensions == NULL) { extensions = PyDict_New(); if (extensions == NULL) @@ -508,10 +509,12 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, if (def->m_base.m_copy == NULL) return -1; } - filename_name = PyTuple_Pack(2,filename, name); - if (filename_name == NULL) + key = PyTuple_Pack(2, filename, name); + if (key == NULL) return -1; - if (PyDict_SetItem(extensions, filename_name, (PyObject*)def) < 0) + res = PyDict_SetItem(extensions, key, (PyObject *)def); + Py_DECREF(key); + if (res < 0) return -1; return 0; } @@ -532,14 +535,15 @@ _PyImport_FixupBuiltin(PyObject *mod, char *name) PyObject * _PyImport_FindExtensionObject(PyObject *name, PyObject *filename) { - PyObject *mod, *mdict, *filename_name; + PyObject *mod, *mdict, *key; PyModuleDef* def; if (extensions == NULL) return NULL; - filename_name = PyTuple_Pack(2,filename, name); - if (filename_name == NULL) + key = PyTuple_Pack(2, filename, name); + if (key == NULL) return NULL; - def = (PyModuleDef*)PyDict_GetItem(extensions, filename_name); + def = (PyModuleDef *)PyDict_GetItem(extensions, key); + Py_DECREF(key); if (def == NULL) return NULL; if (def->m_size == -1) { -- cgit v1.2.1 From 63c6c6e958c8f9749e69ad62e653147ff08680c1 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Mon, 18 Mar 2013 23:13:31 -0700 Subject: sprinkle const --- Python/import.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index a50ef05b07..8773be46ad 100644 --- a/Python/import.c +++ b/Python/import.c @@ -846,7 +846,7 @@ imp_fix_co_filename(PyObject *self, PyObject *args) /* Forward */ -static struct _frozen * find_frozen(PyObject *); +static const struct _frozen * find_frozen(PyObject *); /* Helper to test for built-in module */ @@ -983,10 +983,10 @@ init_builtin(PyObject *name) /* Frozen modules */ -static struct _frozen * +static const struct _frozen * find_frozen(PyObject *name) { - struct _frozen *p; + const struct _frozen *p; if (name == NULL) return NULL; @@ -1003,7 +1003,7 @@ find_frozen(PyObject *name) static PyObject * get_frozen_object(PyObject *name) { - struct _frozen *p = find_frozen(name); + const struct _frozen *p = find_frozen(name); int size; if (p == NULL) { @@ -1027,7 +1027,7 @@ get_frozen_object(PyObject *name) static PyObject * is_frozen_package(PyObject *name) { - struct _frozen *p = find_frozen(name); + const struct _frozen *p = find_frozen(name); int size; if (p == NULL) { @@ -1054,7 +1054,7 @@ is_frozen_package(PyObject *name) int PyImport_ImportFrozenModuleObject(PyObject *name) { - struct _frozen *p; + const struct _frozen *p; PyObject *co, *m, *path; int ispackage; int size; @@ -1781,7 +1781,7 @@ static PyObject * imp_is_frozen(PyObject *self, PyObject *args) { PyObject *name; - struct _frozen *p; + const struct _frozen *p; if (!PyArg_ParseTuple(args, "U:is_frozen", &name)) return NULL; p = find_frozen(name); -- cgit v1.2.1 From 05af1e7fca1463b6a0fc7a790313d93a3e68c222 Mon Sep 17 00:00:00 2001 From: Alexandre Vassalotti Date: Thu, 2 May 2013 10:44:04 -0700 Subject: Closes #17892: Fix the name of _PyObject_CallMethodObjIdArgs --- Python/import.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 805cdd7e2a..b77eda106d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -704,7 +704,7 @@ PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, "no interpreter!"); } - pathobj = _PyObject_CallMethodObjIdArgs(interp->importlib, + pathobj = _PyObject_CallMethodIdObjArgs(interp->importlib, &PyId__get_sourcefile, cpathobj, NULL); if (pathobj == NULL) @@ -1441,7 +1441,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, } if (initializing > 0) { /* _bootstrap._lock_unlock_module() releases the import lock */ - value = _PyObject_CallMethodObjIdArgs(interp->importlib, + value = _PyObject_CallMethodIdObjArgs(interp->importlib, &PyId__lock_unlock_module, abs_name, NULL); if (value == NULL) @@ -1459,7 +1459,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, } else { /* _bootstrap._find_and_load() releases the import lock */ - mod = _PyObject_CallMethodObjIdArgs(interp->importlib, + mod = _PyObject_CallMethodIdObjArgs(interp->importlib, &PyId__find_and_load, abs_name, builtins_import, NULL); if (mod == NULL) { @@ -1528,7 +1528,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, } } else { - final_mod = _PyObject_CallMethodObjIdArgs(interp->importlib, + final_mod = _PyObject_CallMethodIdObjArgs(interp->importlib, &PyId__handle_fromlist, mod, fromlist, builtins_import, NULL); -- cgit v1.2.1 From 5756d7dace379752116766907b63ac77ffc4bc98 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 6 May 2013 21:15:57 +0200 Subject: Issue #1545463: Global variables caught in reference cycles are now garbage-collected at shutdown. --- Python/import.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index b77eda106d..cd4fb788a1 100644 --- a/Python/import.c +++ b/Python/import.c @@ -403,6 +403,14 @@ PyImport_Cleanup(void) } } + /* Collect garbage remaining after deleting the modules. Mostly + reference cycles created by classes. */ + PyGC_Collect(); + + /* Dump GC stats before it's too late, since it uses the warnings + machinery. */ + _PyGC_DumpShutdownStats(); + /* Next, delete sys and builtins (in that order) */ value = PyDict_GetItemString(modules, "sys"); if (value != NULL && PyModule_Check(value)) { -- cgit v1.2.1 From 63d05b117ac621279bea27b1a7903d3085d18240 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 8 May 2013 13:23:25 +0200 Subject: Issue #1545463: At shutdown, defer finalization of codec modules so that stderr remains usable. (should fix Windows buildbot failures on test_gc) --- Python/import.c | 55 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 20 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index cd4fb788a1..1fbafecc1b 100644 --- a/Python/import.c +++ b/Python/import.c @@ -295,6 +295,30 @@ static char* sys_files[] = { NULL }; +static int +is_essential_module(PyObject *name) +{ + Py_ssize_t name_len; + char *name_str = PyUnicode_AsUTF8AndSize(name, &name_len); + + if (name_str == NULL) { + PyErr_Clear(); + return 0; + } + if (strcmp(name_str, "builtins") == 0) + return 1; + if (strcmp(name_str, "sys") == 0) + return 1; + /* These are all needed for stderr to still function */ + if (strcmp(name_str, "codecs") == 0) + return 1; + if (strcmp(name_str, "_codecs") == 0) + return 1; + if (strncmp(name_str, "encodings.", 10) == 0) + return 1; + return 0; +} + /* Un-initialize things, as good as we can */ @@ -374,9 +398,7 @@ PyImport_Cleanup(void) if (value->ob_refcnt != 1) continue; if (PyUnicode_Check(key) && PyModule_Check(value)) { - if (PyUnicode_CompareWithASCIIString(key, "builtins") == 0) - continue; - if (PyUnicode_CompareWithASCIIString(key, "sys") == 0) + if (is_essential_module(key)) continue; if (Py_VerboseFlag) PySys_FormatStderr( @@ -392,9 +414,7 @@ PyImport_Cleanup(void) pos = 0; while (PyDict_Next(modules, &pos, &key, &value)) { if (PyUnicode_Check(key) && PyModule_Check(value)) { - if (PyUnicode_CompareWithASCIIString(key, "builtins") == 0) - continue; - if (PyUnicode_CompareWithASCIIString(key, "sys") == 0) + if (is_essential_module(key)) continue; if (Py_VerboseFlag) PySys_FormatStderr("# cleanup[2] %U\n", key); @@ -411,20 +431,15 @@ PyImport_Cleanup(void) machinery. */ _PyGC_DumpShutdownStats(); - /* Next, delete sys and builtins (in that order) */ - value = PyDict_GetItemString(modules, "sys"); - if (value != NULL && PyModule_Check(value)) { - if (Py_VerboseFlag) - PySys_WriteStderr("# cleanup sys\n"); - _PyModule_Clear(value); - PyDict_SetItemString(modules, "sys", Py_None); - } - value = PyDict_GetItemString(modules, "builtins"); - if (value != NULL && PyModule_Check(value)) { - if (Py_VerboseFlag) - PySys_WriteStderr("# cleanup builtins\n"); - _PyModule_Clear(value); - PyDict_SetItemString(modules, "builtins", Py_None); + /* Next, delete all remaining modules */ + pos = 0; + while (PyDict_Next(modules, &pos, &key, &value)) { + if (PyUnicode_Check(key) && PyModule_Check(value)) { + if (Py_VerboseFlag) + PySys_FormatStderr("# cleanup[3] %U\n", key); + _PyModule_Clear(value); + PyDict_SetItem(modules, key, Py_None); + } } /* Finally, clear and delete the modules directory */ -- cgit v1.2.1