From b5162583f02ef19edb3b4804cac793f1183c6283 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Feb 2011 23:12:28 +0000 Subject: Issue #3080: Mark PyWin_FindRegisteredModule() as private This function was not declared in Python public API (in any .h file) and not documented. Mark it as private to prepare a change of its API. --- Python/import.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 23752eeb72..f4015b52fa 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1547,8 +1547,8 @@ PyImport_GetImporter(PyObject *path) { pathname and an open file. Return NULL if the module is not found. */ #ifdef MS_COREDLL -extern FILE *PyWin_FindRegisteredModule(const char *, struct filedescr **, - char *, Py_ssize_t); +extern FILE *_PyWin_FindRegisteredModule(const char *, struct filedescr **, + char *, Py_ssize_t); #endif static int case_ok(char *, Py_ssize_t, Py_ssize_t, char *); @@ -1631,7 +1631,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, return &fd_builtin; } #ifdef MS_COREDLL - fp = PyWin_FindRegisteredModule(name, &fdp, buf, buflen); + fp = _PyWin_FindRegisteredModule(name, &fdp, buf, buflen); if (fp != NULL) { *p_fp = fp; return fdp; -- cgit v1.2.1 From 90268ce7e125f2d8a1d9b8f41b51d31f96b1c8fd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Feb 2011 00:02:00 +0000 Subject: Issue #3080: Mark _PyImport_FindBuiltin() argument as constant And as a consequence, mark also name argument of _PyImport_FindExtensionUnicode() constant too. But I plan to change this argument type to PyObject* later. --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index f4015b52fa..39cd93f7dd 100644 --- a/Python/import.c +++ b/Python/import.c @@ -636,7 +636,7 @@ _PyImport_FixupBuiltin(PyObject *mod, char *name) } PyObject * -_PyImport_FindExtensionUnicode(char *name, PyObject *filename) +_PyImport_FindExtensionUnicode(const char *name, PyObject *filename) { PyObject *mod, *mdict; PyModuleDef* def; @@ -680,7 +680,7 @@ _PyImport_FindExtensionUnicode(char *name, PyObject *filename) } PyObject * -_PyImport_FindBuiltin(char *name) +_PyImport_FindBuiltin(const char *name) { PyObject *res, *filename; filename = PyUnicode_FromString(name); -- cgit v1.2.1 From b05cc9bc8518f72fcb3e33b764f24dae32d5524f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 4 Mar 2011 12:57:09 +0000 Subject: Issue #3080: Add PyImport_AddModuleObject() and PyImport_ExecCodeModuleObject() --- Python/import.c | 87 +++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 66 insertions(+), 21 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 39cd93f7dd..22a7c8744d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -698,18 +698,18 @@ _PyImport_FindBuiltin(const char *name) 'NEW' REFERENCE! */ PyObject * -PyImport_AddModule(const char *name) +PyImport_AddModuleObject(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); PyObject *m; - if ((m = PyDict_GetItemString(modules, name)) != NULL && + if ((m = PyDict_GetItem(modules, name)) != NULL && PyModule_Check(m)) return m; - m = PyModule_New(name); + m = PyModule_NewObject(name); if (m == NULL) return NULL; - if (PyDict_SetItemString(modules, name, m) != 0) { + if (PyDict_SetItem(modules, name, m) != 0) { Py_DECREF(m); return NULL; } @@ -718,14 +718,27 @@ PyImport_AddModule(const char *name) return m; } +PyObject * +PyImport_AddModule(const char *name) +{ + PyObject *nameobj, *module; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + module = PyImport_AddModuleObject(nameobj); + Py_DECREF(nameobj); + return module; +} + + /* Remove name from sys.modules, if it's there. */ static void -remove_module(const char *name) +remove_module(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); - if (PyDict_GetItemString(modules, name) == NULL) + if (PyDict_GetItem(modules, name) == NULL) return; - if (PyDict_DelItemString(modules, name) < 0) + if (PyDict_DelItem(modules, name) < 0) Py_FatalError("import: deleting existing key in" "sys.modules failed"); } @@ -762,11 +775,43 @@ PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname) PyObject * PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, char *cpathname) +{ + PyObject *m = NULL; + PyObject *nameobj, *pathobj = NULL, *cpathobj = NULL; + + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + + if (pathname != NULL) { + pathobj = PyUnicode_DecodeFSDefault(pathname); + if (pathobj == NULL) + goto error; + } else + pathobj = NULL; + if (cpathname != NULL) { + cpathobj = PyUnicode_DecodeFSDefault(cpathname); + if (cpathobj == NULL) + goto error; + } else + cpathobj = NULL; + m = PyImport_ExecCodeModuleObject(nameobj, co, pathobj, cpathobj); +error: + Py_DECREF(nameobj); + Py_XDECREF(pathobj); + Py_XDECREF(cpathobj); + return m; +} + +PyObject* +PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, + PyObject *cpathname) { PyObject *modules = PyImport_GetModuleDict(); PyObject *m, *d, *v; + PyObject *pathbytes; - m = PyImport_AddModule(name); + m = PyImport_AddModuleObject(name); if (m == NULL) return NULL; /* If the module is being reloaded, we get the old module back @@ -778,12 +823,18 @@ PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, goto error; } /* Remember the filename as the __file__ attribute */ - v = NULL; if (pathname != NULL) { - v = get_sourcefile(pathname); + pathbytes = PyUnicode_EncodeFSDefault(pathname); + if (pathbytes != NULL) { + v = get_sourcefile(PyBytes_AS_STRING(pathbytes)); + Py_DECREF(pathbytes); + } else + v = NULL; if (v == NULL) PyErr_Clear(); } + else + v = NULL; if (v == NULL) { v = ((PyCodeObject *)co)->co_filename; Py_INCREF(v); @@ -793,27 +844,21 @@ PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, Py_DECREF(v); /* Remember the pyc path name as the __cached__ attribute. */ - if (cpathname == NULL) { - v = Py_None; - Py_INCREF(v); - } - else if ((v = PyUnicode_FromString(cpathname)) == NULL) { - PyErr_Clear(); /* Not important enough to report */ + if (cpathname != NULL) + v = cpathname; + else v = Py_None; - Py_INCREF(v); - } if (PyDict_SetItemString(d, "__cached__", v) != 0) PyErr_Clear(); /* Not important enough to report */ - Py_DECREF(v); v = PyEval_EvalCode(co, d, d); if (v == NULL) goto error; Py_DECREF(v); - if ((m = PyDict_GetItemString(modules, name)) == NULL) { + if ((m = PyDict_GetItem(modules, name)) == NULL) { PyErr_Format(PyExc_ImportError, - "Loaded module %.200s not found in sys.modules", + "Loaded module %R not found in sys.modules", name); return NULL; } -- cgit v1.2.1 From 426e754b8e8cf35a60153914ae341ca57e9d5031 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 7 Mar 2011 16:57:48 +0100 Subject: import.c: replace tab by spaces --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 22a7c8744d..b2757dffdc 100644 --- a/Python/import.c +++ b/Python/import.c @@ -329,7 +329,7 @@ _PyImport_ReInitLock(void) /* Forked as a side effect of import */ long me = PyThread_get_thread_ident(); PyThread_acquire_lock(import_lock, 0); - /* XXX: can the previous line fail? */ + /* XXX: can the previous line fail? */ import_lock_thread = me; import_lock_level--; } else { -- cgit v1.2.1 From 13abff792323d15e50b40c4ef978441790bef7b3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 7 Mar 2011 17:08:21 +0100 Subject: Issue #3080: PyImport_Cleanup() uses Unicode Replace strcmp() by PyUnicode_CompareWithASCIIString() --- Python/import.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index b2757dffdc..9366be0589 100644 --- a/Python/import.c +++ b/Python/import.c @@ -418,7 +418,6 @@ void PyImport_Cleanup(void) { Py_ssize_t pos, ndone; - char *name; PyObject *key, *value, *dict; PyInterpreterState *interp = PyThreadState_GET()->interp; PyObject *modules = interp->modules; @@ -491,14 +490,13 @@ PyImport_Cleanup(void) if (value->ob_refcnt != 1) continue; if (PyUnicode_Check(key) && PyModule_Check(value)) { - name = _PyUnicode_AsString(key); - if (strcmp(name, "builtins") == 0) + if (PyUnicode_CompareWithASCIIString(key, "builtins") == 0) continue; - if (strcmp(name, "sys") == 0) + if (PyUnicode_CompareWithASCIIString(key, "sys") == 0) continue; if (Py_VerboseFlag) - PySys_WriteStderr( - "# cleanup[1] %s\n", name); + PySys_FormatStderr( + "# cleanup[1] %U\n", key); _PyModule_Clear(value); PyDict_SetItem(modules, key, Py_None); ndone++; @@ -510,13 +508,12 @@ PyImport_Cleanup(void) pos = 0; while (PyDict_Next(modules, &pos, &key, &value)) { if (PyUnicode_Check(key) && PyModule_Check(value)) { - name = _PyUnicode_AsString(key); - if (strcmp(name, "builtins") == 0) + if (PyUnicode_CompareWithASCIIString(key, "builtins") == 0) continue; - if (strcmp(name, "sys") == 0) + if (PyUnicode_CompareWithASCIIString(key, "sys") == 0) continue; if (Py_VerboseFlag) - PySys_WriteStderr("# cleanup[2] %s\n", name); + PySys_FormatStderr("# cleanup[2] %U\n", key); _PyModule_Clear(value); PyDict_SetItem(modules, key, Py_None); } -- cgit v1.2.1 From 46cbf15e71a6cd5841e2547357932aa1f8c93936 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 20 Mar 2011 01:50:21 +0100 Subject: Issue #3080: Add PyImport_ImportFrozenModuleObject() find_frozen(), get_frozen_object(), is_frozen_package() and other functions related to frozen modules use Unicode strings instead of byte strings. --- Python/import.c | 98 +++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 61 insertions(+), 37 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 9366be0589..09fc52cbda 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1429,7 +1429,7 @@ get_sourcefile(char *file) static PyObject *load_module(char *, FILE *, char *, int, PyObject *); static struct filedescr *find_module(char *, char *, PyObject *, char *, size_t, FILE **, PyObject **); -static struct _frozen * find_frozen(char *); +static struct _frozen * find_frozen(PyObject *); /* Load a package and return its module object WITH INCREMENTED REFERENCE COUNT */ @@ -1617,6 +1617,8 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, size_t saved_namelen; char *saved_buf = NULL; #endif + PyObject *fullname_obj; + if (p_loader != NULL) *p_loader = NULL; @@ -1662,9 +1664,16 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, Py_DECREF(meta_path); } - if (find_frozen(fullname) != NULL) { - strcpy(buf, fullname); - return &fd_frozen; + if (fullname != NULL) { + fullname_obj = PyUnicode_FromString(fullname); + if (fullname == NULL) + return NULL; + if (find_frozen(fullname_obj) != NULL) { + Py_DECREF(fullname_obj); + strcpy(buf, fullname); + return &fd_frozen; + } + Py_DECREF(fullname_obj); } if (path == NULL) { @@ -2226,37 +2235,37 @@ init_builtin(char *name) /* Frozen modules */ static struct _frozen * -find_frozen(char *name) +find_frozen(PyObject *name) { struct _frozen *p; - if (!name) + if (name == NULL) return NULL; for (p = PyImport_FrozenModules; ; p++) { if (p->name == NULL) return NULL; - if (strcmp(p->name, name) == 0) + if (PyUnicode_CompareWithASCIIString(name, p->name) == 0) break; } return p; } static PyObject * -get_frozen_object(char *name) +get_frozen_object(PyObject *name) { struct _frozen *p = find_frozen(name); int size; if (p == NULL) { PyErr_Format(PyExc_ImportError, - "No such frozen object named %.200s", + "No such frozen object named %R", name); return NULL; } if (p->code == NULL) { PyErr_Format(PyExc_ImportError, - "Excluded frozen object named %.200s", + "Excluded frozen object named %R", name); return NULL; } @@ -2267,14 +2276,14 @@ get_frozen_object(char *name) } static PyObject * -is_frozen_package(char *name) +is_frozen_package(PyObject *name) { struct _frozen *p = find_frozen(name); int size; if (p == NULL) { PyErr_Format(PyExc_ImportError, - "No such frozen object named %.200s", + "No such frozen object named %R", name); return NULL; } @@ -2294,19 +2303,20 @@ is_frozen_package(char *name) This function is also used from frozenmain.c */ int -PyImport_ImportFrozenModule(char *name) +PyImport_ImportFrozenModuleObject(PyObject *name) { - struct _frozen *p = find_frozen(name); - PyObject *co; - PyObject *m; + struct _frozen *p; + PyObject *co, *m, *path; int ispackage; int size; + p = find_frozen(name); + if (p == NULL) return 0; if (p->code == NULL) { PyErr_Format(PyExc_ImportError, - "Excluded frozen object named %.200s", + "Excluded frozen object named %R", name); return -1; } @@ -2315,40 +2325,41 @@ PyImport_ImportFrozenModule(char *name) if (ispackage) size = -size; if (Py_VerboseFlag) - PySys_WriteStderr("import %s # frozen%s\n", + PySys_FormatStderr("import %U # frozen%s\n", name, ispackage ? " package" : ""); co = PyMarshal_ReadObjectFromString((char *)p->code, size); if (co == NULL) return -1; if (!PyCode_Check(co)) { PyErr_Format(PyExc_TypeError, - "frozen object %.200s is not a code object", + "frozen object %R is not a code object", name); goto err_return; } if (ispackage) { /* Set __path__ to the package name */ - PyObject *d, *s, *l; + PyObject *d, *l; int err; - m = PyImport_AddModule(name); + m = PyImport_AddModuleObject(name); if (m == NULL) goto err_return; d = PyModule_GetDict(m); - s = PyUnicode_InternFromString(name); - if (s == NULL) - goto err_return; l = PyList_New(1); if (l == NULL) { - Py_DECREF(s); goto err_return; } - PyList_SET_ITEM(l, 0, s); + Py_INCREF(name); + PyList_SET_ITEM(l, 0, name); err = PyDict_SetItemString(d, "__path__", l); Py_DECREF(l); if (err != 0) goto err_return; } - m = PyImport_ExecCodeModuleEx(name, co, ""); + path = PyUnicode_FromString(""); + if (path == NULL) + goto err_return; + m = PyImport_ExecCodeModuleObject(name, co, path, NULL); + Py_DECREF(path); if (m == NULL) goto err_return; Py_DECREF(co); @@ -2359,6 +2370,19 @@ err_return: return -1; } +int +PyImport_ImportFrozenModule(char *name) +{ + PyObject *nameobj; + int ret; + nameobj = PyUnicode_InternFromString(name); + if (nameobj == NULL) + return -1; + ret = PyImport_ImportFrozenModuleObject(nameobj); + Py_DECREF(nameobj); + return ret; +} + /* Import a module, either built-in, frozen, or external, and return its module object WITH INCREMENTED REFERENCE COUNT */ @@ -3282,19 +3306,19 @@ imp_init_builtin(PyObject *self, PyObject *args) static PyObject * imp_init_frozen(PyObject *self, PyObject *args) { - char *name; + PyObject *name; int ret; PyObject *m; - if (!PyArg_ParseTuple(args, "s:init_frozen", &name)) + if (!PyArg_ParseTuple(args, "U:init_frozen", &name)) return NULL; - ret = PyImport_ImportFrozenModule(name); + ret = PyImport_ImportFrozenModuleObject(name); if (ret < 0) return NULL; if (ret == 0) { Py_INCREF(Py_None); return Py_None; } - m = PyImport_AddModule(name); + m = PyImport_AddModuleObject(name); Py_XINCREF(m); return m; } @@ -3302,9 +3326,9 @@ imp_init_frozen(PyObject *self, PyObject *args) static PyObject * imp_get_frozen_object(PyObject *self, PyObject *args) { - char *name; + PyObject *name; - if (!PyArg_ParseTuple(args, "s:get_frozen_object", &name)) + if (!PyArg_ParseTuple(args, "U:get_frozen_object", &name)) return NULL; return get_frozen_object(name); } @@ -3312,9 +3336,9 @@ imp_get_frozen_object(PyObject *self, PyObject *args) static PyObject * imp_is_frozen_package(PyObject *self, PyObject *args) { - char *name; + PyObject *name; - if (!PyArg_ParseTuple(args, "s:is_frozen_package", &name)) + if (!PyArg_ParseTuple(args, "U:is_frozen_package", &name)) return NULL; return is_frozen_package(name); } @@ -3331,9 +3355,9 @@ imp_is_builtin(PyObject *self, PyObject *args) static PyObject * imp_is_frozen(PyObject *self, PyObject *args) { - char *name; + PyObject *name; struct _frozen *p; - if (!PyArg_ParseTuple(args, "s:is_frozen", &name)) + if (!PyArg_ParseTuple(args, "U:is_frozen", &name)) return NULL; p = find_frozen(name); return PyBool_FromLong((long) (p == NULL ? 0 : p->size)); -- cgit v1.2.1 From 17a3c1b5c7bb9b20e89c29ee9fcf6ee78a6cce82 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 7 Mar 2011 18:20:56 +0100 Subject: Issue #3080: Import builtins using Unicode strings - is_builtin(), init_builtin(), load_builtin() and other builtin related functions use Unicode strings, instead of byte strings - Rename _PyImport_FixupExtensionUnicode() to _PyImport_FixupExtensionObject() - Rename _PyImport_FindExtensionUnicode() to _PyImport_FindExtensionObject() --- Python/import.c | 112 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 60 insertions(+), 52 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 09fc52cbda..597ef439c3 100644 --- a/Python/import.c +++ b/Python/import.c @@ -117,7 +117,7 @@ typedef unsigned short mode_t; static long pyc_magic = MAGIC; static const char *pyc_tag = TAG; -/* See _PyImport_FixupExtensionUnicode() below */ +/* See _PyImport_FixupExtensionObject() below */ static PyObject *extensions = NULL; /* This table is defined in config.c: */ @@ -563,10 +563,10 @@ PyImport_GetMagicTag(void) once, we keep a static dictionary 'extensions' keyed by module name (for built-in modules) or by filename (for dynamically loaded modules), containing these modules. A copy of the module's - dictionary is stored by calling _PyImport_FixupExtensionUnicode() + dictionary is stored by calling _PyImport_FixupExtensionObject() immediately after the module initialization function succeeds. A copy can be retrieved from there by calling - _PyImport_FindExtensionUnicode(). + _PyImport_FindExtensionObject(). Modules which do support multiple initialization set their m_size field to a non-negative number (indicating the size of the @@ -575,7 +575,8 @@ PyImport_GetMagicTag(void) */ int -_PyImport_FixupExtensionUnicode(PyObject *mod, char *name, PyObject *filename) +_PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, + PyObject *filename) { PyObject *modules, *dict; struct PyModuleDef *def; @@ -594,10 +595,10 @@ _PyImport_FixupExtensionUnicode(PyObject *mod, char *name, PyObject *filename) return -1; } modules = PyImport_GetModuleDict(); - if (PyDict_SetItemString(modules, name, mod) < 0) + if (PyDict_SetItem(modules, name, mod) < 0) return -1; if (_PyState_AddModule(mod, def) < 0) { - PyDict_DelItemString(modules, name); + PyDict_DelItem(modules, name); return -1; } if (def->m_size == -1) { @@ -623,17 +624,17 @@ int _PyImport_FixupBuiltin(PyObject *mod, char *name) { int res; - PyObject *filename; - filename = PyUnicode_FromString(name); - if (filename == NULL) + PyObject *nameobj; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) return -1; - res = _PyImport_FixupExtensionUnicode(mod, name, filename); - Py_DECREF(filename); + res = _PyImport_FixupExtensionObject(mod, nameobj, nameobj); + Py_DECREF(nameobj); return res; } PyObject * -_PyImport_FindExtensionUnicode(const char *name, PyObject *filename) +_PyImport_FindExtensionObject(PyObject *name, PyObject *filename) { PyObject *mod, *mdict; PyModuleDef* def; @@ -646,7 +647,7 @@ _PyImport_FindExtensionUnicode(const char *name, PyObject *filename) /* Module does not support repeated initialization */ if (def->m_base.m_copy == NULL) return NULL; - mod = PyImport_AddModule(name); + mod = PyImport_AddModuleObject(name); if (mod == NULL) return NULL; mdict = PyModule_GetDict(mod); @@ -661,16 +662,16 @@ _PyImport_FindExtensionUnicode(const char *name, PyObject *filename) mod = def->m_base.m_init(); if (mod == NULL) return NULL; - PyDict_SetItemString(PyImport_GetModuleDict(), name, mod); + PyDict_SetItem(PyImport_GetModuleDict(), name, mod); Py_DECREF(mod); } if (_PyState_AddModule(mod, def) < 0) { - PyDict_DelItemString(PyImport_GetModuleDict(), name); + PyDict_DelItem(PyImport_GetModuleDict(), name); Py_DECREF(mod); return NULL; } if (Py_VerboseFlag) - PySys_FormatStderr("import %s # previously loaded (%U)\n", + PySys_FormatStderr("import %U # previously loaded (%R)\n", name, filename); return mod; @@ -679,12 +680,12 @@ _PyImport_FindExtensionUnicode(const char *name, PyObject *filename) PyObject * _PyImport_FindBuiltin(const char *name) { - PyObject *res, *filename; - filename = PyUnicode_FromString(name); - if (filename == NULL) + PyObject *res, *nameobj; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) return NULL; - res = _PyImport_FindExtensionUnicode(name, filename); - Py_DECREF(filename); + res = _PyImport_FindExtensionObject(nameobj, nameobj); + Py_DECREF(nameobj); return res; } @@ -1491,11 +1492,12 @@ load_package(char *name, char *pathname) /* Helper to test for built-in module */ static int -is_builtin(char *name) +is_builtin(PyObject *name) { - int i; + int i, cmp; for (i = 0; PyImport_Inittab[i].name != NULL; i++) { - if (strcmp(name, PyImport_Inittab[i].name) == 0) { + cmp = PyUnicode_CompareWithASCIIString(name, PyImport_Inittab[i].name); + if (cmp == 0) { if (PyImport_Inittab[i].initfunc == NULL) return -1; else @@ -1617,7 +1619,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, size_t saved_namelen; char *saved_buf = NULL; #endif - PyObject *fullname_obj; + PyObject *fullname_obj, *nameobj; if (p_loader != NULL) *p_loader = NULL; @@ -1677,10 +1679,15 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, } if (path == NULL) { - if (is_builtin(name)) { + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + if (is_builtin(nameobj)) { + Py_DECREF(nameobj); strcpy(buf, name); return &fd_builtin; } + Py_DECREF(nameobj); #ifdef MS_COREDLL fp = _PyWin_FindRegisteredModule(name, &fdp, buf, buflen); if (fp != NULL) { @@ -2086,40 +2093,35 @@ find_init_module(char *buf) #endif /* HAVE_STAT */ -static int init_builtin(char *); /* Forward */ +static int init_builtin(PyObject *); /* Forward */ static PyObject* -load_builtin(char *name, char *pathname, int type) +load_builtin(PyObject *name, int type) { PyObject *m, *modules; int err; - if (pathname != NULL && pathname[0] != '\0') - name = pathname; - if (type == C_BUILTIN) err = init_builtin(name); else - err = PyImport_ImportFrozenModule(name); + err = PyImport_ImportFrozenModuleObject(name); if (err < 0) return NULL; if (err == 0) { PyErr_Format(PyExc_ImportError, - "Purported %s module %.200s not found", - type == C_BUILTIN ? - "builtin" : "frozen", + "Purported %s module %R not found", + type == C_BUILTIN ? "builtin" : "frozen", name); return NULL; } modules = PyImport_GetModuleDict(); - m = PyDict_GetItemString(modules, name); + m = PyDict_GetItem(modules, name); if (m == NULL) { PyErr_Format( PyExc_ImportError, - "%s module %.200s not properly initialized", - type == C_BUILTIN ? - "builtin" : "frozen", + "%s module %R not properly initialized", + type == C_BUILTIN ? "builtin" : "frozen", name); return NULL; } @@ -2168,9 +2170,15 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) break; case C_BUILTIN: - case PY_FROZEN: - m = load_builtin(name, pathname, type); + case PY_FROZEN: { + PyObject *nameobj = PyUnicode_FromString(name); + if (nameobj != NULL) { + m = load_builtin(nameobj, type); + Py_DECREF(nameobj); + } else + m = NULL; break; + } case IMP_HOOK: { if (loader == NULL) { @@ -2199,28 +2207,28 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) an exception set if the initialization failed. */ static int -init_builtin(char *name) +init_builtin(PyObject *name) { struct _inittab *p; - if (_PyImport_FindBuiltin(name) != NULL) + if (_PyImport_FindExtensionObject(name, name) != NULL) return 1; for (p = PyImport_Inittab; p->name != NULL; p++) { PyObject *mod; - if (strcmp(name, p->name) == 0) { + if (PyUnicode_CompareWithASCIIString(name, p->name) == 0) { if (p->initfunc == NULL) { PyErr_Format(PyExc_ImportError, - "Cannot re-init internal module %.200s", + "Cannot re-init internal module %R", name); return -1; } if (Py_VerboseFlag) - PySys_WriteStderr("import %s # builtin\n", name); + PySys_FormatStderr("import %U # builtin\n", name); mod = (*p->initfunc)(); if (mod == 0) return -1; - if (_PyImport_FixupBuiltin(mod, name) < 0) + if (_PyImport_FixupExtensionObject(mod, name, name) < 0) return -1; /* FixupExtension has put the module into sys.modules, so we can release our own reference. */ @@ -3286,10 +3294,10 @@ imp_find_module(PyObject *self, PyObject *args) static PyObject * imp_init_builtin(PyObject *self, PyObject *args) { - char *name; + PyObject *name; int ret; PyObject *m; - if (!PyArg_ParseTuple(args, "s:init_builtin", &name)) + if (!PyArg_ParseTuple(args, "U:init_builtin", &name)) return NULL; ret = init_builtin(name); if (ret < 0) @@ -3298,7 +3306,7 @@ imp_init_builtin(PyObject *self, PyObject *args) Py_INCREF(Py_None); return Py_None; } - m = PyImport_AddModule(name); + m = PyImport_AddModuleObject(name); Py_XINCREF(m); return m; } @@ -3346,8 +3354,8 @@ imp_is_frozen_package(PyObject *self, PyObject *args) static PyObject * imp_is_builtin(PyObject *self, PyObject *args) { - char *name; - if (!PyArg_ParseTuple(args, "s:is_builtin", &name)) + PyObject *name; + if (!PyArg_ParseTuple(args, "U:is_builtin", &name)) return NULL; return PyLong_FromLong(is_builtin(name)); } -- cgit v1.2.1 From d08ceb04095b21f699c9221e7e5d7bb2b86a20ea Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 7 Mar 2011 18:28:15 +0100 Subject: Issue #3080: Use PyUnicode_InternFromString() for builtins _PyImport_FixupBuiltin() and _PyImport_FindBuiltin() use PyUnicode_InternFromString() instead of PyUnicode_FromString(). --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 597ef439c3..d43b881c2f 100644 --- a/Python/import.c +++ b/Python/import.c @@ -625,7 +625,7 @@ _PyImport_FixupBuiltin(PyObject *mod, char *name) { int res; PyObject *nameobj; - nameobj = PyUnicode_FromString(name); + nameobj = PyUnicode_InternFromString(name); if (nameobj == NULL) return -1; res = _PyImport_FixupExtensionObject(mod, nameobj, nameobj); @@ -681,7 +681,7 @@ PyObject * _PyImport_FindBuiltin(const char *name) { PyObject *res, *nameobj; - nameobj = PyUnicode_FromString(name); + nameobj = PyUnicode_InternFromString(name); if (nameobj == NULL) return NULL; res = _PyImport_FindExtensionObject(nameobj, nameobj); -- cgit v1.2.1 From 4691aca1214aaa1b4433a667d875415f9a27c38e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 8 Mar 2011 23:49:04 +0100 Subject: Issue #3080: _PyWin_FindRegisteredModule() returns the path as Unicode * Document the function * Use RegQueryValueW() instead of RegQueryValueA() * Use _Py_fopen() instead of fopen() * Allocate registry key on the heap, not on the stack, and handle memory allocation failure * Handle Python exception in find_module() --- Python/import.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index d43b881c2f..8fab810eec 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1591,8 +1591,8 @@ PyImport_GetImporter(PyObject *path) { pathname and an open file. Return NULL if the module is not found. */ #ifdef MS_COREDLL -extern FILE *_PyWin_FindRegisteredModule(const char *, struct filedescr **, - char *, Py_ssize_t); +extern FILE *_PyWin_FindRegisteredModule(PyObject *, struct filedescr **, + PyObject **p_path); #endif static int case_ok(char *, Py_ssize_t, Py_ssize_t, char *); @@ -1679,6 +1679,9 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, } if (path == NULL) { +#ifdef MS_COREDLL + PyObject *filename, *filename_bytes; +#endif nameobj = PyUnicode_FromString(name); if (nameobj == NULL) return NULL; @@ -1687,14 +1690,24 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, strcpy(buf, name); return &fd_builtin; } - Py_DECREF(nameobj); #ifdef MS_COREDLL - fp = _PyWin_FindRegisteredModule(name, &fdp, buf, buflen); + fp = _PyWin_FindRegisteredModule(nameobj, &fdp, &filename); if (fp != NULL) { + Py_DECREF(nameobj); + filename_bytes = PyUnicode_EncodeFSDefault(filename); + Py_DECREF(filename); + if (filename_bytes == NULL) + return NULL; + strncpy(buf, PyBytes_AS_STRING(filename_bytes), buflen); + buf[buflen-1] = '\0'; + Py_DECREF(filename_bytes); *p_fp = fp; return fdp; } + else if (PyErr_Occurred()) + return NULL; #endif + Py_DECREF(nameobj); path = PySys_GetObject("path"); } -- cgit v1.2.1 From 7c1cd6d18d5ac37ec86183e7fabcf35ff11e3d21 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 15:54:07 -0400 Subject: Issue #3080: _PyImport_LoadDynamicModule() uses Unicode for name and path Document also that dynamic module names are ASCII only --- Python/import.c | 54 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 24 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 8fab810eec..4e5cb2ddbf 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2173,9 +2173,21 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) break; #ifdef HAVE_DYNAMIC_LOADING - case C_EXTENSION: - m = _PyImport_LoadDynamicModule(name, pathname, fp); + case C_EXTENSION: { + PyObject *nameobj, *pathobj; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + pathobj = PyUnicode_DecodeFSDefault(pathname); + if (pathobj == NULL) { + Py_DECREF(nameobj); + return NULL; + } + m = _PyImport_LoadDynamicModule(nameobj, pathobj, fp); + Py_DECREF(nameobj); + Py_DECREF(pathobj); break; + } #endif case PKG_DIRECTORY: @@ -2185,11 +2197,10 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) case C_BUILTIN: case PY_FROZEN: { PyObject *nameobj = PyUnicode_FromString(name); - if (nameobj != NULL) { - m = load_builtin(nameobj, type); - Py_DECREF(nameobj); - } else - m = NULL; + if (nameobj == NULL) + return NULL; + m = load_builtin(nameobj, type); + Py_DECREF(nameobj); break; } @@ -3443,28 +3454,23 @@ imp_load_compiled(PyObject *self, PyObject *args) static PyObject * imp_load_dynamic(PyObject *self, PyObject *args) { - char *name; - PyObject *pathbytes; - char *pathname; - PyObject *fob = NULL; - PyObject *m; - FILE *fp = NULL; - if (!PyArg_ParseTuple(args, "sO&|O:load_dynamic", - &name, PyUnicode_FSConverter, &pathbytes, &fob)) + PyObject *name, *pathname, *fob = NULL, *mod; + FILE *fp; + + if (!PyArg_ParseTuple(args, "UO&|O:load_dynamic", + &name, PyUnicode_FSDecoder, &pathname, &fob)) return NULL; - pathname = PyBytes_AS_STRING(pathbytes); - if (fob) { - fp = get_file(pathname, fob, "r"); - if (fp == NULL) { - Py_DECREF(pathbytes); + if (fob != NULL) { + fp = get_file(NULL, fob, "r"); + if (fp == NULL) return NULL; - } } - m = _PyImport_LoadDynamicModule(name, pathname, fp); - Py_DECREF(pathbytes); + else + fp = NULL; + mod = _PyImport_LoadDynamicModule(name, pathname, fp); if (fp) fclose(fp); - return m; + return mod; } #endif /* HAVE_DYNAMIC_LOADING */ -- cgit v1.2.1 From ab8eed250aee0dfe2e0e8cc1472113d5d2b4b9c7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 12 Mar 2011 08:45:02 -0500 Subject: Issue #3080: find_module() initialize buf and *p_fp Document also the find_module() function --- Python/import.c | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 4e5cb2ddbf..e58d6a04be 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1443,7 +1443,7 @@ load_package(char *name, char *pathname) PyObject *path = NULL; int err; char buf[MAXPATHLEN+1]; - FILE *fp = NULL; + FILE *fp; struct filedescr *fdp; m = PyImport_AddModule(name); @@ -1464,7 +1464,6 @@ load_package(char *name, char *pathname) err = PyDict_SetItemString(d, "__path__", path); if (err != 0) goto error; - buf[0] = '\0'; fdp = find_module(name, "__init__", path, buf, sizeof(buf), &fp, NULL); if (fdp == NULL) { if (PyErr_ExceptionMatches(PyExc_ImportError)) { @@ -1599,6 +1598,31 @@ static int case_ok(char *, Py_ssize_t, Py_ssize_t, char *); static int find_init_module(char *); /* Forward */ static struct filedescr importhookdescr = {"", "", IMP_HOOK}; +/* Find a module: + + - try find_module() of each sys.meta_path hook + - try find_frozen() + - try is_builtin() + - try _PyWin_FindRegisteredModule() (Windows only) + - otherwise, call find_module_path_list() with search_path_list (if not + NULL) or sys.path + + Return: + + - &fd_builtin (C_BUILTIN) if it is a builtin + - &fd_frozen (PY_FROZEN) if it is frozen + - &fd_package (PKG_DIRECTORY) and write the filename into *buf + if it is a package + - &importhookdescr (IMP_HOOK) and write the loader into *p_loader if a + importer loader was found + - a file descriptor (PY_SOURCE, PY_COMPILED, C_EXTENSION, PY_RESOURCE or + PY_CODERESOURCE: see _PyImport_Filetab), write the filename into + *buf and the pointer to the open file into *p_fp + - NULL on error + + By default, write an empty string into *buf, and *p_fp and *p_loader (if + set) are set to NULL. Eg. *buf is an empty string for a builtin package. */ + static struct filedescr * find_module(char *fullname, char *subname, PyObject *path, char *buf, size_t buflen, FILE **p_fp, PyObject **p_loader) @@ -1621,6 +1645,8 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, #endif PyObject *fullname_obj, *nameobj; + *buf = '\0'; + *p_fp = NULL; if (p_loader != NULL) *p_loader = NULL; @@ -2962,10 +2988,10 @@ import_submodule(PyObject *mod, char *subname, char *fullname) Py_INCREF(m); } else { - PyObject *path, *loader = NULL; + PyObject *path, *loader; char buf[MAXPATHLEN+1]; struct filedescr *fdp; - FILE *fp = NULL; + FILE *fp; if (mod == Py_None) path = NULL; @@ -2978,7 +3004,6 @@ import_submodule(PyObject *mod, char *subname, char *fullname) } } - buf[0] = '\0'; fdp = find_module(fullname, subname, path, buf, MAXPATHLEN+1, &fp, &loader); Py_XDECREF(path); @@ -3012,11 +3037,11 @@ PyImport_ReloadModule(PyObject *m) PyInterpreterState *interp = PyThreadState_Get()->interp; PyObject *modules_reloading = interp->modules_reloading; PyObject *modules = PyImport_GetModuleDict(); - PyObject *path = NULL, *loader = NULL, *existing_m = NULL; + PyObject *path = NULL, *loader, *existing_m = NULL; char *name, *subname; char buf[MAXPATHLEN+1]; struct filedescr *fdp; - FILE *fp = NULL; + FILE *fp; PyObject *newm; if (modules_reloading == NULL) { @@ -3074,7 +3099,6 @@ PyImport_ReloadModule(PyObject *m) if (path == NULL) PyErr_Clear(); } - buf[0] = '\0'; fdp = find_module(name, subname, path, buf, MAXPATHLEN+1, &fp, &loader); Py_XDECREF(path); @@ -3252,7 +3276,7 @@ call_find_module(char *name, PyObject *path) PyObject *pathobj; struct filedescr *fdp; char pathname[MAXPATHLEN+1]; - FILE *fp = NULL; + FILE *fp; int fd = -1; char *found_encoding = NULL; char *encoding = NULL; -- cgit v1.2.1 From 8d647fdca120414a1901c3b1414b405ad7f415f4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 12 Mar 2011 09:26:54 -0500 Subject: Issue #3080: Remove useless name buffer from find_module() Rename subname argument to name, and mark it as constant. --- Python/import.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index e58d6a04be..37cd4b81f2 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1428,7 +1428,7 @@ get_sourcefile(char *file) /* Forward */ static PyObject *load_module(char *, FILE *, char *, int, PyObject *); -static struct filedescr *find_module(char *, char *, PyObject *, +static struct filedescr *find_module(char *, const char *, PyObject *, char *, size_t, FILE **, PyObject **); static struct _frozen * find_frozen(PyObject *); @@ -1594,7 +1594,7 @@ extern FILE *_PyWin_FindRegisteredModule(PyObject *, struct filedescr **, PyObject **p_path); #endif -static int case_ok(char *, Py_ssize_t, Py_ssize_t, char *); +static int case_ok(char *, Py_ssize_t, Py_ssize_t, const char *); static int find_init_module(char *); /* Forward */ static struct filedescr importhookdescr = {"", "", IMP_HOOK}; @@ -1624,7 +1624,7 @@ static struct filedescr importhookdescr = {"", "", IMP_HOOK}; set) are set to NULL. Eg. *buf is an empty string for a builtin package. */ static struct filedescr * -find_module(char *fullname, char *subname, PyObject *path, char *buf, +find_module(char *fullname, const char *name, PyObject *path, char *buf, size_t buflen, FILE **p_fp, PyObject **p_loader) { Py_ssize_t i, npath; @@ -1637,7 +1637,6 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, static struct filedescr fd_frozen = {"", "", PY_FROZEN}; static struct filedescr fd_builtin = {"", "", C_BUILTIN}; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; - char name[MAXPATHLEN+1]; #if defined(PYOS_OS2) size_t saved_len; size_t saved_namelen; @@ -1650,12 +1649,11 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, if (p_loader != NULL) *p_loader = NULL; - if (strlen(subname) > MAXPATHLEN) { + if (strlen(name) > MAXPATHLEN) { PyErr_SetString(PyExc_OverflowError, "module name is too long"); return NULL; } - strcpy(name, subname); /* sys.meta_path import hook */ if (p_loader != NULL) { @@ -1867,7 +1865,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, * dynamically loaded module we're going to try, * truncate the name before trying */ - if (strlen(subname) > 8) { + if (strlen(name) > 8) { /* is this an attempt to load a C extension? */ const struct filedescr *scan; scan = _PyImport_DynLoadFiletab; @@ -1880,7 +1878,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, if (scan->suffix != NULL) { /* yes, so truncate the name */ namelen = 8; - len -= strlen(subname) - namelen; + len -= strlen(name) - namelen; buf[len] = '\0'; } } @@ -1972,7 +1970,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, #endif static int -case_ok(char *buf, Py_ssize_t len, Py_ssize_t namelen, char *name) +case_ok(char *buf, Py_ssize_t len, Py_ssize_t namelen, const char *name) { /* Pick a platform-specific implementation; the sequence of #if's here should * match the sequence just above. -- cgit v1.2.1 From f44ffd60a466519364cc45d367f3505f1ecbe090 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 20 Mar 2011 01:34:43 +0100 Subject: Issue #3080: Create find_module_path_list() subfunction --- Python/import.c | 310 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 164 insertions(+), 146 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 37cd4b81f2..565f1336cb 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1598,168 +1598,30 @@ static int case_ok(char *, Py_ssize_t, Py_ssize_t, const char *); static int find_init_module(char *); /* Forward */ static struct filedescr importhookdescr = {"", "", IMP_HOOK}; -/* Find a module: - - - try find_module() of each sys.meta_path hook - - try find_frozen() - - try is_builtin() - - try _PyWin_FindRegisteredModule() (Windows only) - - otherwise, call find_module_path_list() with search_path_list (if not - NULL) or sys.path - - Return: - - - &fd_builtin (C_BUILTIN) if it is a builtin - - &fd_frozen (PY_FROZEN) if it is frozen - - &fd_package (PKG_DIRECTORY) and write the filename into *buf - if it is a package - - &importhookdescr (IMP_HOOK) and write the loader into *p_loader if a - importer loader was found - - a file descriptor (PY_SOURCE, PY_COMPILED, C_EXTENSION, PY_RESOURCE or - PY_CODERESOURCE: see _PyImport_Filetab), write the filename into - *buf and the pointer to the open file into *p_fp - - NULL on error - - By default, write an empty string into *buf, and *p_fp and *p_loader (if - set) are set to NULL. Eg. *buf is an empty string for a builtin package. */ - -static struct filedescr * -find_module(char *fullname, const char *name, PyObject *path, char *buf, - size_t buflen, FILE **p_fp, PyObject **p_loader) +static struct filedescr* +find_module_path_list(char *fullname, const char *name, + PyObject *search_path_list, PyObject *path_hooks, + PyObject *path_importer_cache, + char *buf, size_t buflen, + FILE **p_fp, PyObject **p_loader) { Py_ssize_t i, npath; size_t len, namelen; struct filedescr *fdp = NULL; char *filemode; FILE *fp = NULL; - PyObject *path_hooks, *path_importer_cache; struct stat statbuf; - static struct filedescr fd_frozen = {"", "", PY_FROZEN}; - static struct filedescr fd_builtin = {"", "", C_BUILTIN}; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; #if defined(PYOS_OS2) size_t saved_len; size_t saved_namelen; char *saved_buf = NULL; #endif - PyObject *fullname_obj, *nameobj; - *buf = '\0'; - *p_fp = NULL; - if (p_loader != NULL) - *p_loader = NULL; - - if (strlen(name) > MAXPATHLEN) { - PyErr_SetString(PyExc_OverflowError, - "module name is too long"); - return NULL; - } - - /* sys.meta_path import hook */ - if (p_loader != NULL) { - PyObject *meta_path; - - meta_path = PySys_GetObject("meta_path"); - if (meta_path == NULL || !PyList_Check(meta_path)) { - PyErr_SetString(PyExc_ImportError, - "sys.meta_path must be a list of " - "import hooks"); - return NULL; - } - Py_INCREF(meta_path); /* zap guard */ - npath = PyList_Size(meta_path); - for (i = 0; i < npath; i++) { - PyObject *loader; - PyObject *hook = PyList_GetItem(meta_path, i); - loader = PyObject_CallMethod(hook, "find_module", - "sO", fullname, - path != NULL ? - path : Py_None); - if (loader == NULL) { - Py_DECREF(meta_path); - return NULL; /* true error */ - } - if (loader != Py_None) { - /* a loader was found */ - *p_loader = loader; - Py_DECREF(meta_path); - return &importhookdescr; - } - Py_DECREF(loader); - } - Py_DECREF(meta_path); - } - - if (fullname != NULL) { - fullname_obj = PyUnicode_FromString(fullname); - if (fullname == NULL) - return NULL; - if (find_frozen(fullname_obj) != NULL) { - Py_DECREF(fullname_obj); - strcpy(buf, fullname); - return &fd_frozen; - } - Py_DECREF(fullname_obj); - } - - if (path == NULL) { -#ifdef MS_COREDLL - PyObject *filename, *filename_bytes; -#endif - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - if (is_builtin(nameobj)) { - Py_DECREF(nameobj); - strcpy(buf, name); - return &fd_builtin; - } -#ifdef MS_COREDLL - fp = _PyWin_FindRegisteredModule(nameobj, &fdp, &filename); - if (fp != NULL) { - Py_DECREF(nameobj); - filename_bytes = PyUnicode_EncodeFSDefault(filename); - Py_DECREF(filename); - if (filename_bytes == NULL) - return NULL; - strncpy(buf, PyBytes_AS_STRING(filename_bytes), buflen); - buf[buflen-1] = '\0'; - Py_DECREF(filename_bytes); - *p_fp = fp; - return fdp; - } - else if (PyErr_Occurred()) - return NULL; -#endif - Py_DECREF(nameobj); - path = PySys_GetObject("path"); - } - - if (path == NULL || !PyList_Check(path)) { - PyErr_SetString(PyExc_ImportError, - "sys.path must be a list of directory names"); - return NULL; - } - - path_hooks = PySys_GetObject("path_hooks"); - if (path_hooks == NULL || !PyList_Check(path_hooks)) { - PyErr_SetString(PyExc_ImportError, - "sys.path_hooks must be a list of " - "import hooks"); - return NULL; - } - path_importer_cache = PySys_GetObject("path_importer_cache"); - if (path_importer_cache == NULL || - !PyDict_Check(path_importer_cache)) { - PyErr_SetString(PyExc_ImportError, - "sys.path_importer_cache must be a dict"); - return NULL; - } - - npath = PyList_Size(path); + npath = PyList_Size(search_path_list); namelen = strlen(name); for (i = 0; i < npath; i++) { - PyObject *v = PyList_GetItem(path, i); + PyObject *v = PyList_GetItem(search_path_list, i); PyObject *origv = v; const char *base; Py_ssize_t size; @@ -1925,6 +1787,162 @@ find_module(char *fullname, const char *name, PyObject *path, char *buf, return fdp; } +/* Find a module: + + - try find_module() of each sys.meta_path hook + - try find_frozen() + - try is_builtin() + - try _PyWin_FindRegisteredModule() (Windows only) + - otherwise, call find_module_path_list() with search_path_list (if not + NULL) or sys.path + + Return: + + - &fd_builtin (C_BUILTIN) if it is a builtin + - &fd_frozen (PY_FROZEN) if it is frozen + - &fd_package (PKG_DIRECTORY) and write the filename into *buf + if it is a package + - &importhookdescr (IMP_HOOK) and write the loader into *p_loader if a + importer loader was found + - a file descriptor (PY_SOURCE, PY_COMPILED, C_EXTENSION, PY_RESOURCE or + PY_CODERESOURCE: see _PyImport_Filetab), write the filename into + *buf and the pointer to the open file into *p_fp + - NULL on error + + By default, write an empty string into *buf, and *p_fp and *p_loader (if + set) are set to NULL. Eg. *buf is an empty string for a builtin package. */ + +static struct filedescr * +find_module(char *fullname, const char *name, PyObject *search_path_list, + char *buf, size_t buflen, FILE **p_fp, PyObject **p_loader) +{ + Py_ssize_t i, npath; + static struct filedescr fd_frozen = {"", "", PY_FROZEN}; + static struct filedescr fd_builtin = {"", "", C_BUILTIN}; + PyObject *path_hooks, *path_importer_cache; + PyObject *fullname_obj, *nameobj; + + *buf = '\0'; + *p_fp = NULL; + if (p_loader != NULL) + *p_loader = NULL; + + if (strlen(name) > MAXPATHLEN) { + PyErr_SetString(PyExc_OverflowError, + "module name is too long"); + return NULL; + } + + /* sys.meta_path import hook */ + if (p_loader != NULL) { + PyObject *meta_path; + + meta_path = PySys_GetObject("meta_path"); + if (meta_path == NULL || !PyList_Check(meta_path)) { + PyErr_SetString(PyExc_ImportError, + "sys.meta_path must be a list of " + "import hooks"); + return NULL; + } + Py_INCREF(meta_path); /* zap guard */ + npath = PyList_Size(meta_path); + for (i = 0; i < npath; i++) { + PyObject *loader; + PyObject *hook = PyList_GetItem(meta_path, i); + loader = PyObject_CallMethod(hook, "find_module", + "sO", fullname, + search_path_list != NULL ? + search_path_list : Py_None); + if (loader == NULL) { + Py_DECREF(meta_path); + return NULL; /* true error */ + } + if (loader != Py_None) { + /* a loader was found */ + *p_loader = loader; + Py_DECREF(meta_path); + return &importhookdescr; + } + Py_DECREF(loader); + } + Py_DECREF(meta_path); + } + + if (fullname != NULL) { + fullname_obj = PyUnicode_FromString(fullname); + if (fullname == NULL) + return NULL; + if (find_frozen(fullname_obj) != NULL) { + Py_DECREF(fullname_obj); + strcpy(buf, fullname); + return &fd_frozen; + } + Py_DECREF(fullname_obj); + } + + if (search_path_list == NULL) { +#ifdef MS_COREDLL + FILE *fp; + struct filedescr *fdp; + PyObject *filename, *filename_bytes; +#endif + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + if (is_builtin(nameobj)) { + Py_DECREF(nameobj); + strcpy(buf, name); + return &fd_builtin; + } +#ifdef MS_COREDLL + fp = _PyWin_FindRegisteredModule(nameobj, &fdp, &filename); + if (fp != NULL) { + Py_DECREF(nameobj); + filename_bytes = PyUnicode_EncodeFSDefault(filename); + Py_DECREF(filename); + if (filename_bytes == NULL) + return NULL; + strncpy(buf, PyBytes_AS_STRING(filename_bytes), buflen); + buf[buflen-1] = '\0'; + Py_DECREF(filename_bytes); + *p_fp = fp; + return fdp; + } + else if (PyErr_Occurred()) + return NULL; +#endif + Py_DECREF(nameobj); + search_path_list = PySys_GetObject("path"); + } + + if (search_path_list == NULL || !PyList_Check(search_path_list)) { + PyErr_SetString(PyExc_ImportError, + "sys.path must be a list of directory names"); + return NULL; + } + + path_hooks = PySys_GetObject("path_hooks"); + if (path_hooks == NULL || !PyList_Check(path_hooks)) { + PyErr_SetString(PyExc_ImportError, + "sys.path_hooks must be a list of " + "import hooks"); + return NULL; + } + path_importer_cache = PySys_GetObject("path_importer_cache"); + if (path_importer_cache == NULL || + !PyDict_Check(path_importer_cache)) { + PyErr_SetString(PyExc_ImportError, + "sys.path_importer_cache must be a dict"); + return NULL; + } + + return find_module_path_list(fullname, name, + search_path_list, path_hooks, + path_importer_cache, + buf, buflen, + p_fp, p_loader); +} + /* case_ok(char* buf, Py_ssize_t len, Py_ssize_t namelen, char* name) * The arguments here are tricky, best shown by example: * /a/b/c/d/e/f/g/h/i/j/k/some_long_module_name.py\0 -- cgit v1.2.1 From 2d77d7eabcf50177446677639435d797b85722e7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 12 Mar 2011 16:02:28 -0500 Subject: Issue #3080: Create find_module_path() subfunction --- Python/import.c | 228 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 139 insertions(+), 89 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 565f1336cb..59b284654e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1598,6 +1598,129 @@ static int case_ok(char *, Py_ssize_t, Py_ssize_t, const char *); static int find_init_module(char *); /* Forward */ static struct filedescr importhookdescr = {"", "", IMP_HOOK}; +/* Get the path of a module: get its importer and call importer.find_module() + hook, or check if the module if a package (if path/__init__.py exists). + + -1: error: a Python error occurred + 0: ignore: an error occurred because of invalid data, but the error is not + important enough to be reported. + 1: get path: module not found, but *buf contains its path + 2: found: *p_fd is the file descriptor (IMP_HOOK or PKG_DIRECTORY) + and *buf is the path */ + +static int +find_module_path(char *fullname, const char *name, PyObject *path, + PyObject *path_hooks, PyObject *path_importer_cache, + char *buf, size_t buflen, + PyObject **p_loader, struct filedescr **p_fd) +{ + PyObject *path_bytes; + const char *base; + Py_ssize_t len; + size_t namelen; + struct stat statbuf; + static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; + + if (PyUnicode_Check(path)) { + path_bytes = PyUnicode_EncodeFSDefault(path); + if (path_bytes == NULL) + return -1; + } + else if (PyBytes_Check(path)) { + Py_INCREF(path); + path_bytes = path; + } + else + return 0; + + namelen = strlen(name); + base = PyBytes_AS_STRING(path_bytes); + len = PyBytes_GET_SIZE(path_bytes); + if (len + 2 + namelen + MAXSUFFIXSIZE >= buflen) { + Py_DECREF(path_bytes); + return 0; /* Too long */ + } + strcpy(buf, base); + Py_DECREF(path_bytes); + + if (strlen(buf) != len) { + return 0; /* path_bytes contains '\0' */ + } + + /* sys.path_hooks import hook */ + if (p_loader != NULL) { + PyObject *importer; + + importer = get_path_importer(path_importer_cache, + path_hooks, path); + if (importer == NULL) { + return -1; + } + /* Note: importer is a borrowed reference */ + if (importer != Py_None) { + PyObject *loader; + loader = PyObject_CallMethod(importer, + "find_module", + "s", fullname); + if (loader == NULL) + return -1; /* error */ + if (loader != Py_None) { + /* a loader was found */ + *p_loader = loader; + *p_fd = &importhookdescr; + return 2; + } + Py_DECREF(loader); + return 0; + } + } + /* no hook was found, use builtin import */ + + if (len > 0 && buf[len-1] != SEP +#ifdef ALTSEP + && buf[len-1] != ALTSEP +#endif + ) + buf[len++] = SEP; + strcpy(buf+len, name); + len += namelen; + + /* Check for package import (buf holds a directory name, + and there's an __init__ module in that directory */ +#ifdef HAVE_STAT + if (stat(buf, &statbuf) == 0 && /* it exists */ + S_ISDIR(statbuf.st_mode) && /* it's a directory */ + case_ok(buf, len, namelen, name)) { /* case matches */ + if (find_init_module(buf)) { /* and has __init__.py */ + *p_fd = &fd_package; + return 2; + } + else { + int err; + PyObject *unicode = PyUnicode_DecodeFSDefault(buf); + if (unicode == NULL) + return -1; + err = PyErr_WarnFormat(PyExc_ImportWarning, 1, + "Not importing directory '%U': missing __init__.py", + unicode); + Py_DECREF(unicode); + if (err) + return -1; + } + } +#endif + return 1; +} + +/* Find a module in search_path_list. For each path, try + find_module_filename() or try each _PyImport_Filetab suffix. + + If the module is found, return a file descriptor, write the path in + *p_filename, write the pointer to the file object into *p_fp, and (if + p_loader is not NULL) the loader into *p_loader. + + Otherwise, raise an exception and return NULL. */ + static struct filedescr* find_module_path_list(char *fullname, const char *name, PyObject *search_path_list, PyObject *path_hooks, @@ -1610,8 +1733,6 @@ find_module_path_list(char *fullname, const char *name, struct filedescr *fdp = NULL; char *filemode; FILE *fp = NULL; - struct stat statbuf; - static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; #if defined(PYOS_OS2) size_t saved_len; size_t saved_namelen; @@ -1621,96 +1742,25 @@ find_module_path_list(char *fullname, const char *name, npath = PyList_Size(search_path_list); namelen = strlen(name); for (i = 0; i < npath; i++) { - PyObject *v = PyList_GetItem(search_path_list, i); - PyObject *origv = v; - const char *base; - Py_ssize_t size; - if (!v) - return NULL; - if (PyUnicode_Check(v)) { - v = PyUnicode_EncodeFSDefault(v); - if (v == NULL) - return NULL; - } - else if (!PyBytes_Check(v)) - continue; - else - Py_INCREF(v); - - base = PyBytes_AS_STRING(v); - size = PyBytes_GET_SIZE(v); - len = size; - if (len + 2 + namelen + MAXSUFFIXSIZE >= buflen) { - Py_DECREF(v); - continue; /* Too long */ - } - strcpy(buf, base); - Py_DECREF(v); + PyObject *path; + int ok; - if (strlen(buf) != len) { - continue; /* v contains '\0' */ - } - - /* sys.path_hooks import hook */ - if (p_loader != NULL) { - PyObject *importer; - - importer = get_path_importer(path_importer_cache, - path_hooks, origv); - if (importer == NULL) { - return NULL; - } - /* Note: importer is a borrowed reference */ - if (importer != Py_None) { - PyObject *loader; - loader = PyObject_CallMethod(importer, - "find_module", - "s", fullname); - if (loader == NULL) - return NULL; /* error */ - if (loader != Py_None) { - /* a loader was found */ - *p_loader = loader; - return &importhookdescr; - } - Py_DECREF(loader); - continue; - } - } - /* no hook was found, use builtin import */ + path = PyList_GetItem(search_path_list, i); + if (path == NULL) + return NULL; - if (len > 0 && buf[len-1] != SEP -#ifdef ALTSEP - && buf[len-1] != ALTSEP -#endif - ) - buf[len++] = SEP; - strcpy(buf+len, name); - len += namelen; + ok = find_module_path(fullname, name, path, + path_hooks, path_importer_cache, + buf, buflen, + p_loader, &fdp); + if (ok < 0) + return NULL; + if (ok == 0) + continue; + if (ok == 2) + return fdp; - /* Check for package import (buf holds a directory name, - and there's an __init__ module in that directory */ -#ifdef HAVE_STAT - if (stat(buf, &statbuf) == 0 && /* it exists */ - S_ISDIR(statbuf.st_mode) && /* it's a directory */ - case_ok(buf, len, namelen, name)) { /* case matches */ - if (find_init_module(buf)) { /* and has __init__.py */ - return &fd_package; - } - else { - int err; - PyObject *unicode = PyUnicode_DecodeFSDefault(buf); - if (unicode == NULL) - return NULL; - err = PyErr_WarnFormat(PyExc_ImportWarning, 1, - "Not importing directory '%U': missing __init__.py", - unicode); - Py_DECREF(unicode); - if (err) - return NULL; - } - } -#endif + len = strlen(buf); #if defined(PYOS_OS2) /* take a snapshot of the module spec for restoration * after the 8 character DLL hackery -- cgit v1.2.1 From 8509b2931d82b09524d8922b5ffd1871f5ff6311 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 13:33:46 -0400 Subject: Issue #3080: get_sourcefile(), make_source_pathname(), load_package() Use Unicode for module name and path in get_sourcefile(), make_source_pathname() and load_package() functions. --- Python/import.c | 219 +++++++++++++++++++++++++++++++++----------------------- 1 file changed, 131 insertions(+), 88 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 59b284654e..f6cde957a8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -113,6 +113,8 @@ typedef unsigned short mode_t; #define MAGIC (3180 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-32" #define CACHEDIR "__pycache__" +static const Py_UNICODE CACHEDIR_UNICODE[] = { + '_', '_', 'p', 'y', 'c', 'a', 'c', 'h', 'e', '_', '_', '\0'}; /* Current magic word and string tag as globals. */ static long pyc_magic = MAGIC; static const char *pyc_tag = TAG; @@ -741,8 +743,8 @@ remove_module(PyObject *name) "sys.modules failed"); } -static PyObject * get_sourcefile(char *file); -static char *make_source_pathname(char *pathname, char *buf); +static PyObject * get_sourcefile(PyObject *filename); +static PyObject *make_source_pathname(PyObject *pathname); static char *make_compiled_pathname(char *pathname, char *buf, size_t buflen, int debug); @@ -807,7 +809,6 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, { PyObject *modules = PyImport_GetModuleDict(); PyObject *m, *d, *v; - PyObject *pathbytes; m = PyImport_AddModuleObject(name); if (m == NULL) @@ -822,12 +823,7 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, } /* Remember the filename as the __file__ attribute */ if (pathname != NULL) { - pathbytes = PyUnicode_EncodeFSDefault(pathname); - if (pathbytes != NULL) { - v = get_sourcefile(PyBytes_AS_STRING(pathbytes)); - Py_DECREF(pathbytes); - } else - v = NULL; + v = get_sourcefile(pathname); if (v == NULL) PyErr_Clear(); } @@ -892,6 +888,27 @@ rightmost_sep(char *s) } +/* Like strrchr(string, '/') but searches for the rightmost of either SEP + or ALTSEP, if the latter is defined. +*/ +static Py_UNICODE* +rightmost_sep_unicode(Py_UNICODE *s) +{ + Py_UNICODE *found, c; + for (found = NULL; (c = *s); s++) { + if (c == SEP +#ifdef ALTSEP + || c == ALTSEP +#endif + ) + { + found = s; + } + } + return found; +} + + /* Given a pathname for a Python source file, fill a buffer with the pathname for the corresponding compiled file. Return the pathname for the compiled file, or NULL if there's no space in the buffer. @@ -1005,42 +1022,50 @@ make_compiled_pathname(char *pathname, char *buf, size_t buflen, int debug) source file, if the path matches the PEP 3147 format. This does not check for any file existence, however, if the pyc file name does not match PEP 3147 style, NULL is returned. buf must be at least as big as pathname; - the resulting path will always be shorter. */ + the resulting path will always be shorter. -static char * -make_source_pathname(char *pathname, char *buf) + (...)/__pycache__/foo..pyc -> (...)/foo.py */ + +static PyObject* +make_source_pathname(PyObject *pathobj) { - /* __pycache__/foo..pyc -> foo.py */ + Py_UNICODE buf[MAXPATHLEN]; + Py_UNICODE *pathname; + Py_UNICODE *left, *right, *dot0, *dot1, sep; size_t i, j; - char *left, *right, *dot0, *dot1, sep; + + if (PyUnicode_GET_SIZE(pathobj) > MAXPATHLEN) + return NULL; + pathname = PyUnicode_AS_UNICODE(pathobj); /* Look back two slashes from the end. In between these two slashes must be the string __pycache__ or this is not a PEP 3147 style path. It's possible for there to be only one slash. */ - if ((right = rightmost_sep(pathname)) == NULL) + right = rightmost_sep_unicode(pathname); + if (right == NULL) return NULL; sep = *right; *right = '\0'; - left = rightmost_sep(pathname); + left = rightmost_sep_unicode(pathname); *right = sep; if (left == NULL) left = pathname; else left++; - if (right-left != strlen(CACHEDIR) || - strncmp(left, CACHEDIR, right-left) != 0) + if (right-left != Py_UNICODE_strlen(CACHEDIR_UNICODE) || + Py_UNICODE_strncmp(left, CACHEDIR_UNICODE, right-left) != 0) return NULL; /* Now verify that the path component to the right of the last slash has two dots in it. */ - if ((dot0 = strchr(right + 1, '.')) == NULL) + if ((dot0 = Py_UNICODE_strchr(right + 1, '.')) == NULL) return NULL; - if ((dot1 = strchr(dot0 + 1, '.')) == NULL) + if ((dot1 = Py_UNICODE_strchr(dot0 + 1, '.')) == NULL) return NULL; /* Too many dots? */ - if (strchr(dot1 + 1, '.') != NULL) + if (Py_UNICODE_strchr(dot1 + 1, '.') != NULL) return NULL; /* This is a PEP 3147 path. Start by copying everything from the @@ -1048,10 +1073,11 @@ make_source_pathname(char *pathname, char *buf) copy the file's basename, removing the magic tag and adding a .py suffix. */ - strncpy(buf, pathname, (i=left-pathname)); - strncpy(buf+i, right+1, (j=dot0-right)); - strcpy(buf+i+j, "py"); - return buf; + Py_UNICODE_strncpy(buf, pathname, (i=left-pathname)); + Py_UNICODE_strncpy(buf+i, right+1, (j=dot0-right)); + buf[i+j] = 'p'; + buf[i+j+1] = 'y'; + return PyUnicode_FromUnicode(buf, i+j+2); } /* Given a pathname for a Python source file, its time of last @@ -1390,40 +1416,47 @@ load_source_module(char *name, char *pathname, FILE *fp) * Returns the path to the py file if available, else the given path */ static PyObject * -get_sourcefile(char *file) +get_sourcefile(PyObject *filename) { - char py[MAXPATHLEN + 1]; Py_ssize_t len; - PyObject *u; + Py_UNICODE *fileuni; + PyObject *py; struct stat statbuf; - if (!file || !*file) { + len = PyUnicode_GET_SIZE(filename); + if (len == 0) Py_RETURN_NONE; - } - len = strlen(file); - /* match '*.py?' */ - if (len > MAXPATHLEN || PyOS_strnicmp(&file[len-4], ".py", 3) != 0) { - return PyUnicode_DecodeFSDefault(file); - } + /* don't match *.pyc or *.pyo? */ + fileuni = PyUnicode_AS_UNICODE(filename); + if (len < 5 + || fileuni[len-4] != '.' + || (fileuni[len-3] != 'p' && fileuni[len-3] != 'P') + || (fileuni[len-2] != 'y' && fileuni[len-2] != 'Y')) + goto unchanged; /* Start by trying to turn PEP 3147 path into source path. If that * fails, just chop off the trailing character, i.e. legacy pyc path * to py. */ - if (make_source_pathname(file, py) == NULL) { - strncpy(py, file, len-1); - py[len-1] = '\0'; + py = make_source_pathname(filename); + if (py == NULL) { + PyErr_Clear(); + py = PyUnicode_FromUnicode(fileuni, len - 1); } + if (py == NULL) + goto error; - if (stat(py, &statbuf) == 0 && - S_ISREG(statbuf.st_mode)) { - u = PyUnicode_DecodeFSDefault(py); - } - else { - u = PyUnicode_DecodeFSDefault(file); - } - return u; + if (_Py_stat(py, &statbuf) == 0 && S_ISREG(statbuf.st_mode)) + return py; + Py_DECREF(py); + goto unchanged; + +error: + PyErr_Clear(); +unchanged: + Py_INCREF(filename); + return filename; } /* Forward */ @@ -1436,54 +1469,56 @@ static struct _frozen * find_frozen(PyObject *); REFERENCE COUNT */ static PyObject * -load_package(char *name, char *pathname) +load_package(PyObject *name, PyObject *pathname) { PyObject *m, *d; - PyObject *file = NULL; - PyObject *path = NULL; + PyObject *file = NULL, *path_list = NULL; int err; char buf[MAXPATHLEN+1]; - FILE *fp; + FILE *fp = NULL; struct filedescr *fdp; + char *namestr; - m = PyImport_AddModule(name); + m = PyImport_AddModuleObject(name); if (m == NULL) return NULL; if (Py_VerboseFlag) - PySys_WriteStderr("import %s # directory %s\n", + PySys_FormatStderr("import %U # directory %U\n", name, pathname); - d = PyModule_GetDict(m); file = get_sourcefile(pathname); if (file == NULL) - goto error; - path = Py_BuildValue("[O]", file); - if (path == NULL) - goto error; + return NULL; + path_list = Py_BuildValue("[O]", file); + if (path_list == NULL) { + Py_DECREF(file); + return NULL; + } + d = PyModule_GetDict(m); err = PyDict_SetItemString(d, "__file__", file); + Py_DECREF(file); if (err == 0) - err = PyDict_SetItemString(d, "__path__", path); - if (err != 0) + err = PyDict_SetItemString(d, "__path__", path_list); + if (err != 0) { + Py_DECREF(path_list); + return NULL; + } + namestr = _PyUnicode_AsString(name); + if (namestr == NULL) goto error; - fdp = find_module(name, "__init__", path, buf, sizeof(buf), &fp, NULL); + fdp = find_module(namestr, "__init__", path_list, buf, sizeof(buf), &fp, NULL); + Py_DECREF(path_list); if (fdp == NULL) { if (PyErr_ExceptionMatches(PyExc_ImportError)) { PyErr_Clear(); Py_INCREF(m); + return m; } else - m = NULL; - goto cleanup; + return NULL; } - m = load_module(name, fp, buf, fdp->type, NULL); + m = load_module(namestr, fp, buf, fdp->type, NULL); if (fp != NULL) fclose(fp); - goto cleanup; - - error: - m = NULL; - cleanup: - Py_XDECREF(path); - Py_XDECREF(file); return m; } @@ -2282,9 +2317,21 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) } #endif - case PKG_DIRECTORY: - m = load_package(name, pathname); + case PKG_DIRECTORY: { + PyObject *nameobj, *pathobj; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + pathobj = PyUnicode_DecodeFSDefault(pathname); + if (pathobj == NULL) { + Py_DECREF(nameobj); + return NULL; + } + m = load_package(nameobj, pathobj); + Py_DECREF(nameobj); + Py_DECREF(pathobj); break; + } case C_BUILTIN: case PY_FROZEN: { @@ -3637,13 +3684,12 @@ imp_load_module(PyObject *self, PyObject *args) static PyObject * imp_load_package(PyObject *self, PyObject *args) { - char *name; - PyObject *pathname; + PyObject *name, *pathname; PyObject * ret; - if (!PyArg_ParseTuple(args, "sO&:load_package", - &name, PyUnicode_FSConverter, &pathname)) + if (!PyArg_ParseTuple(args, "UO&:load_package", + &name, PyUnicode_FSDecoder, &pathname)) return NULL; - ret = load_package(name, PyBytes_AS_STRING(pathname)); + ret = load_package(name, pathname); Py_DECREF(pathname); return ret; } @@ -3716,25 +3762,22 @@ static PyObject * imp_source_from_cache(PyObject *self, PyObject *args, PyObject *kws) { static char *kwlist[] = {"path", NULL}; - - PyObject *pathname_obj; - char *pathname; - char buf[MAXPATHLEN+1]; + PyObject *pathname, *source; if (!PyArg_ParseTupleAndKeywords( args, kws, "O&", kwlist, - PyUnicode_FSConverter, &pathname_obj)) + PyUnicode_FSDecoder, &pathname)) return NULL; - pathname = PyBytes_AS_STRING(pathname_obj); - if (make_source_pathname(pathname, buf) == NULL) { - PyErr_Format(PyExc_ValueError, "Not a PEP 3147 pyc path: %s", + source = make_source_pathname(pathname); + if (source == NULL) { + PyErr_Format(PyExc_ValueError, "Not a PEP 3147 pyc path: %R", pathname); - Py_DECREF(pathname_obj); + Py_DECREF(pathname); return NULL; } - Py_DECREF(pathname_obj); - return PyUnicode_FromString(buf); + Py_DECREF(pathname); + return source; } PyDoc_STRVAR(doc_source_from_cache, -- cgit v1.2.1 From 26e6d2c66ccad73bad2edeb441f54141164d0bc1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 20 Mar 2011 00:41:24 +0100 Subject: Issue #3080: Use Unicode to import source and compiled modules * Use Unicode for module name and path in the following functions: * get_file() * load_source_module(), parse_source_module() * make_compiled_pathname(), check_compiled_module(), read_compiled_module(), load_compiled_module(), write_compiled_module(), update_compiled_module() * On Windows, use CreateDirectoryW() instead of mkdir() * update_compiled_module() cannot fail anymore --- Python/import.c | 352 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 201 insertions(+), 151 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index f6cde957a8..0efffe9149 100644 --- a/Python/import.c +++ b/Python/import.c @@ -107,8 +107,8 @@ typedef unsigned short mode_t; /* MAGIC must change whenever the bytecode emitted by the compiler may no longer be understood by older implementations of the eval loop (usually due to the addition of new opcodes) - TAG must change for each major Python release. The magic number will take - care of any bytecode changes that occur during development. + TAG and PYC_TAG_UNICODE must change for each major Python release. The magic + number will take care of any bytecode changes that occur during development. */ #define MAGIC (3180 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-32" @@ -118,6 +118,8 @@ static const Py_UNICODE CACHEDIR_UNICODE[] = { /* Current magic word and string tag as globals. */ static long pyc_magic = MAGIC; static const char *pyc_tag = TAG; +static const Py_UNICODE PYC_TAG_UNICODE[] = { + 'c', 'p', 'y', 't', 'h', 'o', 'n', '-', '3', '2', '\0'}; /* See _PyImport_FixupExtensionObject() below */ static PyObject *extensions = NULL; @@ -745,8 +747,7 @@ remove_module(PyObject *name) static PyObject * get_sourcefile(PyObject *filename); static PyObject *make_source_pathname(PyObject *pathname); -static char *make_compiled_pathname(char *pathname, char *buf, size_t buflen, - int debug); +static PyObject* make_compiled_pathname(Py_UNICODE *pathname, int debug); /* Execute a code object in a module and return the module object * WITH INCREMENTED REFERENCE COUNT. If an error occurs, name is @@ -867,32 +868,11 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, } -/* Like strrchr(string, '/') but searches for the rightmost of either SEP - or ALTSEP, if the latter is defined. -*/ -static char * -rightmost_sep(char *s) -{ - char *found, c; - for (found = NULL; (c = *s); s++) { - if (c == SEP -#ifdef ALTSEP - || c == ALTSEP -#endif - ) - { - found = s; - } - } - return found; -} - - /* Like strrchr(string, '/') but searches for the rightmost of either SEP or ALTSEP, if the latter is defined. */ static Py_UNICODE* -rightmost_sep_unicode(Py_UNICODE *s) +rightmost_sep(Py_UNICODE *s) { Py_UNICODE *found, c; for (found = NULL; (c = *s); s++) { @@ -912,15 +892,18 @@ rightmost_sep_unicode(Py_UNICODE *s) /* Given a pathname for a Python source file, fill a buffer with the pathname for the corresponding compiled file. Return the pathname for the compiled file, or NULL if there's no space in the buffer. - Doesn't set an exception. */ + Doesn't set an exception. + + foo.py -> __pycache__/foo..pyc */ -static char * -make_compiled_pathname(char *pathname, char *buf, size_t buflen, int debug) +static PyObject* +make_compiled_pathname(Py_UNICODE *pathname, int debug) { - /* foo.py -> __pycache__/foo..pyc */ - size_t len = strlen(pathname); + Py_UNICODE buf[MAXPATHLEN]; + size_t buflen = (size_t)MAXPATHLEN; + size_t len = Py_UNICODE_strlen(pathname); size_t i, save; - char *pos; + Py_UNICODE *pos; int sep = SEP; /* Sanity check that the buffer has roughly enough space to hold what @@ -932,35 +915,37 @@ make_compiled_pathname(char *pathname, char *buf, size_t buflen, int debug) sanity check before writing the extension to ensure we do not overflow the buffer. */ - if (len + strlen(CACHEDIR) + strlen(pyc_tag) + 5 > buflen) + if (len + Py_UNICODE_strlen(CACHEDIR_UNICODE) + Py_UNICODE_strlen(PYC_TAG_UNICODE) + 5 > buflen) return NULL; /* Find the last path separator and copy everything from the start of the source string up to and including the separator. */ - if ((pos = rightmost_sep(pathname)) == NULL) { + pos = rightmost_sep(pathname); + if (pos == NULL) { i = 0; } else { sep = *pos; i = pos - pathname + 1; - strncpy(buf, pathname, i); + Py_UNICODE_strncpy(buf, pathname, i); } save = i; buf[i++] = '\0'; /* Add __pycache__/ */ - strcat(buf, CACHEDIR); - i += strlen(CACHEDIR) - 1; + Py_UNICODE_strcat(buf, CACHEDIR_UNICODE); + i += Py_UNICODE_strlen(CACHEDIR_UNICODE) - 1; buf[i++] = sep; buf[i++] = '\0'; /* Add the base filename, but remove the .py or .pyw extension, since the tag name must go before the extension. */ - strcat(buf, pathname + save); - if ((pos = strrchr(buf, '.')) != NULL) + Py_UNICODE_strcat(buf, pathname + save); + pos = Py_UNICODE_strrchr(buf, '.'); + if (pos != NULL) *++pos = '\0'; - strcat(buf, pyc_tag); + Py_UNICODE_strcat(buf, PYC_TAG_UNICODE); /* The length test above assumes that we're only adding one character to the end of what would normally be the extension. What if there is no extension, or the string ends in '.' or '.p', and otherwise @@ -1010,11 +995,15 @@ make_compiled_pathname(char *pathname, char *buf, size_t buflen, int debug) #if 0 printf("strlen(buf): %d; buflen: %d\n", (int)strlen(buf), (int)buflen); #endif - if (strlen(buf) + 5 > buflen) + len = Py_UNICODE_strlen(buf); + if (len + 5 > buflen) return NULL; - strcat(buf, debug ? ".pyc" : ".pyo"); - assert(strlen(buf) < buflen); - return buf; + buf[len] = '.'; len++; + buf[len] = 'p'; len++; + buf[len] = 'y'; len++; + buf[len] = debug ? 'c' : 'o'; len++; + assert(len <= buflen); + return PyUnicode_FromUnicode(buf, len); } @@ -1042,12 +1031,12 @@ make_source_pathname(PyObject *pathobj) must be the string __pycache__ or this is not a PEP 3147 style path. It's possible for there to be only one slash. */ - right = rightmost_sep_unicode(pathname); + right = rightmost_sep(pathname); if (right == NULL) return NULL; sep = *right; *right = '\0'; - left = rightmost_sep_unicode(pathname); + left = rightmost_sep(pathname); *right = sep; if (left == NULL) left = pathname; @@ -1088,31 +1077,31 @@ make_source_pathname(PyObject *pathobj) Doesn't set an exception. */ static FILE * -check_compiled_module(char *pathname, time_t mtime, char *cpathname) +check_compiled_module(PyObject *pathname, time_t mtime, PyObject *cpathname) { FILE *fp; long magic; long pyc_mtime; - fp = fopen(cpathname, "rb"); + fp = _Py_fopen(cpathname, "rb"); if (fp == NULL) return NULL; magic = PyMarshal_ReadLongFromFile(fp); if (magic != pyc_magic) { if (Py_VerboseFlag) - PySys_WriteStderr("# %s has bad magic\n", cpathname); + PySys_FormatStderr("# %R has bad magic\n", cpathname); fclose(fp); return NULL; } pyc_mtime = PyMarshal_ReadLongFromFile(fp); if (pyc_mtime != mtime) { if (Py_VerboseFlag) - PySys_WriteStderr("# %s has bad mtime\n", cpathname); + PySys_FormatStderr("# %R has bad mtime\n", cpathname); fclose(fp); return NULL; } if (Py_VerboseFlag) - PySys_WriteStderr("# %s matches %s\n", cpathname, pathname); + PySys_FormatStderr("# %R matches %R\n", cpathname, pathname); return fp; } @@ -1120,7 +1109,7 @@ check_compiled_module(char *pathname, time_t mtime, char *cpathname) /* Read a code object from a file and check it for validity */ static PyCodeObject * -read_compiled_module(char *cpathname, FILE *fp) +read_compiled_module(PyObject *cpathname, FILE *fp) { PyObject *co; @@ -1129,7 +1118,7 @@ read_compiled_module(char *cpathname, FILE *fp) return NULL; if (!PyCode_Check(co)) { PyErr_Format(PyExc_ImportError, - "Non-code object in %.200s", cpathname); + "Non-code object in %R", cpathname); Py_DECREF(co); return NULL; } @@ -1141,7 +1130,7 @@ read_compiled_module(char *cpathname, FILE *fp) module object WITH INCREMENTED REFERENCE COUNT */ static PyObject * -load_compiled_module(char *name, char *cpathname, FILE *fp) +load_compiled_module(PyObject *name, PyObject *cpathname, FILE *fp) { long magic; PyCodeObject *co; @@ -1150,7 +1139,7 @@ load_compiled_module(char *name, char *cpathname, FILE *fp) magic = PyMarshal_ReadLongFromFile(fp); if (magic != pyc_magic) { PyErr_Format(PyExc_ImportError, - "Bad magic number in %.200s", cpathname); + "Bad magic number in %R", cpathname); return NULL; } (void) PyMarshal_ReadLongFromFile(fp); @@ -1158,10 +1147,10 @@ load_compiled_module(char *name, char *cpathname, FILE *fp) if (co == NULL) return NULL; if (Py_VerboseFlag) - PySys_WriteStderr("import %s # precompiled from %s\n", - name, cpathname); - m = PyImport_ExecCodeModuleWithPathnames( - name, (PyObject *)co, cpathname, cpathname); + PySys_FormatStderr("import %U # precompiled from %R\n", + name, cpathname); + m = PyImport_ExecCodeModuleObject(name, (PyObject *)co, + cpathname, cpathname); Py_DECREF(co); return m; @@ -1170,27 +1159,37 @@ load_compiled_module(char *name, char *cpathname, FILE *fp) /* Parse a source file and return the corresponding code object */ static PyCodeObject * -parse_source_module(const char *pathname, FILE *fp) +parse_source_module(PyObject *pathname, FILE *fp) { - PyCodeObject *co = NULL; + PyCodeObject *co; + PyObject *pathbytes; mod_ty mod; PyCompilerFlags flags; - PyArena *arena = PyArena_New(); - if (arena == NULL) + PyArena *arena; + + pathbytes = PyUnicode_EncodeFSDefault(pathname); + if (pathbytes == NULL) + return NULL; + + arena = PyArena_New(); + if (arena == NULL) { + Py_DECREF(pathbytes); return NULL; + } flags.cf_flags = 0; - mod = PyParser_ASTFromFile(fp, pathname, NULL, + mod = PyParser_ASTFromFile(fp, PyBytes_AS_STRING(pathbytes), NULL, Py_file_input, 0, 0, &flags, NULL, arena); - if (mod) { - co = PyAST_Compile(mod, pathname, NULL, arena); - } + if (mod != NULL) + co = PyAST_Compile(mod, PyBytes_AS_STRING(pathbytes), NULL, arena); + else + co = NULL; + Py_DECREF(pathbytes); PyArena_Free(arena); return co; } - /* Helper to open a bytecode file for writing in exclusive mode */ static FILE * @@ -1231,10 +1230,10 @@ open_exclusive(char *filename, mode_t mode) remove the file. */ static void -write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat) +write_compiled_module(PyCodeObject *co, PyObject *cpathname, + struct stat *srcstat) { FILE *fp; - char *dirpath; time_t mtime = srcstat->st_mtime; #ifdef MS_WINDOWS /* since Windows uses different permissions */ mode_t mode = srcstat->st_mode & ~S_IEXEC; @@ -1243,39 +1242,64 @@ write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat) mode_t dirmode = (srcstat->st_mode | S_IXUSR | S_IXGRP | S_IXOTH | S_IWUSR | S_IWGRP | S_IWOTH); + PyObject *dirbytes; #endif - int saved; + PyObject *cpathbytes, *dirname; + Py_UNICODE *dirsep; + int res, ok; /* Ensure that the __pycache__ directory exists. */ - dirpath = rightmost_sep(cpathname); - if (dirpath == NULL) { + dirsep = rightmost_sep(PyUnicode_AS_UNICODE(cpathname)); + if (dirsep == NULL) { if (Py_VerboseFlag) - PySys_WriteStderr( - "# no %s path found %s\n", - CACHEDIR, cpathname); + PySys_FormatStderr("# no %s path found %R\n", CACHEDIR, cpathname); + return; + } + dirname = PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(cpathname), + dirsep - PyUnicode_AS_UNICODE(cpathname)); + if (dirname == NULL) { + PyErr_Clear(); return; } - saved = *dirpath; - *dirpath = '\0'; #ifdef MS_WINDOWS - if (_mkdir(cpathname) < 0 && errno != EEXIST) { + res = CreateDirectoryW(PyUnicode_AS_UNICODE(dirname), NULL); + ok = (res != 0); + if (!ok && GetLastError() == ERROR_ALREADY_EXISTS) + ok = 1; #else - if (mkdir(cpathname, dirmode) < 0 && errno != EEXIST) { + dirbytes = PyUnicode_EncodeFSDefault(dirname); + if (dirbytes == NULL) { + PyErr_Clear(); + return; + } + res = mkdir(PyBytes_AS_STRING(dirbytes), dirmode); + Py_DECREF(dirbytes); + if (0 <= res) + ok = 1; + else + ok = (errno == EEXIST); #endif - *dirpath = saved; + if (!ok) { if (Py_VerboseFlag) - PySys_WriteStderr( - "# cannot create cache dir %s\n", cpathname); + PySys_FormatStderr("# cannot create cache dir %R\n", dirname); + Py_DECREF(dirname); + return; + } + Py_DECREF(dirname); + + cpathbytes = PyUnicode_EncodeFSDefault(cpathname); + if (cpathbytes == NULL) { + PyErr_Clear(); return; } - *dirpath = saved; - fp = open_exclusive(cpathname, mode); + fp = open_exclusive(PyBytes_AS_STRING(cpathbytes), mode); if (fp == NULL) { if (Py_VerboseFlag) - PySys_WriteStderr( - "# can't create %s\n", cpathname); + PySys_FormatStderr( + "# can't create %R\n", cpathname); + Py_DECREF(cpathbytes); return; } PyMarshal_WriteLongToFile(pyc_magic, fp, Py_MARSHAL_VERSION); @@ -1284,12 +1308,18 @@ write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat) PyMarshal_WriteObjectToFile((PyObject *)co, fp, Py_MARSHAL_VERSION); if (fflush(fp) != 0 || ferror(fp)) { if (Py_VerboseFlag) - PySys_WriteStderr("# can't write %s\n", cpathname); + PySys_FormatStderr("# can't write %R\n", cpathname); /* Don't keep partial file */ fclose(fp); - (void) unlink(cpathname); +#ifdef MS_WINDOWS + (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname)); +#else + (void) unlink(PyBytes_AS_STRING(cpathbytes)); +#endif + Py_DECREF(cpathbytes); return; } + Py_DECREF(cpathbytes); /* Now write the true mtime */ fseek(fp, 4L, 0); assert(mtime < LONG_MAX); @@ -1297,7 +1327,7 @@ write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat) fflush(fp); fclose(fp); if (Py_VerboseFlag) - PySys_WriteStderr("# wrote %s\n", cpathname); + PySys_FormatStderr("# wrote %R\n", cpathname); } static void @@ -1324,26 +1354,18 @@ update_code_filenames(PyCodeObject *co, PyObject *oldname, PyObject *newname) } } -static int -update_compiled_module(PyCodeObject *co, char *pathname) +static void +update_compiled_module(PyCodeObject *co, PyObject *newname) { - PyObject *oldname, *newname; + PyObject *oldname; - newname = PyUnicode_DecodeFSDefault(pathname); - if (newname == NULL) - return -1; - - if (!PyUnicode_Compare(co->co_filename, newname)) { - Py_DECREF(newname); - return 0; - } + if (PyUnicode_Compare(co->co_filename, newname) == 0) + return; oldname = co->co_filename; Py_INCREF(oldname); update_code_filenames(co, oldname, newname); Py_DECREF(oldname); - Py_DECREF(newname); - return 1; } /* Load a source module from a given file and return its module @@ -1351,20 +1373,19 @@ update_compiled_module(PyCodeObject *co, char *pathname) byte-compiled file, use that instead. */ static PyObject * -load_source_module(char *name, char *pathname, FILE *fp) +load_source_module(PyObject *name, PyObject *pathname, FILE *fp) { struct stat st; FILE *fpc; - char buf[MAXPATHLEN+1]; - char *cpathname; + PyObject *cpathname = NULL, *cpathbytes = NULL; PyCodeObject *co; - PyObject *m; + PyObject *m = NULL; if (fstat(fileno(fp), &st) != 0) { PyErr_Format(PyExc_RuntimeError, - "unable to get file status from '%s'", + "unable to get file status from %R", pathname); - return NULL; + goto error; } #if SIZEOF_TIME_T > 4 /* Python's .pyc timestamp handling presumes that the timestamp fits @@ -1374,41 +1395,50 @@ load_source_module(char *name, char *pathname, FILE *fp) if (st.st_mtime >> 32) { PyErr_SetString(PyExc_OverflowError, "modification time overflows a 4 byte field"); - return NULL; + goto error; } #endif cpathname = make_compiled_pathname( - pathname, buf, (size_t)MAXPATHLEN + 1, !Py_OptimizeFlag); - if (cpathname != NULL && - (fpc = check_compiled_module(pathname, st.st_mtime, cpathname))) { + PyUnicode_AS_UNICODE(pathname), + !Py_OptimizeFlag); + + if (cpathname != NULL) + fpc = check_compiled_module(pathname, st.st_mtime, cpathname); + else + fpc = NULL; + + if (fpc) { co = read_compiled_module(cpathname, fpc); fclose(fpc); if (co == NULL) - return NULL; - if (update_compiled_module(co, pathname) < 0) - return NULL; + goto error; + update_compiled_module(co, pathname); if (Py_VerboseFlag) - PySys_WriteStderr("import %s # precompiled from %s\n", - name, cpathname); - pathname = cpathname; + PySys_FormatStderr("import %U # precompiled from %R\n", + name, cpathname); + m = PyImport_ExecCodeModuleObject(name, (PyObject *)co, + cpathname, cpathname); } else { co = parse_source_module(pathname, fp); if (co == NULL) - return NULL; + goto error; if (Py_VerboseFlag) - PySys_WriteStderr("import %s # from %s\n", + PySys_FormatStderr("import %U # from %R\n", name, pathname); - if (cpathname) { + if (cpathname != NULL) { PyObject *ro = PySys_GetObject("dont_write_bytecode"); if (ro == NULL || !PyObject_IsTrue(ro)) write_compiled_module(co, cpathname, &st); } + m = PyImport_ExecCodeModuleObject(name, (PyObject *)co, + pathname, cpathname); } - m = PyImport_ExecCodeModuleWithPathnames( - name, (PyObject *)co, pathname, cpathname); Py_DECREF(co); +error: + Py_XDECREF(cpathbytes); + Py_XDECREF(cpathname); return m; } @@ -2291,13 +2321,37 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) switch (type) { - case PY_SOURCE: - m = load_source_module(name, pathname, fp); + case PY_SOURCE: { + PyObject *nameobj, *pathobj; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + pathobj = PyUnicode_DecodeFSDefault(pathname); + if (pathobj == NULL) { + Py_DECREF(nameobj); + return NULL; + } + m = load_source_module(nameobj, pathobj, fp); + Py_DECREF(nameobj); + Py_DECREF(pathobj); break; + } - case PY_COMPILED: - m = load_compiled_module(name, pathname, fp); + case PY_COMPILED: { + PyObject *nameobj, *pathobj; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + pathobj = PyUnicode_DecodeFSDefault(pathname); + if (pathobj == NULL) { + Py_DECREF(nameobj); + return NULL; + } + m = load_compiled_module(nameobj, pathobj, fp); + Py_DECREF(nameobj); + Py_DECREF(pathobj); break; + } #ifdef HAVE_DYNAMIC_LOADING case C_EXTENSION: { @@ -3533,13 +3587,13 @@ imp_is_frozen(PyObject *self, PyObject *args) } static FILE * -get_file(char *pathname, PyObject *fob, char *mode) +get_file(PyObject *pathname, PyObject *fob, char *mode) { FILE *fp; if (mode[0] == 'U') mode = "r" PY_STDIOTEXTMODE; if (fob == NULL) { - fp = fopen(pathname, mode); + fp = _Py_fopen(pathname, mode); } else { int fd = PyObject_AsFileDescriptor(fob); @@ -3565,22 +3619,21 @@ error: static PyObject * imp_load_compiled(PyObject *self, PyObject *args) { - char *name; - PyObject *pathname; + PyObject *name, *pathname; PyObject *fob = NULL; PyObject *m; FILE *fp; - if (!PyArg_ParseTuple(args, "sO&|O:load_compiled", + if (!PyArg_ParseTuple(args, "UO&|O:load_compiled", &name, - PyUnicode_FSConverter, &pathname, + PyUnicode_FSDecoder, &pathname, &fob)) return NULL; - fp = get_file(PyBytes_AS_STRING(pathname), fob, "rb"); + fp = get_file(pathname, fob, "rb"); if (fp == NULL) { Py_DECREF(pathname); return NULL; } - m = load_compiled_module(name, PyBytes_AS_STRING(pathname), fp); + m = load_compiled_module(name, pathname, fp); fclose(fp); Py_DECREF(pathname); return m; @@ -3615,22 +3668,21 @@ imp_load_dynamic(PyObject *self, PyObject *args) static PyObject * imp_load_source(PyObject *self, PyObject *args) { - char *name; - PyObject *pathname; + PyObject *name, *pathname; PyObject *fob = NULL; PyObject *m; FILE *fp; - if (!PyArg_ParseTuple(args, "sO&|O:load_source", + if (!PyArg_ParseTuple(args, "UO&|O:load_source", &name, - PyUnicode_FSConverter, &pathname, + PyUnicode_FSDecoder, &pathname, &fob)) return NULL; - fp = get_file(PyBytes_AS_STRING(pathname), fob, "r"); + fp = get_file(pathname, fob, "r"); if (fp == NULL) { Py_DECREF(pathname); return NULL; } - m = load_source_module(name, PyBytes_AS_STRING(pathname), fp); + m = load_source_module(name, pathname, fp); Py_DECREF(pathname); fclose(fp); return m; @@ -3719,33 +3771,31 @@ imp_cache_from_source(PyObject *self, PyObject *args, PyObject *kws) { static char *kwlist[] = {"path", "debug_override", NULL}; - char buf[MAXPATHLEN+1]; - PyObject *pathbytes; - char *cpathname; + PyObject *pathname, *cpathname; PyObject *debug_override = NULL; int debug = !Py_OptimizeFlag; if (!PyArg_ParseTupleAndKeywords( args, kws, "O&|O", kwlist, - PyUnicode_FSConverter, &pathbytes, &debug_override)) + PyUnicode_FSDecoder, &pathname, &debug_override)) return NULL; if (debug_override != NULL && (debug = PyObject_IsTrue(debug_override)) < 0) { - Py_DECREF(pathbytes); + Py_DECREF(pathname); return NULL; } cpathname = make_compiled_pathname( - PyBytes_AS_STRING(pathbytes), - buf, MAXPATHLEN+1, debug); - Py_DECREF(pathbytes); + PyUnicode_AS_UNICODE(pathname), + debug); + Py_DECREF(pathname); if (cpathname == NULL) { PyErr_Format(PyExc_SystemError, "path buffer too short"); return NULL; } - return PyUnicode_DecodeFSDefault(buf); + return cpathname; } PyDoc_STRVAR(doc_cache_from_source, -- cgit v1.2.1 From 1bd9d3e0561cad30f87017b5ee00be6f7c92ced8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 13 Mar 2011 21:46:30 -0400 Subject: Issue #3080: load_module() expects name and path as Unicode --- Python/import.c | 136 +++++++++++++++++++++++--------------------------------- 1 file changed, 55 insertions(+), 81 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 0efffe9149..00a9eccaf8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1490,7 +1490,7 @@ unchanged: } /* Forward */ -static PyObject *load_module(char *, FILE *, char *, int, PyObject *); +static PyObject *load_module(PyObject *, FILE *, PyObject *, int, PyObject *); static struct filedescr *find_module(char *, const char *, PyObject *, char *, size_t, FILE **, PyObject **); static struct _frozen * find_frozen(PyObject *); @@ -1501,7 +1501,7 @@ static struct _frozen * find_frozen(PyObject *); static PyObject * load_package(PyObject *name, PyObject *pathname) { - PyObject *m, *d; + PyObject *m, *d, *bufobj; PyObject *file = NULL, *path_list = NULL; int err; char buf[MAXPATHLEN+1]; @@ -1546,7 +1546,13 @@ load_package(PyObject *name, PyObject *pathname) else return NULL; } - m = load_module(namestr, fp, buf, fdp->type, NULL); + bufobj = PyUnicode_DecodeFSDefault(buf); + if (bufobj != NULL) { + m = load_module(name, fp, bufobj, fdp->type, NULL); + Py_DECREF(bufobj); + } + else + m = NULL; if (fp != NULL) fclose(fp); return m; @@ -2303,7 +2309,7 @@ load_builtin(PyObject *name, int type) its module object WITH INCREMENTED REFERENCE COUNT */ static PyObject * -load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) +load_module(PyObject *name, FILE *fp, PyObject *pathname, int type, PyObject *loader) { PyObject *m; @@ -2313,7 +2319,7 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) case PY_COMPILED: if (fp == NULL) { PyErr_Format(PyExc_ValueError, - "file object required for import (type code %d)", + "file object required for import (type code %d)", type); return NULL; } @@ -2321,81 +2327,28 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) switch (type) { - case PY_SOURCE: { - PyObject *nameobj, *pathobj; - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - pathobj = PyUnicode_DecodeFSDefault(pathname); - if (pathobj == NULL) { - Py_DECREF(nameobj); - return NULL; - } - m = load_source_module(nameobj, pathobj, fp); - Py_DECREF(nameobj); - Py_DECREF(pathobj); + case PY_SOURCE: + m = load_source_module(name, pathname, fp); break; - } - case PY_COMPILED: { - PyObject *nameobj, *pathobj; - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - pathobj = PyUnicode_DecodeFSDefault(pathname); - if (pathobj == NULL) { - Py_DECREF(nameobj); - return NULL; - } - m = load_compiled_module(nameobj, pathobj, fp); - Py_DECREF(nameobj); - Py_DECREF(pathobj); + case PY_COMPILED: + m = load_compiled_module(name, pathname, fp); break; - } #ifdef HAVE_DYNAMIC_LOADING - case C_EXTENSION: { - PyObject *nameobj, *pathobj; - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - pathobj = PyUnicode_DecodeFSDefault(pathname); - if (pathobj == NULL) { - Py_DECREF(nameobj); - return NULL; - } - m = _PyImport_LoadDynamicModule(nameobj, pathobj, fp); - Py_DECREF(nameobj); - Py_DECREF(pathobj); + case C_EXTENSION: + m = _PyImport_LoadDynamicModule(name, pathname, fp); break; - } #endif - case PKG_DIRECTORY: { - PyObject *nameobj, *pathobj; - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - pathobj = PyUnicode_DecodeFSDefault(pathname); - if (pathobj == NULL) { - Py_DECREF(nameobj); - return NULL; - } - m = load_package(nameobj, pathobj); - Py_DECREF(nameobj); - Py_DECREF(pathobj); + case PKG_DIRECTORY: + m = load_package(name, pathname); break; - } case C_BUILTIN: - case PY_FROZEN: { - PyObject *nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - m = load_builtin(nameobj, type); - Py_DECREF(nameobj); + case PY_FROZEN: + m = load_builtin(name, type); break; - } case IMP_HOOK: { if (loader == NULL) { @@ -2403,13 +2356,13 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) "import hook without loader"); return NULL; } - m = PyObject_CallMethod(loader, "load_module", "s", name); + m = PyObject_CallMethod(loader, "load_module", "O", name); break; } default: PyErr_Format(PyExc_ImportError, - "Don't know how to import %.200s (type code %d)", + "Don't know how to import %U (type code %d)", name, type); m = NULL; @@ -3144,7 +3097,7 @@ static PyObject * import_submodule(PyObject *mod, char *subname, char *fullname) { PyObject *modules = PyImport_GetModuleDict(); - PyObject *m = NULL; + PyObject *m = NULL, *fullnameobj, *bufobj; /* Require: if mod == None: subname == fullname @@ -3181,7 +3134,19 @@ import_submodule(PyObject *mod, char *subname, char *fullname) Py_INCREF(Py_None); return Py_None; } - m = load_module(fullname, fp, buf, fdp->type, loader); + fullnameobj = PyUnicode_FromString(fullname); + if (fullnameobj != NULL) { + bufobj = PyUnicode_DecodeFSDefault(buf); + if (bufobj != NULL) { + m = load_module(fullnameobj, fp, bufobj, fdp->type, loader); + Py_DECREF(bufobj); + } + else + m = NULL; + Py_DECREF(fullnameobj); + } + else + m = NULL; Py_XDECREF(loader); if (fp) fclose(fp); @@ -3209,7 +3174,7 @@ PyImport_ReloadModule(PyObject *m) char buf[MAXPATHLEN+1]; struct filedescr *fdp; FILE *fp; - PyObject *newm; + PyObject *newm, *nameobj, *bufobj; if (modules_reloading == NULL) { Py_FatalError("PyImport_ReloadModule: " @@ -3275,7 +3240,19 @@ PyImport_ReloadModule(PyObject *m) return NULL; } - newm = load_module(name, fp, buf, fdp->type, loader); + nameobj = PyUnicode_FromString(name); + if (nameobj != NULL) { + bufobj = PyUnicode_DecodeFSDefault(buf); + if (bufobj != NULL) { + newm = load_module(nameobj, fp, bufobj, fdp->type, loader); + Py_DECREF(bufobj); + } + else + newm = NULL; + Py_DECREF(nameobj); + } + else + newm = NULL; Py_XDECREF(loader); if (fp) @@ -3691,18 +3668,15 @@ imp_load_source(PyObject *self, PyObject *args) static PyObject * imp_load_module(PyObject *self, PyObject *args) { - char *name; - PyObject *fob; - PyObject *pathname; - PyObject * ret; + PyObject *name, *fob, *pathname, *ret; char *suffix; /* Unused */ char *mode; int type; FILE *fp; - if (!PyArg_ParseTuple(args, "sOO&(ssi):load_module", + if (!PyArg_ParseTuple(args, "UOO&(ssi):load_module", &name, &fob, - PyUnicode_FSConverter, &pathname, + PyUnicode_FSDecoder, &pathname, &suffix, &mode, &type)) return NULL; if (*mode) { @@ -3726,7 +3700,7 @@ imp_load_module(PyObject *self, PyObject *args) return NULL; } } - ret = load_module(name, fp, PyBytes_AS_STRING(pathname), type, NULL); + ret = load_module(name, fp, pathname, type, NULL); Py_DECREF(pathname); if (fp) fclose(fp); -- cgit v1.2.1 From d5668295d2303564770b7dbdfbb02e7d007292bc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 13 Mar 2011 21:57:27 -0400 Subject: Issue #3080: PyImport_ImportModuleNoBlock() uses Unicode --- Python/import.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 00a9eccaf8..08237aff95 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2591,8 +2591,7 @@ PyImport_ImportModule(const char *name) PyObject * PyImport_ImportModuleNoBlock(const char *name) { - PyObject *result; - PyObject *modules; + PyObject *nameobj, *modules, *result; long me; /* Try to get the module from sys.modules[name] */ @@ -2600,14 +2599,16 @@ PyImport_ImportModuleNoBlock(const char *name) if (modules == NULL) return NULL; - result = PyDict_GetItemString(modules, name); + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + result = PyDict_GetItem(modules, nameobj); if (result != NULL) { + Py_DECREF(nameobj); Py_INCREF(result); return result; } - else { - PyErr_Clear(); - } + PyErr_Clear(); #ifdef WITH_THREAD /* check the import lock * me might be -1 but I ignore the error here, the lock function @@ -2615,18 +2616,20 @@ PyImport_ImportModuleNoBlock(const char *name) me = PyThread_get_thread_ident(); if (import_lock_thread == -1 || import_lock_thread == me) { /* no thread or me is holding the lock */ - return PyImport_ImportModule(name); + result = PyImport_Import(nameobj); } else { PyErr_Format(PyExc_ImportError, - "Failed to import %.200s because the import lock" + "Failed to import %U because the import lock" "is held by another thread.", - name); - return NULL; + nameobj); + result = NULL; } #else - return PyImport_ImportModule(name); + result = PyImport_Import(nameobj); #endif + Py_DECREF(nameobj); + return result; } /* Forward declarations for helper routines */ -- cgit v1.2.1 From 93d6b25bf514369f9325ae1dd062a7ba9f1c1097 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Mar 2011 09:33:57 +0100 Subject: Issue #3080: Use Unicode for the "The Magnum Opus of dotted-name import" Use Unicode for module name and paths in the following functions: * PyImport_ImportModuleLevel() * add_submodule() * ensure_from_list() * get_parent() * import_module_level() * import_submodule() * load_next() * mark_miss() --- Python/import.c | 381 +++++++++++++++++++++++++++++++++----------------------- 1 file changed, 222 insertions(+), 159 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 08237aff95..95faeee5d8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2633,28 +2633,37 @@ PyImport_ImportModuleNoBlock(const char *name) } /* Forward declarations for helper routines */ -static PyObject *get_parent(PyObject *globals, char *buf, - Py_ssize_t *p_buflen, int level); +static PyObject *get_parent(PyObject *globals, + PyObject **p_name, + int level); static PyObject *load_next(PyObject *mod, PyObject *altmod, - char **p_name, char *buf, Py_ssize_t *p_buflen); -static int mark_miss(char *name); + PyObject *inputname, PyObject **p_outputname, + Py_UNICODE *buf, Py_ssize_t *p_buflen, + Py_ssize_t bufsize); +static int mark_miss(PyObject *name); static int ensure_fromlist(PyObject *mod, PyObject *fromlist, - char *buf, Py_ssize_t buflen, int recursive); -static PyObject * import_submodule(PyObject *mod, char *name, char *fullname); + PyObject *buf, int recursive); +static PyObject * import_submodule(PyObject *mod, PyObject *name, + PyObject *fullname); /* The Magnum Opus of dotted-name import :-) */ static PyObject * -import_module_level(char *name, PyObject *globals, PyObject *locals, +import_module_level(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) { - char buf[MAXPATHLEN+1]; - Py_ssize_t buflen = 0; - PyObject *parent, *head, *next, *tail; + Py_UNICODE buf[MAXPATHLEN+1]; + Py_ssize_t buflen; + Py_ssize_t bufsize = MAXPATHLEN+1; + PyObject *parent, *head, *next, *tail, *inputname, *outputname; + PyObject *parent_name, *ensure_name; + const Py_UNICODE *nameunicode; - if (strchr(name, '/') != NULL -#ifdef MS_WINDOWS - || strchr(name, '\\') != NULL + nameunicode = PyUnicode_AS_UNICODE(name); + + if (Py_UNICODE_strchr(nameunicode, SEP) != NULL +#ifdef ALTSEP + || Py_UNICODE_strchr(nameunicode, ALTSEP) != NULL #endif ) { PyErr_SetString(PyExc_ImportError, @@ -2662,25 +2671,45 @@ import_module_level(char *name, PyObject *globals, PyObject *locals, return NULL; } - parent = get_parent(globals, buf, &buflen, level); + parent = get_parent(globals, &parent_name, level); if (parent == NULL) return NULL; - head = load_next(parent, level < 0 ? Py_None : parent, &name, buf, - &buflen); + buflen = PyUnicode_GET_SIZE(parent_name); + if (buflen+1 > bufsize) { + Py_DECREF(parent_name); + PyErr_SetString(PyExc_ValueError, + "Module name too long"); + return NULL; + } + Py_UNICODE_strcpy(buf, PyUnicode_AS_UNICODE(parent_name)); + Py_DECREF(parent_name); + + head = load_next(parent, level < 0 ? Py_None : parent, name, &outputname, + buf, &buflen, bufsize); if (head == NULL) return NULL; tail = head; Py_INCREF(tail); - while (name) { - next = load_next(tail, tail, &name, buf, &buflen); - Py_DECREF(tail); - if (next == NULL) { - Py_DECREF(head); - return NULL; + + if (outputname != NULL) { + while (1) { + inputname = outputname; + next = load_next(tail, tail, inputname, &outputname, + buf, &buflen, bufsize); + Py_DECREF(tail); + Py_DECREF(inputname); + if (next == NULL) { + Py_DECREF(head); + return NULL; + } + tail = next; + + if (outputname == NULL) { + break; + } } - tail = next; } if (tail == Py_None) { /* If tail is Py_None, both get_parent and load_next found @@ -2688,8 +2717,7 @@ import_module_level(char *name, PyObject *globals, PyObject *locals, doctored faulty bytecode */ Py_DECREF(tail); Py_DECREF(head); - PyErr_SetString(PyExc_ValueError, - "Empty module name"); + PyErr_SetString(PyExc_ValueError, "Empty module name"); return NULL; } @@ -2704,21 +2732,33 @@ import_module_level(char *name, PyObject *globals, PyObject *locals, } Py_DECREF(head); - if (!ensure_fromlist(tail, fromlist, buf, buflen, 0)) { + + ensure_name = PyUnicode_FromUnicode(buf, Py_UNICODE_strlen(buf)); + if (ensure_name == NULL) { + Py_DECREF(tail); + return NULL; + } + if (!ensure_fromlist(tail, fromlist, ensure_name, 0)) { Py_DECREF(tail); + Py_DECREF(ensure_name); return NULL; } + Py_DECREF(ensure_name); return tail; } PyObject * PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, - PyObject *fromlist, int level) + PyObject *fromlist, int level) { - PyObject *result; + PyObject *nameobj, *result; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; _PyImport_AcquireLock(); - result = import_module_level(name, globals, locals, fromlist, level); + result = import_module_level(nameobj, globals, locals, fromlist, level); + Py_DECREF(nameobj); if (_PyImport_ReleaseLock() < 0) { Py_XDECREF(result); PyErr_SetString(PyExc_RuntimeError, @@ -2733,15 +2773,20 @@ PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, sys.modules entry for foo.bar. If globals is from a package's __init__.py, the package's entry in sys.modules is returned, as a borrowed reference. - The *name* of the returned package is returned in buf, with the length of - the name in *p_buflen. + The name of the returned package is returned in *p_name. If globals doesn't come from a package or a module in a package, or a corresponding entry is not found in sys.modules, Py_None is returned. */ static PyObject * -get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) +get_parent(PyObject *globals, + PyObject **p_name, + int level) { + Py_UNICODE name[MAXPATHLEN+1]; + const Py_ssize_t bufsize = MAXPATHLEN+1; + PyObject *nameobj; + static PyObject *namestr = NULL; static PyObject *pathstr = NULL; static PyObject *pkgstr = NULL; @@ -2749,7 +2794,7 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) int orig_level = level; if (globals == NULL || !PyDict_Check(globals) || !level) - return Py_None; + goto return_none; if (namestr == NULL) { namestr = PyUnicode_InternFromString("__name__"); @@ -2767,55 +2812,46 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) return NULL; } - *buf = '\0'; - *p_buflen = 0; pkgname = PyDict_GetItem(globals, pkgstr); if ((pkgname != NULL) && (pkgname != Py_None)) { /* __package__ is set, so use it */ - char *pkgname_str; - Py_ssize_t len; - if (!PyUnicode_Check(pkgname)) { PyErr_SetString(PyExc_ValueError, "__package__ set to non-string"); return NULL; } - pkgname_str = _PyUnicode_AsStringAndSize(pkgname, &len); - if (len == 0) { + if (PyUnicode_GET_SIZE(pkgname) == 0) { if (level > 0) { PyErr_SetString(PyExc_ValueError, "Attempted relative import in non-package"); return NULL; } - return Py_None; + goto return_none; } - if (len > MAXPATHLEN) { + if (PyUnicode_GET_SIZE(pkgname)+1 > bufsize) { PyErr_SetString(PyExc_ValueError, "Package name too long"); return NULL; } - strcpy(buf, pkgname_str); + Py_UNICODE_strcpy(name, PyUnicode_AS_UNICODE(pkgname)); } else { /* __package__ not set, so figure it out and set it */ modname = PyDict_GetItem(globals, namestr); if (modname == NULL || !PyUnicode_Check(modname)) - return Py_None; + goto return_none; modpath = PyDict_GetItem(globals, pathstr); if (modpath != NULL) { /* __path__ is set, so modname is already the package name */ - char *modname_str; - Py_ssize_t len; int error; - modname_str = _PyUnicode_AsStringAndSize(modname, &len); - if (len > MAXPATHLEN) { + if (PyUnicode_GET_SIZE(modname)+1 > bufsize) { PyErr_SetString(PyExc_ValueError, "Module name too long"); return NULL; } - strcpy(buf, modname_str); + Py_UNICODE_strcpy(name, PyUnicode_AS_UNICODE(modname)); error = PyDict_SetItem(globals, pkgstr, modname); if (error) { PyErr_SetString(PyExc_ValueError, @@ -2824,9 +2860,9 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) } } else { /* Normal module, so work out the package name if any */ - char *start = _PyUnicode_AsString(modname); - char *lastdot = strrchr(start, '.'); - size_t len; + Py_UNICODE *start = PyUnicode_AS_UNICODE(modname); + Py_UNICODE *lastdot = Py_UNICODE_strrchr(start, '.'); + Py_ssize_t len; int error; if (lastdot == NULL && level > 0) { PyErr_SetString(PyExc_ValueError, @@ -2840,17 +2876,17 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) "Could not set __package__"); return NULL; } - return Py_None; + goto return_none; } len = lastdot - start; - if (len >= MAXPATHLEN) { + if (len+1 > bufsize) { PyErr_SetString(PyExc_ValueError, "Module name too long"); return NULL; } - strncpy(buf, start, len); - buf[len] = '\0'; - pkgname = PyUnicode_FromString(buf); + Py_UNICODE_strncpy(name, start, len); + name[len] = '\0'; + pkgname = PyUnicode_FromUnicode(name, len); if (pkgname == NULL) { return NULL; } @@ -2864,7 +2900,7 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) } } while (--level > 0) { - char *dot = strrchr(buf, '.'); + Py_UNICODE *dot = Py_UNICODE_strrchr(name, '.'); if (dot == NULL) { PyErr_SetString(PyExc_ValueError, "Attempted relative import beyond " @@ -2873,137 +2909,181 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) } *dot = '\0'; } - *p_buflen = strlen(buf); + + nameobj = PyUnicode_FromUnicode(name, Py_UNICODE_strlen(name)); + if (nameobj == NULL) + return NULL; modules = PyImport_GetModuleDict(); - parent = PyDict_GetItemString(modules, buf); + parent = PyDict_GetItem(modules, nameobj); if (parent == NULL) { - if (orig_level < 1) { - PyObject *err_msg = PyBytes_FromFormat( - "Parent module '%.200s' not found " - "while handling absolute import", buf); - if (err_msg == NULL) { - return NULL; - } - if (!PyErr_WarnEx(PyExc_RuntimeWarning, - PyBytes_AsString(err_msg), 1)) { - *buf = '\0'; - *p_buflen = 0; - parent = Py_None; - } - Py_DECREF(err_msg); - } else { + int err; + + if (orig_level >= 1) { PyErr_Format(PyExc_SystemError, - "Parent module '%.200s' not loaded, " - "cannot perform relative import", buf); + "Parent module %R not loaded, " + "cannot perform relative import", nameobj); + Py_DECREF(nameobj); + return NULL; } + + err = PyErr_WarnFormat( + PyExc_RuntimeWarning, 1, + "Parent module %R not found while handling absolute import", + nameobj); + Py_DECREF(nameobj); + if (err) + return NULL; + + goto return_none; } + *p_name = nameobj; return parent; /* We expect, but can't guarantee, if parent != None, that: - - parent.__name__ == buf + - parent.__name__ == name - parent.__dict__ is globals If this is violated... Who cares? */ + +return_none: + nameobj = PyUnicode_FromUnicode(NULL, 0); + if (nameobj == NULL) + return NULL; + *p_name = nameobj; + return Py_None; } /* altmod is either None or same as mod */ static PyObject * -load_next(PyObject *mod, PyObject *altmod, char **p_name, char *buf, - Py_ssize_t *p_buflen) +load_next(PyObject *mod, PyObject *altmod, + PyObject *inputname, PyObject **p_outputname, + Py_UNICODE *buf, Py_ssize_t *p_buflen, Py_ssize_t bufsize) { - char *name = *p_name; - char *dot = strchr(name, '.'); - size_t len; - char *p; - PyObject *result; + const Py_UNICODE *dot; + Py_ssize_t len; + Py_UNICODE *p; + PyObject *fullname, *name, *result, *mark_name; + const Py_UNICODE *nameuni; + + *p_outputname = NULL; - if (strlen(name) == 0) { + if (PyUnicode_GET_SIZE(inputname) == 0) { /* completely empty module name should only happen in 'from . import' (or '__import__("")')*/ Py_INCREF(mod); - *p_name = NULL; return mod; } - if (dot == NULL) { - *p_name = NULL; - len = strlen(name); - } - else { - *p_name = dot+1; - len = dot-name; + nameuni = PyUnicode_AS_UNICODE(inputname); + if (nameuni == NULL) + return NULL; + + dot = Py_UNICODE_strchr(nameuni, '.'); + if (dot != NULL) { + len = dot - nameuni; + if (len == 0) { + PyErr_SetString(PyExc_ValueError, + "Empty module name"); + return NULL; + } } - if (len == 0) { + else + len = PyUnicode_GET_SIZE(inputname); + + if (*p_buflen+len+1 >= bufsize) { PyErr_SetString(PyExc_ValueError, - "Empty module name"); + "Module name too long"); return NULL; } p = buf + *p_buflen; - if (p != buf) + if (p != buf) { *p++ = '.'; - if (p+len-buf >= MAXPATHLEN) { - PyErr_SetString(PyExc_ValueError, - "Module name too long"); - return NULL; + *p_buflen += 1; } - strncpy(p, name, len); + Py_UNICODE_strncpy(p, nameuni, len); p[len] = '\0'; - *p_buflen = p+len-buf; + *p_buflen += len; - result = import_submodule(mod, p, buf); + fullname = PyUnicode_FromUnicode(buf, *p_buflen); + if (fullname == NULL) + return NULL; + name = PyUnicode_FromUnicode(p, len); + if (name == NULL) { + Py_DECREF(fullname); + return NULL; + } + result = import_submodule(mod, name, fullname); + Py_DECREF(fullname); if (result == Py_None && altmod != mod) { Py_DECREF(result); /* Here, altmod must be None and mod must not be None */ - result = import_submodule(altmod, p, p); + result = import_submodule(altmod, name, name); + Py_DECREF(name); if (result != NULL && result != Py_None) { - if (mark_miss(buf) != 0) { + mark_name = PyUnicode_FromUnicode(buf, *p_buflen); + if (mark_name == NULL) { + Py_DECREF(result); + return NULL; + } + if (mark_miss(mark_name) != 0) { Py_DECREF(result); + Py_DECREF(mark_name); return NULL; } - strncpy(buf, name, len); + Py_DECREF(mark_name); + Py_UNICODE_strncpy(buf, nameuni, len); buf[len] = '\0'; *p_buflen = len; } } + else + Py_DECREF(name); if (result == NULL) return NULL; if (result == Py_None) { Py_DECREF(result); PyErr_Format(PyExc_ImportError, - "No module named %.200s", name); + "No module named %R", inputname); return NULL; } + if (dot != NULL) { + *p_outputname = PyUnicode_FromUnicode(dot+1, Py_UNICODE_strlen(dot+1)); + if (*p_outputname == NULL) { + Py_DECREF(result); + return NULL; + } + } + return result; } static int -mark_miss(char *name) +mark_miss(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); - return PyDict_SetItemString(modules, name, Py_None); + return PyDict_SetItem(modules, name, Py_None); } static int -ensure_fromlist(PyObject *mod, PyObject *fromlist, char *buf, Py_ssize_t buflen, +ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *name, int recursive) { int i; + PyObject *fullname; + Py_ssize_t fromlist_len; if (!PyObject_HasAttrString(mod, "__path__")) return 1; - for (i = 0; ; i++) { + fromlist_len = PySequence_Size(fromlist); + + for (i = 0; i < fromlist_len; i++) { PyObject *item = PySequence_GetItem(fromlist, i); int hasit; - if (item == NULL) { - if (PyErr_ExceptionMatches(PyExc_IndexError)) { - PyErr_Clear(); - return 1; - } + if (item == NULL) return 0; - } if (!PyUnicode_Check(item)) { PyErr_SetString(PyExc_TypeError, "Item in ``from list'' not a string"); @@ -3020,7 +3100,7 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, char *buf, Py_ssize_t buflen, if (all == NULL) PyErr_Clear(); else { - int ret = ensure_fromlist(mod, all, buf, buflen, 1); + int ret = ensure_fromlist(mod, all, name, 1); Py_DECREF(all); if (!ret) return 0; @@ -3029,27 +3109,14 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, char *buf, Py_ssize_t buflen, } hasit = PyObject_HasAttr(mod, item); if (!hasit) { - PyObject *item8; - char *subname; PyObject *submod; - char *p; - item8 = PyUnicode_EncodeFSDefault(item); - if (!item8) { - PyErr_SetString(PyExc_ValueError, "Cannot encode path item"); - return 0; + fullname = PyUnicode_FromFormat("%U.%U", name, item); + if (fullname != NULL) { + submod = import_submodule(mod, item, fullname); + Py_DECREF(fullname); } - subname = PyBytes_AS_STRING(item8); - if (buflen + strlen(subname) >= MAXPATHLEN) { - PyErr_SetString(PyExc_ValueError, - "Module name too long"); - Py_DECREF(item); - return 0; - } - p = buf + buflen; - *p++ = '.'; - strcpy(p, subname); - submod = import_submodule(mod, subname, buf); - Py_DECREF(item8); + else + submod = NULL; Py_XDECREF(submod); if (submod == NULL) { Py_DECREF(item); @@ -3059,12 +3126,12 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, char *buf, Py_ssize_t buflen, Py_DECREF(item); } - /* NOTREACHED */ + return 1; } static int -add_submodule(PyObject *mod, PyObject *submod, char *fullname, char *subname, - PyObject *modules) +add_submodule(PyObject *mod, PyObject *submod, PyObject *fullname, + PyObject *subname, PyObject *modules) { if (mod == Py_None) return 1; @@ -3075,7 +3142,7 @@ add_submodule(PyObject *mod, PyObject *submod, char *fullname, char *subname, load failed with a SyntaxError -- then there's no trace in sys.modules. In that case, of course, do nothing extra.) */ if (submod == NULL) { - submod = PyDict_GetItemString(modules, fullname); + submod = PyDict_GetItem(modules, fullname); if (submod == NULL) return 1; } @@ -3086,28 +3153,28 @@ add_submodule(PyObject *mod, PyObject *submod, char *fullname, char *subname, PyObject *dict = PyModule_GetDict(mod); if (!dict) return 0; - if (PyDict_SetItemString(dict, subname, submod) < 0) + if (PyDict_SetItem(dict, subname, submod) < 0) return 0; } else { - if (PyObject_SetAttrString(mod, subname, submod) < 0) + if (PyObject_SetAttr(mod, subname, submod) < 0) return 0; } return 1; } static PyObject * -import_submodule(PyObject *mod, char *subname, char *fullname) +import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) { PyObject *modules = PyImport_GetModuleDict(); - PyObject *m = NULL, *fullnameobj, *bufobj; + PyObject *m = NULL, *bufobj; /* Require: if mod == None: subname == fullname else: mod.__name__ + "." + subname == fullname */ - if ((m = PyDict_GetItemString(modules, fullname)) != NULL) { + if ((m = PyDict_GetItem(modules, fullname)) != NULL) { Py_INCREF(m); } else { @@ -3127,7 +3194,9 @@ import_submodule(PyObject *mod, char *subname, char *fullname) } } - fdp = find_module(fullname, subname, path, buf, MAXPATHLEN+1, + fdp = find_module(_PyUnicode_AsString(fullname), + _PyUnicode_AsString(subname), + path, buf, MAXPATHLEN+1, &fp, &loader); Py_XDECREF(path); if (fdp == NULL) { @@ -3137,23 +3206,17 @@ import_submodule(PyObject *mod, char *subname, char *fullname) Py_INCREF(Py_None); return Py_None; } - fullnameobj = PyUnicode_FromString(fullname); - if (fullnameobj != NULL) { - bufobj = PyUnicode_DecodeFSDefault(buf); - if (bufobj != NULL) { - m = load_module(fullnameobj, fp, bufobj, fdp->type, loader); - Py_DECREF(bufobj); - } - else - m = NULL; - Py_DECREF(fullnameobj); + bufobj = PyUnicode_DecodeFSDefault(buf); + if (bufobj != NULL) { + m = load_module(fullname, fp, bufobj, fdp->type, loader); + Py_DECREF(bufobj); } else m = NULL; Py_XDECREF(loader); if (fp) fclose(fp); - if (!add_submodule(mod, m, fullname, subname, modules)) { + if (m != NULL && !add_submodule(mod, m, fullname, subname, modules)) { Py_XDECREF(m); m = NULL; } -- cgit v1.2.1 From 74c6f0ba2d358a8d6717772150740b5f49142711 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 13 Mar 2011 22:38:06 -0400 Subject: Issue #3080: Use %R to format module name in error messages %R format instead of %U --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 95faeee5d8..90ba4b5e22 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2620,7 +2620,7 @@ PyImport_ImportModuleNoBlock(const char *name) } else { PyErr_Format(PyExc_ImportError, - "Failed to import %U because the import lock" + "Failed to import %R because the import lock" "is held by another thread.", nameobj); result = NULL; @@ -3285,7 +3285,7 @@ PyImport_ReloadModule(PyObject *m) parent = PyDict_GetItem(modules, parentname); if (parent == NULL) { PyErr_Format(PyExc_ImportError, - "reload(): parent %U not in sys.modules", + "reload(): parent %R not in sys.modules", parentname); Py_DECREF(parentname); imp_modules_reloading_clear(); -- cgit v1.2.1 From ec2c770704d651436f9217bf78ef289f63162209 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 13 Mar 2011 22:38:38 -0400 Subject: Issue #3080: Reindent and simplify import_submodule() --- Python/import.c | 81 +++++++++++++++++++++++++++------------------------------ 1 file changed, 39 insertions(+), 42 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 90ba4b5e22..cce2de1b44 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2779,9 +2779,7 @@ PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, corresponding entry is not found in sys.modules, Py_None is returned. */ static PyObject * -get_parent(PyObject *globals, - PyObject **p_name, - int level) +get_parent(PyObject *globals, PyObject **p_name, int level) { Py_UNICODE name[MAXPATHLEN+1]; const Py_ssize_t bufsize = MAXPATHLEN+1; @@ -3167,7 +3165,10 @@ static PyObject * import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) { PyObject *modules = PyImport_GetModuleDict(); - PyObject *m = NULL, *bufobj; + PyObject *m = NULL, *bufobj, *path, *loader; + char buf[MAXPATHLEN+1]; + struct filedescr *fdp; + FILE *fp; /* Require: if mod == None: subname == fullname @@ -3176,52 +3177,48 @@ import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) if ((m = PyDict_GetItem(modules, fullname)) != NULL) { Py_INCREF(m); + return m; } - else { - PyObject *path, *loader; - char buf[MAXPATHLEN+1]; - struct filedescr *fdp; - FILE *fp; - - if (mod == Py_None) - path = NULL; - else { - path = PyObject_GetAttrString(mod, "__path__"); - if (path == NULL) { - PyErr_Clear(); - Py_INCREF(Py_None); - return Py_None; - } - } - fdp = find_module(_PyUnicode_AsString(fullname), - _PyUnicode_AsString(subname), - path, buf, MAXPATHLEN+1, - &fp, &loader); - Py_XDECREF(path); - if (fdp == NULL) { - if (!PyErr_ExceptionMatches(PyExc_ImportError)) - return NULL; + if (mod == Py_None) + path = NULL; + else { + path = PyObject_GetAttrString(mod, "__path__"); + if (path == NULL) { PyErr_Clear(); Py_INCREF(Py_None); return Py_None; } - bufobj = PyUnicode_DecodeFSDefault(buf); - if (bufobj != NULL) { - m = load_module(fullname, fp, bufobj, fdp->type, loader); - Py_DECREF(bufobj); - } - else - m = NULL; - Py_XDECREF(loader); - if (fp) - fclose(fp); - if (m != NULL && !add_submodule(mod, m, fullname, subname, modules)) { - Py_XDECREF(m); - m = NULL; - } } + fdp = find_module(_PyUnicode_AsString(fullname), + _PyUnicode_AsString(subname), + path, buf, MAXPATHLEN+1, + &fp, &loader); + Py_XDECREF(path); + if (fdp == NULL) { + if (!PyErr_ExceptionMatches(PyExc_ImportError)) + return NULL; + PyErr_Clear(); + Py_INCREF(Py_None); + return Py_None; + } + bufobj = PyUnicode_DecodeFSDefault(buf); + if (bufobj != NULL) { + m = load_module(fullname, fp, bufobj, fdp->type, loader); + Py_DECREF(bufobj); + } + else + m = NULL; + Py_XDECREF(loader); + if (fp) + fclose(fp); + if (m == NULL) + return NULL; + if (!add_submodule(mod, m, fullname, subname, modules)) { + Py_XDECREF(m); + return NULL; + } return m; } -- cgit v1.2.1 From 7b12e183cf51fb59b0d62f26e37f91ec881a0dd9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 13 Mar 2011 23:11:02 -0400 Subject: Issue #3080: Drop OS/2 support for the import machinery Sorry Andrew I MacIntyre! --- Python/import.c | 52 ---------------------------------------------------- 1 file changed, 52 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index cce2de1b44..e6d7c060fe 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1804,11 +1804,6 @@ find_module_path_list(char *fullname, const char *name, struct filedescr *fdp = NULL; char *filemode; FILE *fp = NULL; -#if defined(PYOS_OS2) - size_t saved_len; - size_t saved_namelen; - char *saved_buf = NULL; -#endif npath = PyList_Size(search_path_list); namelen = strlen(name); @@ -1832,40 +1827,7 @@ find_module_path_list(char *fullname, const char *name, return fdp; len = strlen(buf); -#if defined(PYOS_OS2) - /* take a snapshot of the module spec for restoration - * after the 8 character DLL hackery - */ - saved_buf = strdup(buf); - saved_len = len; - saved_namelen = namelen; -#endif /* PYOS_OS2 */ for (fdp = _PyImport_Filetab; fdp->suffix != NULL; fdp++) { -#if defined(PYOS_OS2) && defined(HAVE_DYNAMIC_LOADING) - /* OS/2 limits DLLs to 8 character names (w/o - extension) - * so if the name is longer than that and its a - * dynamically loaded module we're going to try, - * truncate the name before trying - */ - if (strlen(name) > 8) { - /* is this an attempt to load a C extension? */ - const struct filedescr *scan; - scan = _PyImport_DynLoadFiletab; - while (scan->suffix != NULL) { - if (!strcmp(scan->suffix, fdp->suffix)) - break; - else - scan++; - } - if (scan->suffix != NULL) { - /* yes, so truncate the name */ - namelen = 8; - len -= strlen(name) - namelen; - buf[len] = '\0'; - } - } -#endif /* PYOS_OS2 */ strcpy(buf+len, fdp->suffix); if (Py_VerboseFlag > 1) PySys_WriteStderr("# trying %s\n", buf); @@ -1881,21 +1843,7 @@ find_module_path_list(char *fullname, const char *name, fp = NULL; } } -#if defined(PYOS_OS2) - /* restore the saved snapshot */ - strcpy(buf, saved_buf); - len = saved_len; - namelen = saved_namelen; -#endif - } -#if defined(PYOS_OS2) - /* don't need/want the module name snapshot anymore */ - if (saved_buf) - { - free(saved_buf); - saved_buf = NULL; } -#endif if (fp != NULL) break; } -- cgit v1.2.1 From e7fd2eaecbfdc259fc9ae199608d33d721d88f57 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 09:21:33 -0400 Subject: Issue #3080: find_module() expects module fullname and subname as Unicode And PyImport_ReloadModule() uses Unicode for the module name. --- Python/import.c | 170 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 84 insertions(+), 86 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index e6d7c060fe..a3f263146c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1491,7 +1491,7 @@ unchanged: /* Forward */ static PyObject *load_module(PyObject *, FILE *, PyObject *, int, PyObject *); -static struct filedescr *find_module(char *, const char *, PyObject *, +static struct filedescr *find_module(PyObject *, PyObject *, PyObject *, char *, size_t, FILE **, PyObject **); static struct _frozen * find_frozen(PyObject *); @@ -1507,7 +1507,13 @@ load_package(PyObject *name, PyObject *pathname) char buf[MAXPATHLEN+1]; FILE *fp = NULL; struct filedescr *fdp; - char *namestr; + static PyObject *initstr = NULL; + + if (initstr == NULL) { + initstr = PyUnicode_InternFromString("__init__"); + if (initstr == NULL) + return NULL; + } m = PyImport_AddModuleObject(name); if (m == NULL) @@ -1532,10 +1538,7 @@ load_package(PyObject *name, PyObject *pathname) Py_DECREF(path_list); return NULL; } - namestr = _PyUnicode_AsString(name); - if (namestr == NULL) - goto error; - fdp = find_module(namestr, "__init__", path_list, buf, sizeof(buf), &fp, NULL); + fdp = find_module(name, initstr, path_list, buf, sizeof(buf), &fp, NULL); Py_DECREF(path_list); if (fdp == NULL) { if (PyErr_ExceptionMatches(PyExc_ImportError)) { @@ -1680,7 +1683,7 @@ static struct filedescr importhookdescr = {"", "", IMP_HOOK}; and *buf is the path */ static int -find_module_path(char *fullname, const char *name, PyObject *path, +find_module_path(PyObject *fullname, PyObject *name, PyObject *path, PyObject *path_hooks, PyObject *path_importer_cache, char *buf, size_t buflen, PyObject **p_loader, struct filedescr **p_fd) @@ -1691,6 +1694,7 @@ find_module_path(char *fullname, const char *name, PyObject *path, size_t namelen; struct stat statbuf; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; + char *namestr; if (PyUnicode_Check(path)) { path_bytes = PyUnicode_EncodeFSDefault(path); @@ -1704,7 +1708,8 @@ find_module_path(char *fullname, const char *name, PyObject *path, else return 0; - namelen = strlen(name); + namestr = _PyUnicode_AsString(name); + namelen = strlen(namestr); base = PyBytes_AS_STRING(path_bytes); len = PyBytes_GET_SIZE(path_bytes); if (len + 2 + namelen + MAXSUFFIXSIZE >= buflen) { @@ -1731,8 +1736,7 @@ find_module_path(char *fullname, const char *name, PyObject *path, if (importer != Py_None) { PyObject *loader; loader = PyObject_CallMethod(importer, - "find_module", - "s", fullname); + "find_module", "O", fullname); if (loader == NULL) return -1; /* error */ if (loader != Py_None) { @@ -1753,7 +1757,7 @@ find_module_path(char *fullname, const char *name, PyObject *path, #endif ) buf[len++] = SEP; - strcpy(buf+len, name); + strcpy(buf+len, namestr); len += namelen; /* Check for package import (buf holds a directory name, @@ -1761,7 +1765,7 @@ find_module_path(char *fullname, const char *name, PyObject *path, #ifdef HAVE_STAT if (stat(buf, &statbuf) == 0 && /* it exists */ S_ISDIR(statbuf.st_mode) && /* it's a directory */ - case_ok(buf, len, namelen, name)) { /* case matches */ + case_ok(buf, len, namelen, namestr)) { /* case matches */ if (find_init_module(buf)) { /* and has __init__.py */ *p_fd = &fd_package; return 2; @@ -1784,7 +1788,7 @@ find_module_path(char *fullname, const char *name, PyObject *path, } /* Find a module in search_path_list. For each path, try - find_module_filename() or try each _PyImport_Filetab suffix. + find_module_path() or try each _PyImport_Filetab suffix. If the module is found, return a file descriptor, write the path in *p_filename, write the pointer to the file object into *p_fp, and (if @@ -1793,7 +1797,7 @@ find_module_path(char *fullname, const char *name, PyObject *path, Otherwise, raise an exception and return NULL. */ static struct filedescr* -find_module_path_list(char *fullname, const char *name, +find_module_path_list(PyObject *fullname, PyObject *name, PyObject *search_path_list, PyObject *path_hooks, PyObject *path_importer_cache, char *buf, size_t buflen, @@ -1804,9 +1808,11 @@ find_module_path_list(char *fullname, const char *name, struct filedescr *fdp = NULL; char *filemode; FILE *fp = NULL; + char *namestr; npath = PyList_Size(search_path_list); - namelen = strlen(name); + namestr = _PyUnicode_AsString(name); + namelen = strlen(namestr); for (i = 0; i < npath; i++) { PyObject *path; int ok; @@ -1836,7 +1842,7 @@ find_module_path_list(char *fullname, const char *name, filemode = "r" PY_STDIOTEXTMODE; fp = fopen(buf, filemode); if (fp != NULL) { - if (case_ok(buf, len, namelen, name)) + if (case_ok(buf, len, namelen, namestr)) break; else { /* continue search */ fclose(fp); @@ -1849,7 +1855,7 @@ find_module_path_list(char *fullname, const char *name, } if (fp == NULL) { PyErr_Format(PyExc_ImportError, - "No module named %.200s", name); + "No module named %U", name); return NULL; } *p_fp = fp; @@ -1882,21 +1888,20 @@ find_module_path_list(char *fullname, const char *name, set) are set to NULL. Eg. *buf is an empty string for a builtin package. */ static struct filedescr * -find_module(char *fullname, const char *name, PyObject *search_path_list, +find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, char *buf, size_t buflen, FILE **p_fp, PyObject **p_loader) { Py_ssize_t i, npath; static struct filedescr fd_frozen = {"", "", PY_FROZEN}; static struct filedescr fd_builtin = {"", "", C_BUILTIN}; PyObject *path_hooks, *path_importer_cache; - PyObject *fullname_obj, *nameobj; *buf = '\0'; *p_fp = NULL; if (p_loader != NULL) *p_loader = NULL; - if (strlen(name) > MAXPATHLEN) { + if (PyUnicode_GET_SIZE(name) > MAXPATHLEN) { PyErr_SetString(PyExc_OverflowError, "module name is too long"); return NULL; @@ -1919,7 +1924,7 @@ find_module(char *fullname, const char *name, PyObject *search_path_list, PyObject *loader; PyObject *hook = PyList_GetItem(meta_path, i); loader = PyObject_CallMethod(hook, "find_module", - "sO", fullname, + "OO", fullname, search_path_list != NULL ? search_path_list : Py_None); if (loader == NULL) { @@ -1937,16 +1942,9 @@ find_module(char *fullname, const char *name, PyObject *search_path_list, Py_DECREF(meta_path); } - if (fullname != NULL) { - fullname_obj = PyUnicode_FromString(fullname); - if (fullname == NULL) - return NULL; - if (find_frozen(fullname_obj) != NULL) { - Py_DECREF(fullname_obj); - strcpy(buf, fullname); - return &fd_frozen; - } - Py_DECREF(fullname_obj); + if (find_frozen(fullname) != NULL) { + strcpy(buf, _PyUnicode_AsString(fullname)); + return &fd_frozen; } if (search_path_list == NULL) { @@ -1955,18 +1953,13 @@ find_module(char *fullname, const char *name, PyObject *search_path_list, struct filedescr *fdp; PyObject *filename, *filename_bytes; #endif - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - if (is_builtin(nameobj)) { - Py_DECREF(nameobj); - strcpy(buf, name); + if (is_builtin(name)) { + strcpy(buf, _PyUnicode_AsString(name)); return &fd_builtin; } #ifdef MS_COREDLL - fp = _PyWin_FindRegisteredModule(nameobj, &fdp, &filename); + fp = _PyWin_FindRegisteredModule(name, &fdp, &filename); if (fp != NULL) { - Py_DECREF(nameobj); filename_bytes = PyUnicode_EncodeFSDefault(filename); Py_DECREF(filename); if (filename_bytes == NULL) @@ -1980,7 +1973,6 @@ find_module(char *fullname, const char *name, PyObject *search_path_list, else if (PyErr_Occurred()) return NULL; #endif - Py_DECREF(nameobj); search_path_list = PySys_GetObject("path"); } @@ -3139,8 +3131,8 @@ import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) } } - fdp = find_module(_PyUnicode_AsString(fullname), - _PyUnicode_AsString(subname), + fdp = find_module(fullname, + subname, path, buf, MAXPATHLEN+1, &fp, &loader); Py_XDECREF(path); @@ -3180,12 +3172,13 @@ PyImport_ReloadModule(PyObject *m) PyInterpreterState *interp = PyThreadState_Get()->interp; PyObject *modules_reloading = interp->modules_reloading; PyObject *modules = PyImport_GetModuleDict(); - PyObject *path = NULL, *loader, *existing_m = NULL; - char *name, *subname; char buf[MAXPATHLEN+1]; + PyObject *path = NULL, *loader = NULL, *existing_m = NULL; + PyObject *nameobj, *bufobj, *subnameobj; + Py_UNICODE *name, *subname; struct filedescr *fdp; - FILE *fp; - PyObject *newm, *nameobj, *bufobj; + FILE *fp = NULL; + PyObject *newm = NULL; if (modules_reloading == NULL) { Py_FatalError("PyImport_ReloadModule: " @@ -3198,69 +3191,75 @@ PyImport_ReloadModule(PyObject *m) "reload() argument must be module"); return NULL; } - name = (char*)PyModule_GetName(m); - if (name == NULL) + nameobj = PyModule_GetNameObject(m); + if (nameobj == NULL) return NULL; - if (m != PyDict_GetItemString(modules, name)) { + if (m != PyDict_GetItem(modules, nameobj)) { PyErr_Format(PyExc_ImportError, - "reload(): module %.200s not in sys.modules", - name); + "reload(): module %U not in sys.modules", + nameobj); + Py_DECREF(nameobj); return NULL; } - existing_m = PyDict_GetItemString(modules_reloading, name); + existing_m = PyDict_GetItem(modules_reloading, nameobj); if (existing_m != NULL) { /* Due to a recursive reload, this module is already being reloaded. */ + Py_DECREF(nameobj); Py_INCREF(existing_m); return existing_m; } - if (PyDict_SetItemString(modules_reloading, name, m) < 0) + if (PyDict_SetItem(modules_reloading, nameobj, m) < 0) { + Py_DECREF(nameobj); return NULL; + } - subname = strrchr(name, '.'); - if (subname == NULL) - subname = name; + name = PyUnicode_AS_UNICODE(nameobj); + subname = Py_UNICODE_strrchr(name, '.'); + if (subname == NULL) { + Py_INCREF(nameobj); + subnameobj = nameobj; + } else { PyObject *parentname, *parent; - parentname = PyUnicode_FromStringAndSize(name, (subname-name)); + Py_ssize_t len; + len = subname - name; + parentname = PyUnicode_FromUnicode(name, len); if (parentname == NULL) { - imp_modules_reloading_clear(); - return NULL; + goto error; } parent = PyDict_GetItem(modules, parentname); if (parent == NULL) { PyErr_Format(PyExc_ImportError, - "reload(): parent %R not in sys.modules", + "reload(): parent %U not in sys.modules", parentname); Py_DECREF(parentname); - imp_modules_reloading_clear(); - return NULL; + goto error; } Py_DECREF(parentname); - subname++; path = PyObject_GetAttrString(parent, "__path__"); if (path == NULL) PyErr_Clear(); + subname++; + len = PyUnicode_GET_SIZE(nameobj) - (len + 1); + subnameobj = PyUnicode_FromUnicode(subname, len); } - fdp = find_module(name, subname, path, buf, MAXPATHLEN+1, &fp, &loader); + if (subnameobj == NULL) + goto error; + fdp = find_module(nameobj, subnameobj, + path, buf, MAXPATHLEN+1, &fp, &loader); + Py_DECREF(subnameobj); Py_XDECREF(path); if (fdp == NULL) { Py_XDECREF(loader); - imp_modules_reloading_clear(); - return NULL; + goto error; } - nameobj = PyUnicode_FromString(name); - if (nameobj != NULL) { - bufobj = PyUnicode_DecodeFSDefault(buf); - if (bufobj != NULL) { - newm = load_module(nameobj, fp, bufobj, fdp->type, loader); - Py_DECREF(bufobj); - } - else - newm = NULL; - Py_DECREF(nameobj); + bufobj = PyUnicode_DecodeFSDefault(buf); + if (bufobj != NULL) { + newm = load_module(nameobj, fp, bufobj, fdp->type, loader); + Py_DECREF(bufobj); } else newm = NULL; @@ -3274,9 +3273,12 @@ PyImport_ReloadModule(PyObject *m) * going to return NULL in this case regardless of whether * replacing name succeeds, so the return value is ignored. */ - PyDict_SetItemString(modules, name, m); + PyDict_SetItem(modules, nameobj, m); } + +error: imp_modules_reloading_clear(); + Py_DECREF(nameobj); return newm; } @@ -3424,7 +3426,7 @@ imp_get_suffixes(PyObject *self, PyObject *noargs) } static PyObject * -call_find_module(char *name, PyObject *path) +call_find_module(PyObject *name, PyObject *path) { extern int fclose(FILE *); PyObject *fob, *ret; @@ -3483,15 +3485,11 @@ call_find_module(char *name, PyObject *path) static PyObject * imp_find_module(PyObject *self, PyObject *args) { - PyObject *name; - PyObject *ret, *path = NULL; - if (!PyArg_ParseTuple(args, "O&|O:find_module", - PyUnicode_FSConverter, &name, - &path)) + PyObject *name, *path_list = NULL; + if (!PyArg_ParseTuple(args, "U|O:find_module", + &name, &path_list)) return NULL; - ret = call_find_module(PyBytes_AS_STRING(name), path); - Py_DECREF(name); - return ret; + return call_find_module(name, path_list); } static PyObject * -- cgit v1.2.1 From c23cd9690f4b004e965101735800e7c352fcfc78 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 13:22:54 -0400 Subject: Issue #3080: Rename some path variables to path_list --- Python/import.c | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index a3f263146c..73d38fe10f 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1538,7 +1538,8 @@ load_package(PyObject *name, PyObject *pathname) Py_DECREF(path_list); return NULL; } - fdp = find_module(name, initstr, path_list, buf, sizeof(buf), &fp, NULL); + fdp = find_module(name, initstr, path_list, + buf, sizeof(buf), &fp, NULL); Py_DECREF(path_list); if (fdp == NULL) { if (PyErr_ExceptionMatches(PyExc_ImportError)) { @@ -3105,7 +3106,7 @@ static PyObject * import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) { PyObject *modules = PyImport_GetModuleDict(); - PyObject *m = NULL, *bufobj, *path, *loader; + PyObject *m = NULL, *bufobj, *path_list, *loader; char buf[MAXPATHLEN+1]; struct filedescr *fdp; FILE *fp; @@ -3121,21 +3122,19 @@ import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) } if (mod == Py_None) - path = NULL; + path_list = NULL; else { - path = PyObject_GetAttrString(mod, "__path__"); - if (path == NULL) { + path_list = PyObject_GetAttrString(mod, "__path__"); + if (path_list == NULL) { PyErr_Clear(); Py_INCREF(Py_None); return Py_None; } } - fdp = find_module(fullname, - subname, - path, buf, MAXPATHLEN+1, - &fp, &loader); - Py_XDECREF(path); + fdp = find_module(fullname, subname, path_list, + buf, MAXPATHLEN+1, &fp, &loader); + Py_XDECREF(path_list); if (fdp == NULL) { if (!PyErr_ExceptionMatches(PyExc_ImportError)) return NULL; @@ -3173,7 +3172,7 @@ PyImport_ReloadModule(PyObject *m) PyObject *modules_reloading = interp->modules_reloading; PyObject *modules = PyImport_GetModuleDict(); char buf[MAXPATHLEN+1]; - PyObject *path = NULL, *loader = NULL, *existing_m = NULL; + PyObject *path_list = NULL, *loader = NULL, *existing_m = NULL; PyObject *nameobj, *bufobj, *subnameobj; Py_UNICODE *name, *subname; struct filedescr *fdp; @@ -3237,8 +3236,8 @@ PyImport_ReloadModule(PyObject *m) goto error; } Py_DECREF(parentname); - path = PyObject_GetAttrString(parent, "__path__"); - if (path == NULL) + path_list = PyObject_GetAttrString(parent, "__path__"); + if (path_list == NULL) PyErr_Clear(); subname++; len = PyUnicode_GET_SIZE(nameobj) - (len + 1); @@ -3246,10 +3245,10 @@ PyImport_ReloadModule(PyObject *m) } if (subnameobj == NULL) goto error; - fdp = find_module(nameobj, subnameobj, - path, buf, MAXPATHLEN+1, &fp, &loader); + fdp = find_module(nameobj, subnameobj, path_list, + buf, MAXPATHLEN+1, &fp, &loader); Py_DECREF(subnameobj); - Py_XDECREF(path); + Py_XDECREF(path_list); if (fdp == NULL) { Py_XDECREF(loader); @@ -3426,7 +3425,7 @@ imp_get_suffixes(PyObject *self, PyObject *noargs) } static PyObject * -call_find_module(PyObject *name, PyObject *path) +call_find_module(PyObject *name, PyObject *path_list) { extern int fclose(FILE *); PyObject *fob, *ret; @@ -3439,9 +3438,10 @@ call_find_module(PyObject *name, PyObject *path) char *encoding = NULL; pathname[0] = '\0'; - if (path == Py_None) - path = NULL; - fdp = find_module(NULL, name, path, pathname, MAXPATHLEN+1, &fp, NULL); + if (path_list == Py_None) + path_list = NULL; + fdp = find_module(NULL, name, path_list, + pathname, MAXPATHLEN+1, &fp, NULL); if (fdp == NULL) return NULL; if (fp != NULL) { -- cgit v1.2.1 From d5658be50acead958aef78e5c41c8d4d4bd0e857 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 13:40:04 -0400 Subject: Issue #3080: find_module() sets an empty path for builtin and frozen modules --- Python/import.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 73d38fe10f..2c21112c4b 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1943,10 +1943,8 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, Py_DECREF(meta_path); } - if (find_frozen(fullname) != NULL) { - strcpy(buf, _PyUnicode_AsString(fullname)); + if (find_frozen(fullname) != NULL) return &fd_frozen; - } if (search_path_list == NULL) { #ifdef MS_COREDLL @@ -1954,10 +1952,8 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, struct filedescr *fdp; PyObject *filename, *filename_bytes; #endif - if (is_builtin(name)) { - strcpy(buf, _PyUnicode_AsString(name)); + if (is_builtin(name)) return &fd_builtin; - } #ifdef MS_COREDLL fp = _PyWin_FindRegisteredModule(name, &fdp, &filename); if (fp != NULL) { -- cgit v1.2.1 From 2c6f2066619d885ebc3fd2c23dd6ca17f18ba71c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 14:04:10 -0400 Subject: Issue #3080: Refactor find_module_path(), use return instead of break Prepare also the API change of case_ok() --- Python/import.c | 77 +++++++++++++++++++++++++++------------------------------ 1 file changed, 37 insertions(+), 40 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 2c21112c4b..979058e583 100644 --- a/Python/import.c +++ b/Python/import.c @@ -144,6 +144,8 @@ static const struct filedescr _PyImport_StandardFiletab[] = { {0, 0} }; +static PyObject *initstr = NULL; + /* Initialize things */ @@ -155,6 +157,10 @@ _PyImport_Init(void) int countD = 0; int countS = 0; + initstr = PyUnicode_InternFromString("__init__"); + if (initstr == NULL) + Py_FatalError("Can't initialize import variables"); + /* prepare _PyImport_Filetab: copy entries from _PyImport_DynLoadFiletab and _PyImport_StandardFiletab. */ @@ -1507,13 +1513,6 @@ load_package(PyObject *name, PyObject *pathname) char buf[MAXPATHLEN+1]; FILE *fp = NULL; struct filedescr *fdp; - static PyObject *initstr = NULL; - - if (initstr == NULL) { - initstr = PyUnicode_InternFromString("__init__"); - if (initstr == NULL) - return NULL; - } m = PyImport_AddModuleObject(name); if (m == NULL) @@ -1765,23 +1764,25 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, and there's an __init__ module in that directory */ #ifdef HAVE_STAT if (stat(buf, &statbuf) == 0 && /* it exists */ - S_ISDIR(statbuf.st_mode) && /* it's a directory */ - case_ok(buf, len, namelen, namestr)) { /* case matches */ - if (find_init_module(buf)) { /* and has __init__.py */ - *p_fd = &fd_package; - return 2; - } - else { - int err; - PyObject *unicode = PyUnicode_DecodeFSDefault(buf); - if (unicode == NULL) - return -1; - err = PyErr_WarnFormat(PyExc_ImportWarning, 1, - "Not importing directory '%U': missing __init__.py", - unicode); - Py_DECREF(unicode); - if (err) - return -1; + S_ISDIR(statbuf.st_mode)) /* it's a directory */ + { + if (case_ok(buf, len, namelen, namestr)) { /* case matches */ + if (find_init_module(buf)) { /* and has __init__.py */ + *p_fd = &fd_package; + return 2; + } + else { + int err; + PyObject *unicode = PyUnicode_DecodeFSDefault(buf); + if (unicode == NULL) + return -1; + err = PyErr_WarnFormat(PyExc_ImportWarning, 1, + "Not importing directory '%U': missing __init__.py", + unicode); + Py_DECREF(unicode); + if (err) + return -1; + } } } #endif @@ -1842,25 +1843,21 @@ find_module_path_list(PyObject *fullname, PyObject *name, if (filemode[0] == 'U') filemode = "r" PY_STDIOTEXTMODE; fp = fopen(buf, filemode); - if (fp != NULL) { - if (case_ok(buf, len, namelen, namestr)) - break; - else { /* continue search */ - fclose(fp); - fp = NULL; - } + if (fp == NULL) + continue; + + if (case_ok(buf, len, namelen, namestr)) { + *p_fp = fp; + return fdp; } + + fclose(fp); + fp = NULL; } - if (fp != NULL) - break; - } - if (fp == NULL) { - PyErr_Format(PyExc_ImportError, - "No module named %U", name); - return NULL; } - *p_fp = fp; - return fdp; + PyErr_Format(PyExc_ImportError, + "No module named %U", name); + return NULL; } /* Find a module: -- cgit v1.2.1 From 87e72d7343dc9ff213470b182fbba2bd73e7a6d1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 20 Mar 2011 03:07:28 +0100 Subject: Issue #3080: find_init_module() expects Unicode --- Python/import.c | 88 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 45 insertions(+), 43 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 979058e583..8a242a57eb 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1668,8 +1668,9 @@ extern FILE *_PyWin_FindRegisteredModule(PyObject *, struct filedescr **, PyObject **p_path); #endif +/* Forward */ static int case_ok(char *, Py_ssize_t, Py_ssize_t, const char *); -static int find_init_module(char *); /* Forward */ +static int find_init_module(PyObject *); static struct filedescr importhookdescr = {"", "", IMP_HOOK}; /* Get the path of a module: get its importer and call importer.find_module() @@ -1766,24 +1767,27 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, if (stat(buf, &statbuf) == 0 && /* it exists */ S_ISDIR(statbuf.st_mode)) /* it's a directory */ { + PyObject *bufobj = PyUnicode_DecodeFSDefault(buf); + if (bufobj == NULL) + return -1; if (case_ok(buf, len, namelen, namestr)) { /* case matches */ - if (find_init_module(buf)) { /* and has __init__.py */ + if (find_init_module(bufobj)) { /* and has __init__.py */ + Py_DECREF(bufobj); *p_fd = &fd_package; return 2; } else { int err; - PyObject *unicode = PyUnicode_DecodeFSDefault(buf); - if (unicode == NULL) - return -1; err = PyErr_WarnFormat(PyExc_ImportWarning, 1, - "Not importing directory '%U': missing __init__.py", - unicode); - Py_DECREF(unicode); - if (err) + "Not importing directory %R: missing __init__.py", + bufobj); + if (err) { + Py_DECREF(bufobj); return -1; + } } } + Py_DECREF(bufobj); } #endif return 1; @@ -2154,49 +2158,47 @@ case_ok(char *buf, Py_ssize_t len, Py_ssize_t namelen, const char *name) #ifdef HAVE_STAT -/* Helper to look for __init__.py or __init__.py[co] in potential package */ +/* Helper to look for __init__.py or __init__.py[co] in potential package. + Return 1 if __init__ was found, 0 if not, or -1 on error. */ static int -find_init_module(char *buf) +find_init_module(PyObject *directory) { - const size_t save_len = strlen(buf); - size_t i = save_len; - char *pname; /* pointer to start of __init__ */ + size_t len; struct stat statbuf; - -/* For calling case_ok(buf, len, namelen, name): - * /a/b/c/d/e/f/g/h/i/j/k/some_long_module_name.py\0 - * ^ ^ ^ ^ - * |--------------------- buf ---------------------| - * |------------------- len ------------------| - * |------ name -------| - * |----- namelen -----| - */ - if (save_len + 13 >= MAXPATHLEN) - return 0; - buf[i++] = SEP; - pname = buf + i; - strcpy(pname, "__init__.py"); - if (stat(buf, &statbuf) == 0) { - if (case_ok(buf, - save_len + 9, /* len("/__init__") */ - 8, /* len("__init__") */ - pname)) { - buf[save_len] = '\0'; + PyObject *filename; + int match; + char *filestr; + size_t filelen; + + len = PyUnicode_GET_SIZE(directory); + filename = PyUnicode_FromFormat("%U%c__init__.py", directory, SEP); + if (filename == NULL) + return -1; + if (_Py_stat(filename, &statbuf) == 0) { + /* 9=len("/__init__") */ + filestr = _PyUnicode_AsString(filename); + filelen = strlen(filestr); + if (case_ok(filestr, filelen-9, 8, "__init__")) { + Py_DECREF(filename); return 1; } } - i += strlen(pname); - strcpy(buf+i, Py_OptimizeFlag ? "o" : "c"); - if (stat(buf, &statbuf) == 0) { - if (case_ok(buf, - save_len + 9, /* len("/__init__") */ - 8, /* len("__init__") */ - pname)) { - buf[save_len] = '\0'; + Py_DECREF(filename); + + filename = PyUnicode_FromFormat("%U%c__init__.py%c", + directory, SEP, Py_OptimizeFlag ? 'o' : 'c'); + if (filename == NULL) + return -1; + if (_Py_stat(filename, &statbuf) == 0) { + /* 9=len("/__init__") */ + filestr = _PyUnicode_AsString(filename); + filelen = strlen(filestr); + if (case_ok(filestr, filelen-9, 8, "__init__")) { + Py_DECREF(filename); return 1; } } - buf[save_len] = '\0'; + Py_DECREF(filename); return 0; } -- cgit v1.2.1 From 7bca4dba35971a2705590aee419a18ad29fefd06 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 14:34:13 -0400 Subject: Issue #3080: case_ok() expects Unicode strings --- Python/import.c | 181 +++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 132 insertions(+), 49 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 8a242a57eb..bba0120321 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1669,7 +1669,7 @@ extern FILE *_PyWin_FindRegisteredModule(PyObject *, struct filedescr **, #endif /* Forward */ -static int case_ok(char *, Py_ssize_t, Py_ssize_t, const char *); +static int case_ok(PyObject *, Py_ssize_t, PyObject *); static int find_init_module(PyObject *); static struct filedescr importhookdescr = {"", "", IMP_HOOK}; @@ -1767,12 +1767,20 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, if (stat(buf, &statbuf) == 0 && /* it exists */ S_ISDIR(statbuf.st_mode)) /* it's a directory */ { - PyObject *bufobj = PyUnicode_DecodeFSDefault(buf); - if (bufobj == NULL) + int match; + PyObject *filename; + + filename = PyUnicode_DecodeFSDefault(buf); + if (filename == NULL) + return -1; + match = case_ok(filename, 0, name); + if (match < 0) { + Py_DECREF(filename); return -1; - if (case_ok(buf, len, namelen, namestr)) { /* case matches */ - if (find_init_module(bufobj)) { /* and has __init__.py */ - Py_DECREF(bufobj); + } + if (match) { /* case matches */ + if (find_init_module(filename)) { /* and has __init__.py */ + Py_DECREF(filename); *p_fd = &fd_package; return 2; } @@ -1780,14 +1788,14 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, int err; err = PyErr_WarnFormat(PyExc_ImportWarning, 1, "Not importing directory %R: missing __init__.py", - bufobj); + filename); if (err) { - Py_DECREF(bufobj); + Py_DECREF(filename); return -1; } } } - Py_DECREF(bufobj); + Py_DECREF(filename); } #endif return 1; @@ -1815,6 +1823,8 @@ find_module_path_list(PyObject *fullname, PyObject *name, char *filemode; FILE *fp = NULL; char *namestr; + PyObject *filename; + int match; npath = PyList_Size(search_path_list); namestr = _PyUnicode_AsString(name); @@ -1840,20 +1850,34 @@ find_module_path_list(PyObject *fullname, PyObject *name, len = strlen(buf); for (fdp = _PyImport_Filetab; fdp->suffix != NULL; fdp++) { - strcpy(buf+len, fdp->suffix); - if (Py_VerboseFlag > 1) - PySys_WriteStderr("# trying %s\n", buf); filemode = fdp->mode; if (filemode[0] == 'U') filemode = "r" PY_STDIOTEXTMODE; - fp = fopen(buf, filemode); - if (fp == NULL) - continue; - if (case_ok(buf, len, namelen, namestr)) { + strcpy(buf+len, fdp->suffix); + filename = PyUnicode_DecodeFSDefault(buf); + if (filename == NULL) + return NULL; + + if (Py_VerboseFlag > 1) + PySys_FormatStderr("# trying %R\n", filename); + + fp = _Py_fopen(filename, filemode); + if (fp == NULL) { + Py_DECREF(filename); + continue; + } + match = case_ok(filename, -(Py_ssize_t)strlen(fdp->suffix), name); + if (match < 0) { + Py_DECREF(filename); + return NULL; + } + if (match) { + Py_DECREF(filename); *p_fp = fp; return fdp; } + Py_DECREF(filename); fclose(fp); fp = NULL; @@ -2002,7 +2026,7 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, p_fp, p_loader); } -/* case_ok(char* buf, Py_ssize_t len, Py_ssize_t namelen, char* name) +/* case_bytes(char* buf, Py_ssize_t len, Py_ssize_t namelen, char* name) * The arguments here are tricky, best shown by example: * /a/b/c/d/e/f/g/h/i/j/k/some_long_module_name.py\0 * ^ ^ ^ ^ @@ -2016,18 +2040,18 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, * We've already done a successful stat() or fopen() on buf, so know that * there's some match, possibly case-insensitive. * - * case_ok() is to return 1 if there's a case-sensitive match for - * name, else 0. case_ok() is also to return 1 if envar PYTHONCASEOK + * case_bytes() is to return 1 if there's a case-sensitive match for + * name, else 0. case_bytes() is also to return 1 if envar PYTHONCASEOK * exists. * - * case_ok() is used to implement case-sensitive import semantics even + * case_bytes() is used to implement case-sensitive import semantics even * on platforms with case-insensitive filesystems. It's trivial to implement * for case-sensitive filesystems. It's pretty much a cross-platform * nightmare for systems with case-insensitive filesystems. */ /* First we may need a pile of platform-specific header files; the sequence - * of #if's here should match the sequence in the body of case_ok(). + * of #if's here should match the sequence in the body of case_bytes(). */ #if defined(MS_WINDOWS) #include @@ -2046,33 +2070,24 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, #include #endif +#if defined(DJGPP) \ + || ((defined(__MACH__) && defined(__APPLE__) || defined(__CYGWIN__)) \ + && defined(HAVE_DIRENT_H)) \ + || defined(PYOS_OS2) +# define USE_CASE_OK_BYTES +#endif + + +#ifdef USE_CASE_OK_BYTES static int -case_ok(char *buf, Py_ssize_t len, Py_ssize_t namelen, const char *name) +case_bytes(char *buf, Py_ssize_t len, Py_ssize_t namelen, const char *name) { /* Pick a platform-specific implementation; the sequence of #if's here should * match the sequence just above. */ -/* MS_WINDOWS */ -#if defined(MS_WINDOWS) - WIN32_FIND_DATA data; - HANDLE h; - - if (Py_GETENV("PYTHONCASEOK") != NULL) - return 1; - - h = FindFirstFile(buf, &data); - if (h == INVALID_HANDLE_VALUE) { - PyErr_Format(PyExc_NameError, - "Can't find file for module %.100s\n(filename %.300s)", - name, buf); - return 0; - } - FindClose(h); - return strncmp(data.cFileName, name, namelen) == 0; - /* DJGPP */ -#elif defined(DJGPP) +#if defined(DJGPP) struct ffblk ffblk; int done; @@ -2151,6 +2166,70 @@ case_ok(char *buf, Py_ssize_t len, Py_ssize_t namelen, const char *name) /* assuming it's a case-sensitive filesystem, so there's nothing to do! */ #else +# error "USE_CASE_OK_BYTES is not correctly defined" +#endif +} +#endif + +/* + * Check if a filename case matchs the name case. We've already done a + * successful stat() or fopen() on buf, so know that there's some match, + * possibly case-insensitive. + * + * case_ok() is to return 1 if there's a case-sensitive match for name, 0 if it + * the filename doesn't match, or -1 on error. case_ok() is also to return 1 + * if envar PYTHONCASEOK exists. + * + * case_ok() is used to implement case-sensitive import semantics even + * on platforms with case-insensitive filesystems. It's trivial to implement + * for case-sensitive filesystems. It's pretty much a cross-platform + * nightmare for systems with case-insensitive filesystems. + */ + +static int +case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) +{ +#ifdef MS_WINDOWS + WIN32_FIND_DATAW data; + HANDLE h; + int cmp; + + if (Py_GETENV("PYTHONCASEOK") != NULL) + return 1; + + h = FindFirstFileW(PyUnicode_AS_UNICODE(filename), &data); + if (h == INVALID_HANDLE_VALUE) { + PyErr_Format(PyExc_NameError, + "Can't find file for module %R\n(filename %R)", + name, filename); + return 0; + } + FindClose(h); + cmp = wcsncmp(data.cFileName, + PyUnicode_AS_UNICODE(name), + PyUnicode_GET_SIZE(name)); + return cmp == 0; +#elif defined(USE_CASE_OK_BYTES) + int match; + PyObject *filebytes, *namebytes; + filebytes = PyUnicode_EncodeFSDefault(filename); + if (filebytes == NULL) + return -1; + namebytes = PyUnicode_EncodeFSDefault(name); + if (namebytes == NULL) { + Py_DECREF(filebytes); + return -1; + } + match = case_bytes( + PyBytes_AS_STRING(filebytes), + PyBytes_GET_SIZE(filebytes) + prefix_delta, + PyBytes_AS_STRING(namebytes), + PyBytes_GET_SIZE(namebytes)); + Py_DECREF(filebytes); + Py_DECREF(namebytes); + return match; +#else + /* assuming it's a case-sensitive filesystem, so there's nothing to do! */ return 1; #endif @@ -2167,8 +2246,6 @@ find_init_module(PyObject *directory) struct stat statbuf; PyObject *filename; int match; - char *filestr; - size_t filelen; len = PyUnicode_GET_SIZE(directory); filename = PyUnicode_FromFormat("%U%c__init__.py", directory, SEP); @@ -2176,9 +2253,12 @@ find_init_module(PyObject *directory) return -1; if (_Py_stat(filename, &statbuf) == 0) { /* 9=len("/__init__") */ - filestr = _PyUnicode_AsString(filename); - filelen = strlen(filestr); - if (case_ok(filestr, filelen-9, 8, "__init__")) { + match = case_ok(filename, 9, initstr); + if (match < 0) { + Py_DECREF(filename); + return -1; + } + if (match) { Py_DECREF(filename); return 1; } @@ -2191,9 +2271,12 @@ find_init_module(PyObject *directory) return -1; if (_Py_stat(filename, &statbuf) == 0) { /* 9=len("/__init__") */ - filestr = _PyUnicode_AsString(filename); - filelen = strlen(filestr); - if (case_ok(filestr, filelen-9, 8, "__init__")) { + match = case_ok(filename, 9, initstr); + if (match < 0) { + Py_DECREF(filename); + return -1; + } + if (match) { Py_DECREF(filename); return 1; } -- cgit v1.2.1 From 02352565abd1eed9eb32c5c273bc640869f2100b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 15:19:39 -0400 Subject: Issue #3080: find_module() returns the path as Unicode --- Python/import.c | 171 ++++++++++++++++++++++++-------------------------------- 1 file changed, 74 insertions(+), 97 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index bba0120321..5dcbf17710 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1498,7 +1498,7 @@ unchanged: /* Forward */ static PyObject *load_module(PyObject *, FILE *, PyObject *, int, PyObject *); static struct filedescr *find_module(PyObject *, PyObject *, PyObject *, - char *, size_t, FILE **, PyObject **); + PyObject **, FILE **, PyObject **); static struct _frozen * find_frozen(PyObject *); /* Load a package and return its module object WITH INCREMENTED @@ -1510,7 +1510,6 @@ load_package(PyObject *name, PyObject *pathname) PyObject *m, *d, *bufobj; PyObject *file = NULL, *path_list = NULL; int err; - char buf[MAXPATHLEN+1]; FILE *fp = NULL; struct filedescr *fdp; @@ -1538,7 +1537,7 @@ load_package(PyObject *name, PyObject *pathname) return NULL; } fdp = find_module(name, initstr, path_list, - buf, sizeof(buf), &fp, NULL); + &bufobj, &fp, NULL); Py_DECREF(path_list); if (fdp == NULL) { if (PyErr_ExceptionMatches(PyExc_ImportError)) { @@ -1549,13 +1548,8 @@ load_package(PyObject *name, PyObject *pathname) else return NULL; } - bufobj = PyUnicode_DecodeFSDefault(buf); - if (bufobj != NULL) { - m = load_module(name, fp, bufobj, fdp->type, NULL); - Py_DECREF(bufobj); - } - else - m = NULL; + m = load_module(name, fp, bufobj, fdp->type, NULL); + Py_XDECREF(bufobj); if (fp != NULL) fclose(fp); return m; @@ -1686,43 +1680,40 @@ static struct filedescr importhookdescr = {"", "", IMP_HOOK}; static int find_module_path(PyObject *fullname, PyObject *name, PyObject *path, PyObject *path_hooks, PyObject *path_importer_cache, - char *buf, size_t buflen, - PyObject **p_loader, struct filedescr **p_fd) + PyObject **p_path, PyObject **p_loader, struct filedescr **p_fd) { - PyObject *path_bytes; - const char *base; + Py_UNICODE buf[MAXPATHLEN+1]; + Py_ssize_t buflen = MAXPATHLEN+1; + PyObject *path_unicode, *filename; + const Py_UNICODE *base; Py_ssize_t len; - size_t namelen; struct stat statbuf; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; - char *namestr; if (PyUnicode_Check(path)) { - path_bytes = PyUnicode_EncodeFSDefault(path); - if (path_bytes == NULL) - return -1; + Py_INCREF(path); + path_unicode = path; } else if (PyBytes_Check(path)) { - Py_INCREF(path); - path_bytes = path; + path_unicode = PyUnicode_DecodeFSDefaultAndSize( + PyBytes_AS_STRING(path), PyBytes_GET_SIZE(path)); + if (path_unicode == NULL) + return -1; } else return 0; - namestr = _PyUnicode_AsString(name); - namelen = strlen(namestr); - base = PyBytes_AS_STRING(path_bytes); - len = PyBytes_GET_SIZE(path_bytes); - if (len + 2 + namelen + MAXSUFFIXSIZE >= buflen) { - Py_DECREF(path_bytes); + base = PyUnicode_AS_UNICODE(path_unicode); + len = PyUnicode_GET_SIZE(path_unicode); + if (len + 2 + PyUnicode_GET_SIZE(name) + MAXSUFFIXSIZE >= buflen) { + Py_DECREF(path_unicode); return 0; /* Too long */ } - strcpy(buf, base); - Py_DECREF(path_bytes); + Py_UNICODE_strcpy(buf, PyUnicode_AS_UNICODE(path_unicode)); + Py_DECREF(path_unicode); - if (strlen(buf) != len) { - return 0; /* path_bytes contains '\0' */ - } + if (Py_UNICODE_strlen(buf) != len) + return 0; /* path contains '\0' */ /* sys.path_hooks import hook */ if (p_loader != NULL) { @@ -1758,21 +1749,21 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, #endif ) buf[len++] = SEP; - strcpy(buf+len, namestr); - len += namelen; + Py_UNICODE_strcpy(buf+len, PyUnicode_AS_UNICODE(name)); + len += PyUnicode_GET_SIZE(name); + + filename = PyUnicode_FromUnicode(buf, len); + if (filename == NULL) + return -1; /* Check for package import (buf holds a directory name, and there's an __init__ module in that directory */ #ifdef HAVE_STAT - if (stat(buf, &statbuf) == 0 && /* it exists */ + if (_Py_stat(filename, &statbuf) == 0 && /* it exists */ S_ISDIR(statbuf.st_mode)) /* it's a directory */ { int match; - PyObject *filename; - filename = PyUnicode_DecodeFSDefault(buf); - if (filename == NULL) - return -1; match = case_ok(filename, 0, name); if (match < 0) { Py_DECREF(filename); @@ -1780,7 +1771,7 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, } if (match) { /* case matches */ if (find_init_module(filename)) { /* and has __init__.py */ - Py_DECREF(filename); + *p_path = filename; *p_fd = &fd_package; return 2; } @@ -1795,9 +1786,9 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, } } } - Py_DECREF(filename); } #endif + *p_path = filename; return 1; } @@ -1814,21 +1805,16 @@ static struct filedescr* find_module_path_list(PyObject *fullname, PyObject *name, PyObject *search_path_list, PyObject *path_hooks, PyObject *path_importer_cache, - char *buf, size_t buflen, - FILE **p_fp, PyObject **p_loader) + PyObject **p_path, FILE **p_fp, PyObject **p_loader) { Py_ssize_t i, npath; - size_t len, namelen; struct filedescr *fdp = NULL; char *filemode; FILE *fp = NULL; - char *namestr; - PyObject *filename; + PyObject *prefix, *filename; int match; npath = PyList_Size(search_path_list); - namestr = _PyUnicode_AsString(name); - namelen = strlen(namestr); for (i = 0; i < npath; i++) { PyObject *path; int ok; @@ -1837,27 +1823,29 @@ find_module_path_list(PyObject *fullname, PyObject *name, if (path == NULL) return NULL; + prefix = NULL; ok = find_module_path(fullname, name, path, path_hooks, path_importer_cache, - buf, buflen, - p_loader, &fdp); + &prefix, p_loader, &fdp); if (ok < 0) return NULL; if (ok == 0) continue; - if (ok == 2) + if (ok == 2) { + *p_path = prefix; return fdp; + } - len = strlen(buf); for (fdp = _PyImport_Filetab; fdp->suffix != NULL; fdp++) { filemode = fdp->mode; if (filemode[0] == 'U') filemode = "r" PY_STDIOTEXTMODE; - strcpy(buf+len, fdp->suffix); - filename = PyUnicode_DecodeFSDefault(buf); - if (filename == NULL) + filename = PyUnicode_FromFormat("%U%s", prefix, fdp->suffix); + if (filename == NULL) { + Py_DECREF(prefix); return NULL; + } if (Py_VerboseFlag > 1) PySys_FormatStderr("# trying %R\n", filename); @@ -1869,11 +1857,13 @@ find_module_path_list(PyObject *fullname, PyObject *name, } match = case_ok(filename, -(Py_ssize_t)strlen(fdp->suffix), name); if (match < 0) { + Py_DECREF(prefix); Py_DECREF(filename); return NULL; } if (match) { - Py_DECREF(filename); + Py_DECREF(prefix); + *p_path = filename; *p_fp = fp; return fdp; } @@ -1882,6 +1872,7 @@ find_module_path_list(PyObject *fullname, PyObject *name, fclose(fp); fp = NULL; } + Py_DECREF(prefix); } PyErr_Format(PyExc_ImportError, "No module named %U", name); @@ -1897,32 +1888,35 @@ find_module_path_list(PyObject *fullname, PyObject *name, - otherwise, call find_module_path_list() with search_path_list (if not NULL) or sys.path + fullname can be NULL, but only if p_loader is NULL. + Return: - &fd_builtin (C_BUILTIN) if it is a builtin - &fd_frozen (PY_FROZEN) if it is frozen - - &fd_package (PKG_DIRECTORY) and write the filename into *buf + - &fd_package (PKG_DIRECTORY) and write the filename into *p_path if it is a package - &importhookdescr (IMP_HOOK) and write the loader into *p_loader if a importer loader was found - a file descriptor (PY_SOURCE, PY_COMPILED, C_EXTENSION, PY_RESOURCE or PY_CODERESOURCE: see _PyImport_Filetab), write the filename into - *buf and the pointer to the open file into *p_fp + *p_path and the pointer to the open file into *p_fp - NULL on error - By default, write an empty string into *buf, and *p_fp and *p_loader (if - set) are set to NULL. Eg. *buf is an empty string for a builtin package. */ + By default, *p_path, *p_fp and *p_loader (if set) are set to NULL. + Eg. *p_path is set to NULL for a builtin package. +*/ static struct filedescr * find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, - char *buf, size_t buflen, FILE **p_fp, PyObject **p_loader) + PyObject **p_path, FILE **p_fp, PyObject **p_loader) { Py_ssize_t i, npath; static struct filedescr fd_frozen = {"", "", PY_FROZEN}; static struct filedescr fd_builtin = {"", "", C_BUILTIN}; PyObject *path_hooks, *path_importer_cache; - *buf = '\0'; + *p_path = NULL; *p_fp = NULL; if (p_loader != NULL) *p_loader = NULL; @@ -1975,20 +1969,12 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, #ifdef MS_COREDLL FILE *fp; struct filedescr *fdp; - PyObject *filename, *filename_bytes; #endif if (is_builtin(name)) return &fd_builtin; #ifdef MS_COREDLL - fp = _PyWin_FindRegisteredModule(name, &fdp, &filename); + fp = _PyWin_FindRegisteredModule(name, &fdp, p_path); if (fp != NULL) { - filename_bytes = PyUnicode_EncodeFSDefault(filename); - Py_DECREF(filename); - if (filename_bytes == NULL) - return NULL; - strncpy(buf, PyBytes_AS_STRING(filename_bytes), buflen); - buf[buflen-1] = '\0'; - Py_DECREF(filename_bytes); *p_fp = fp; return fdp; } @@ -2022,8 +2008,7 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, return find_module_path_list(fullname, name, search_path_list, path_hooks, path_importer_cache, - buf, buflen, - p_fp, p_loader); + p_path, p_fp, p_loader); } /* case_bytes(char* buf, Py_ssize_t len, Py_ssize_t namelen, char* name) @@ -3185,7 +3170,6 @@ import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) { PyObject *modules = PyImport_GetModuleDict(); PyObject *m = NULL, *bufobj, *path_list, *loader; - char buf[MAXPATHLEN+1]; struct filedescr *fdp; FILE *fp; @@ -3211,7 +3195,7 @@ import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) } fdp = find_module(fullname, subname, path_list, - buf, MAXPATHLEN+1, &fp, &loader); + &bufobj, &fp, &loader); Py_XDECREF(path_list); if (fdp == NULL) { if (!PyErr_ExceptionMatches(PyExc_ImportError)) @@ -3220,13 +3204,8 @@ import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) Py_INCREF(Py_None); return Py_None; } - bufobj = PyUnicode_DecodeFSDefault(buf); - if (bufobj != NULL) { - m = load_module(fullname, fp, bufobj, fdp->type, loader); - Py_DECREF(bufobj); - } - else - m = NULL; + m = load_module(fullname, fp, bufobj, fdp->type, loader); + Py_XDECREF(bufobj); Py_XDECREF(loader); if (fp) fclose(fp); @@ -3249,7 +3228,6 @@ PyImport_ReloadModule(PyObject *m) PyInterpreterState *interp = PyThreadState_Get()->interp; PyObject *modules_reloading = interp->modules_reloading; PyObject *modules = PyImport_GetModuleDict(); - char buf[MAXPATHLEN+1]; PyObject *path_list = NULL, *loader = NULL, *existing_m = NULL; PyObject *nameobj, *bufobj, *subnameobj; Py_UNICODE *name, *subname; @@ -3324,7 +3302,7 @@ PyImport_ReloadModule(PyObject *m) if (subnameobj == NULL) goto error; fdp = find_module(nameobj, subnameobj, path_list, - buf, MAXPATHLEN+1, &fp, &loader); + &bufobj, &fp, &loader); Py_DECREF(subnameobj); Py_XDECREF(path_list); @@ -3333,13 +3311,8 @@ PyImport_ReloadModule(PyObject *m) goto error; } - bufobj = PyUnicode_DecodeFSDefault(buf); - if (bufobj != NULL) { - newm = load_module(nameobj, fp, bufobj, fdp->type, loader); - Py_DECREF(bufobj); - } - else - newm = NULL; + newm = load_module(nameobj, fp, bufobj, fdp->type, loader); + Py_XDECREF(bufobj); Py_XDECREF(loader); if (fp) @@ -3509,17 +3482,15 @@ call_find_module(PyObject *name, PyObject *path_list) PyObject *fob, *ret; PyObject *pathobj; struct filedescr *fdp; - char pathname[MAXPATHLEN+1]; FILE *fp; int fd = -1; char *found_encoding = NULL; char *encoding = NULL; - pathname[0] = '\0'; if (path_list == Py_None) path_list = NULL; fdp = find_module(NULL, name, path_list, - pathname, MAXPATHLEN+1, &fp, NULL); + &pathobj, &fp, NULL); if (fdp == NULL) return NULL; if (fp != NULL) { @@ -3535,14 +3506,17 @@ call_find_module(PyObject *name, PyObject *path_list) memory. */ found_encoding = PyTokenizer_FindEncoding(fd); lseek(fd, 0, 0); /* Reset position */ - if (found_encoding == NULL && PyErr_Occurred()) + if (found_encoding == NULL && PyErr_Occurred()) { + Py_XDECREF(pathobj); return NULL; + } encoding = (found_encoding != NULL) ? found_encoding : (char*)PyUnicode_GetDefaultEncoding(); } - fob = PyFile_FromFd(fd, pathname, fdp->mode, -1, + fob = PyFile_FromFd(fd, NULL, fdp->mode, -1, (char*)encoding, NULL, NULL, 1); if (fob == NULL) { + Py_XDECREF(pathobj); close(fd); PyMem_FREE(found_encoding); return NULL; @@ -3552,7 +3526,10 @@ call_find_module(PyObject *name, PyObject *path_list) fob = Py_None; Py_INCREF(fob); } - pathobj = PyUnicode_DecodeFSDefault(pathname); + if (pathobj == NULL) { + Py_INCREF(Py_None); + pathobj = Py_None; + } ret = Py_BuildValue("NN(ssi)", fob, pathobj, fdp->suffix, fdp->mode, fdp->type); PyMem_FREE(found_encoding); -- cgit v1.2.1 From 8784da32ff0f704a48459abf452eba8345f0d053 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 14:53:28 -0400 Subject: Issue #3080: imp.new_module() uses Unicode --- Python/import.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 5dcbf17710..3237ddc84e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3787,10 +3787,10 @@ imp_load_package(PyObject *self, PyObject *args) static PyObject * imp_new_module(PyObject *self, PyObject *args) { - char *name; - if (!PyArg_ParseTuple(args, "s:new_module", &name)) + PyObject *name; + if (!PyArg_ParseTuple(args, "U:new_module", &name)) return NULL; - return PyModule_New(name); + return PyModule_NewObject(name); } static PyObject * -- cgit v1.2.1 From 24cf6f1d864c6c51a299ed32ba9e1fe8701a4b54 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 15:05:12 -0400 Subject: Fix imp.cache_from_source() if the directory name contains a dot If the directory name contains a dot but not the filename, don't strip at the dot. --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 3237ddc84e..e5cb627003 100644 --- a/Python/import.c +++ b/Python/import.c @@ -943,12 +943,12 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug) Py_UNICODE_strcat(buf, CACHEDIR_UNICODE); i += Py_UNICODE_strlen(CACHEDIR_UNICODE) - 1; buf[i++] = sep; - buf[i++] = '\0'; + buf[i] = '\0'; /* Add the base filename, but remove the .py or .pyw extension, since the tag name must go before the extension. */ Py_UNICODE_strcat(buf, pathname + save); - pos = Py_UNICODE_strrchr(buf, '.'); + pos = Py_UNICODE_strrchr(buf + i, '.'); if (pos != NULL) *++pos = '\0'; Py_UNICODE_strcat(buf, PYC_TAG_UNICODE); -- cgit v1.2.1 From 0b5200ff91987d460004543f52dc7443f57a6b36 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 15:15:47 -0400 Subject: Issue #3080: Use repr() to format the module name on error --- Python/import.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index e5cb627003..65944473cc 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1431,7 +1431,7 @@ load_source_module(PyObject *name, PyObject *pathname, FILE *fp) goto error; if (Py_VerboseFlag) PySys_FormatStderr("import %U # from %R\n", - name, pathname); + name, pathname); if (cpathname != NULL) { PyObject *ro = PySys_GetObject("dont_write_bytecode"); if (ro == NULL || !PyObject_IsTrue(ro)) @@ -1517,8 +1517,8 @@ load_package(PyObject *name, PyObject *pathname) if (m == NULL) return NULL; if (Py_VerboseFlag) - PySys_FormatStderr("import %U # directory %U\n", - name, pathname); + PySys_FormatStderr("import %U # directory %R\n", + name, pathname); file = get_sourcefile(pathname); if (file == NULL) return NULL; @@ -1875,7 +1875,7 @@ find_module_path_list(PyObject *fullname, PyObject *name, Py_DECREF(prefix); } PyErr_Format(PyExc_ImportError, - "No module named %U", name); + "No module named %R", name); return NULL; } @@ -2366,7 +2366,7 @@ load_module(PyObject *name, FILE *fp, PyObject *pathname, int type, PyObject *lo default: PyErr_Format(PyExc_ImportError, - "Don't know how to import %U (type code %d)", + "Don't know how to import %R (type code %d)", name, type); m = NULL; @@ -3251,7 +3251,7 @@ PyImport_ReloadModule(PyObject *m) return NULL; if (m != PyDict_GetItem(modules, nameobj)) { PyErr_Format(PyExc_ImportError, - "reload(): module %U not in sys.modules", + "reload(): module %R not in sys.modules", nameobj); Py_DECREF(nameobj); return NULL; @@ -3286,7 +3286,7 @@ PyImport_ReloadModule(PyObject *m) parent = PyDict_GetItem(modules, parentname); if (parent == NULL) { PyErr_Format(PyExc_ImportError, - "reload(): parent %U not in sys.modules", + "reload(): parent %R not in sys.modules", parentname); Py_DECREF(parentname); goto error; -- cgit v1.2.1 From 67a022a7f5cdf458fa1e76d7b5482fec1784890f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 15:54:52 -0400 Subject: Issue #3080: Add PyImport_ImportModuleLevelObject() function Use it for the builtin __import__ function. --- Python/import.c | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 65944473cc..7f9042ac01 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2753,25 +2753,37 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, } PyObject * -PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, - PyObject *fromlist, int level) +PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, + PyObject *locals, PyObject *fromlist, + int level) { - PyObject *nameobj, *result; - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; + PyObject *mod; _PyImport_AcquireLock(); - result = import_module_level(nameobj, globals, locals, fromlist, level); - Py_DECREF(nameobj); + mod = import_module_level(name, globals, locals, fromlist, level); if (_PyImport_ReleaseLock() < 0) { - Py_XDECREF(result); + Py_XDECREF(mod); PyErr_SetString(PyExc_RuntimeError, "not holding the import lock"); return NULL; } - return result; + return mod; +} + +PyObject * +PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, + PyObject *fromlist, int level) +{ + PyObject *nameobj, *mod; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + mod = PyImport_ImportModuleLevelObject(nameobj, globals, locals, + fromlist, level); + Py_DECREF(nameobj); + return mod; } + /* Return the package that an import is being performed in. If globals comes from the module foo.bar.bat (not itself a package), this returns the sys.modules entry for foo.bar. If globals is from a package's __init__.py, -- cgit v1.2.1 From 83ee336336ccdc6b1fcc97ce0f693409ecf4b60a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 20 Mar 2011 04:28:55 +0100 Subject: Issue #3080: Fix case_ok() using case_bytes() Invert name and namelen arguments. --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 7f9042ac01..0194d89956 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2208,8 +2208,8 @@ case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) match = case_bytes( PyBytes_AS_STRING(filebytes), PyBytes_GET_SIZE(filebytes) + prefix_delta, - PyBytes_AS_STRING(namebytes), - PyBytes_GET_SIZE(namebytes)); + PyBytes_GET_SIZE(namebytes), + PyBytes_AS_STRING(namebytes)); Py_DECREF(filebytes); Py_DECREF(namebytes); return match; -- cgit v1.2.1 From f8715e031dd6628c290bfc9341b092063449a6f8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 20 Mar 2011 04:58:29 +0100 Subject: Issue #3080: Fix call to case_ok() in find_init_module() --- Python/import.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 0194d89956..38960b165d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2237,8 +2237,8 @@ find_init_module(PyObject *directory) if (filename == NULL) return -1; if (_Py_stat(filename, &statbuf) == 0) { - /* 9=len("/__init__") */ - match = case_ok(filename, 9, initstr); + /* 3=len(".py") */ + match = case_ok(filename, -3, initstr); if (match < 0) { Py_DECREF(filename); return -1; @@ -2255,8 +2255,8 @@ find_init_module(PyObject *directory) if (filename == NULL) return -1; if (_Py_stat(filename, &statbuf) == 0) { - /* 9=len("/__init__") */ - match = case_ok(filename, 9, initstr); + /* 4=len(".pyc") */ + match = case_ok(filename, -4, initstr); if (match < 0) { Py_DECREF(filename); return -1; -- cgit v1.2.1 From 6d71d6018e93b53f8af5ed0d268a4c834a5cdda2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 20 Mar 2011 22:37:17 +0100 Subject: Issue #3080: imp.load_module() accepts None for the module path imp.find_module() returns None as module path for builtin and frozen builtins. --- Python/import.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 38960b165d..abc3906719 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3744,17 +3744,22 @@ imp_load_source(PyObject *self, PyObject *args) static PyObject * imp_load_module(PyObject *self, PyObject *args) { - PyObject *name, *fob, *pathname, *ret; + PyObject *name, *fob, *pathname, *pathname_obj, *ret; char *suffix; /* Unused */ char *mode; int type; FILE *fp; - if (!PyArg_ParseTuple(args, "UOO&(ssi):load_module", - &name, &fob, - PyUnicode_FSDecoder, &pathname, - &suffix, &mode, &type)) + if (!PyArg_ParseTuple(args, "UOO(ssi):load_module", + &name, &fob, &pathname_obj, &suffix, &mode, &type)) return NULL; + if (pathname_obj != Py_None) { + if (!PyUnicode_FSDecoder(pathname_obj, &pathname)) + return NULL; + } + else + pathname = NULL; + if (*mode) { /* Mode must start with 'r' or 'U' and must not contain '+'. Implicit in this test is the assumption that the mode @@ -3763,7 +3768,7 @@ imp_load_module(PyObject *self, PyObject *args) if (!(*mode == 'r' || *mode == 'U') || strchr(mode, '+')) { PyErr_Format(PyExc_ValueError, "invalid file open mode %.200s", mode); - Py_DECREF(pathname); + Py_XDECREF(pathname); return NULL; } } @@ -3772,12 +3777,12 @@ imp_load_module(PyObject *self, PyObject *args) else { fp = get_file(NULL, fob, mode); if (fp == NULL) { - Py_DECREF(pathname); + Py_XDECREF(pathname); return NULL; } } ret = load_module(name, fp, pathname, type, NULL); - Py_DECREF(pathname); + Py_XDECREF(pathname); if (fp) fclose(fp); return ret; -- cgit v1.2.1 From 05292378fc049f9a5ffa9480d0ad27b25f2c5528 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2011 01:22:27 +0100 Subject: Issue #3080: On DJGPP, case_bytes() returns -1 to signal an error if the file cannot be found. --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index abc3906719..3b8ce931d9 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2084,7 +2084,7 @@ case_bytes(char *buf, Py_ssize_t len, Py_ssize_t namelen, const char *name) PyErr_Format(PyExc_NameError, "Can't find file for module %.100s\n(filename %.300s)", name, buf); - return 0; + return -1; } return strncmp(ffblk.ff_name, name, namelen) == 0; -- cgit v1.2.1 From aad56d7f47263bddcaca2765d03df11d81bc0926 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2011 10:46:35 +0100 Subject: Issue #11630, issue #3080: Fix refleak introduced by ef2b6305d395 --- Python/import.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 3b8ce931d9..907ccd797c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3705,12 +3705,15 @@ imp_load_dynamic(PyObject *self, PyObject *args) return NULL; if (fob != NULL) { fp = get_file(NULL, fob, "r"); - if (fp == NULL) + if (fp == NULL) { + Py_DECREF(pathname); return NULL; + } } else fp = NULL; mod = _PyImport_LoadDynamicModule(name, pathname, fp); + Py_DECREF(pathname); if (fp) fclose(fp); return mod; -- cgit v1.2.1 From 725c21b3a4aec4426ccaa4c0425ee4e777a9eb35 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 23 Mar 2011 16:14:42 -0700 Subject: Make importlib compatible with __import__ by "fixing" code.co_filename paths. __import__ does a little trick when importing from bytecode by back-patching the co_filename paths to point to the file location where the code object was loaded from, *not* where the code object was originally created. This allows co_filename to point to a valid path. Problem is that co_filename is immutable from Python, so a private function -- imp._fix_co_filename() -- had to be introduced in order to get things working properly. Originally the plan was to add a file argument to marshal.loads(), but that failed as the algorithm used by __import__ is not fully recursive as one might expect, so to be fully backwards-compatible the code used by __import__ needed to be exposed. This closes issue #6811 by taking a different approach than outlined in the issue. --- Python/import.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 907ccd797c..b074b834e9 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1374,6 +1374,32 @@ update_compiled_module(PyCodeObject *co, PyObject *newname) Py_DECREF(oldname); } +static PyObject * +imp_fix_co_filename(PyObject *self, PyObject *args) +{ + PyObject *co; + PyObject *file_path; + + if (!PyArg_ParseTuple(args, "OO:_fix_co_filename", &co, &file_path)) + return NULL; + + if (!PyCode_Check(co)) { + PyErr_SetString(PyExc_TypeError, + "first argument must be a code object"); + return NULL; + } + + if (!PyUnicode_Check(file_path)) { + PyErr_SetString(PyExc_TypeError, + "second argument must be a string"); + return NULL; + } + + update_compiled_module((PyCodeObject*)co, file_path); + + Py_RETURN_NONE; +} + /* Load a source module from a given file and return its module object WITH INCREMENTED REFERENCE COUNT. If there's a matching byte-compiled file, use that instead. */ @@ -3976,6 +4002,7 @@ static PyMethodDef imp_methods[] = { #endif {"load_package", imp_load_package, METH_VARARGS}, {"load_source", imp_load_source, METH_VARARGS}, + {"_fix_co_filename", imp_fix_co_filename, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; -- cgit v1.2.1 From f41c2fdbb05ad29ad7c2a6a65dfac2e9d919f022 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 5 Apr 2011 01:48:03 +0200 Subject: Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. --- Python/import.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index b074b834e9..4159a8ece6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -124,12 +124,12 @@ static const Py_UNICODE PYC_TAG_UNICODE[] = { /* See _PyImport_FixupExtensionObject() below */ static PyObject *extensions = NULL; +/* Function from Parser/tokenizer.c */ +extern char * PyTokenizer_FindEncodingFilename(int, PyObject *); + /* This table is defined in config.c: */ extern struct _inittab _PyImport_Inittab[]; -/* Method from Parser/tokenizer.c */ -extern char * PyTokenizer_FindEncoding(int); - struct _inittab *PyImport_Inittab = _PyImport_Inittab; /* these tables define the module suffixes that Python recognizes */ @@ -3540,9 +3540,9 @@ call_find_module(PyObject *name, PyObject *path_list) } if (fd != -1) { if (strchr(fdp->mode, 'b') == NULL) { - /* PyTokenizer_FindEncoding() returns PyMem_MALLOC'ed + /* PyTokenizer_FindEncodingFilename() returns PyMem_MALLOC'ed memory. */ - found_encoding = PyTokenizer_FindEncoding(fd); + found_encoding = PyTokenizer_FindEncodingFilename(fd, pathobj); lseek(fd, 0, 0); /* Reset position */ if (found_encoding == NULL && PyErr_Occurred()) { Py_XDECREF(pathobj); -- cgit v1.2.1 From 0cd7388f789c312864700c151157a939f07cc090 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 20 Apr 2011 03:27:51 +0200 Subject: Close #11619: write_compiled_module() doesn't encode the filename Reimplement open_exclusive() using _wopen() to avoid encoding the filename to the filesystem encoding: use the Unicode version of the Windows API. --- Python/import.c | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 4159a8ece6..5360d57c51 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1198,6 +1198,7 @@ parse_source_module(PyObject *pathname, FILE *fp) /* Helper to open a bytecode file for writing in exclusive mode */ +#ifndef MS_WINDOWS static FILE * open_exclusive(char *filename, mode_t mode) { @@ -1228,6 +1229,7 @@ open_exclusive(char *filename, mode_t mode) return fopen(filename, "wb"); #endif } +#endif /* Write a compiled module to a file, placing the time of last @@ -1250,7 +1252,12 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, S_IWUSR | S_IWGRP | S_IWOTH); PyObject *dirbytes; #endif - PyObject *cpathbytes, *dirname; +#ifdef MS_WINDOWS + int fd; +#else + PyObject *cpathbytes; +#endif + PyObject *dirname; Py_UNICODE *dirsep; int res, ok; @@ -1294,6 +1301,16 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, } Py_DECREF(dirname); +#ifdef MS_WINDOWS + (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname)); + fd = _wopen(PyUnicode_AS_UNICODE(cpathname), + O_EXCL | O_CREAT | O_WRONLY | O_TRUNC | O_BINARY, + mode); + if (0 <= fd) + fp = fdopen(fd, "wb"); + else + fp = NULL; +#else cpathbytes = PyUnicode_EncodeFSDefault(cpathname); if (cpathbytes == NULL) { PyErr_Clear(); @@ -1301,11 +1318,14 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, } fp = open_exclusive(PyBytes_AS_STRING(cpathbytes), mode); +#endif if (fp == NULL) { if (Py_VerboseFlag) PySys_FormatStderr( "# can't create %R\n", cpathname); +#ifndef MS_WINDOWS Py_DECREF(cpathbytes); +#endif return; } PyMarshal_WriteLongToFile(pyc_magic, fp, Py_MARSHAL_VERSION); @@ -1321,11 +1341,13 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname)); #else (void) unlink(PyBytes_AS_STRING(cpathbytes)); -#endif Py_DECREF(cpathbytes); +#endif return; } +#ifndef MS_WINDOWS Py_DECREF(cpathbytes); +#endif /* Now write the true mtime */ fseek(fp, 4L, 0); assert(mtime < LONG_MAX); -- cgit v1.2.1 From bf30ea7ad386f8851207ea9d410a3215b03d31af Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Wed, 25 May 2011 09:29:00 -0500 Subject: make PyImport_ImportModuleLevel's first arg const like similiar functions (closes #12173) --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 5360d57c51..bfb976c77c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2818,7 +2818,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } PyObject * -PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, +PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) { PyObject *nameobj, *mod; -- cgit v1.2.1 From 54ecf696bb489a653324f5b8a20018495b8f0142 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 26 May 2011 13:47:08 +0200 Subject: Remove useless assignments Warnings found by the the Clang Static Analyzer. --- Python/import.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index bfb976c77c..1f28d2233d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1733,7 +1733,6 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, Py_UNICODE buf[MAXPATHLEN+1]; Py_ssize_t buflen = MAXPATHLEN+1; PyObject *path_unicode, *filename; - const Py_UNICODE *base; Py_ssize_t len; struct stat statbuf; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; @@ -1751,7 +1750,6 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, else return 0; - base = PyUnicode_AS_UNICODE(path_unicode); len = PyUnicode_GET_SIZE(path_unicode); if (len + 2 + PyUnicode_GET_SIZE(name) + MAXSUFFIXSIZE >= buflen) { Py_DECREF(path_unicode); @@ -2275,12 +2273,10 @@ case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) static int find_init_module(PyObject *directory) { - size_t len; struct stat statbuf; PyObject *filename; int match; - len = PyUnicode_GET_SIZE(directory); filename = PyUnicode_FromFormat("%U%c__init__.py", directory, SEP); if (filename == NULL) return -1; -- cgit v1.2.1 From dbea271885eccda0db2e5248fd932cc56ec8a495 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Fri, 3 Jun 2011 17:50:16 -0500 Subject: some horrible preprocessing tricks to automatically update the tag --- Python/import.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 1f28d2233d..aaf649f858 100644 --- a/Python/import.c +++ b/Python/import.c @@ -110,8 +110,12 @@ typedef unsigned short mode_t; TAG and PYC_TAG_UNICODE must change for each major Python release. The magic number will take care of any bytecode changes that occur during development. */ +#define QUOTE(arg) #arg +#define STRIFY(name) QUOTE(name) +#define MAJOR STRIFY(PY_MAJOR_VERSION) +#define MINOR STRIFY(PY_MINOR_VERSION) #define MAGIC (3180 | ((long)'\r'<<16) | ((long)'\n'<<24)) -#define TAG "cpython-32" +#define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" static const Py_UNICODE CACHEDIR_UNICODE[] = { '_', '_', 'p', 'y', 'c', 'a', 'c', 'h', 'e', '_', '_', '\0'}; @@ -119,7 +123,11 @@ static const Py_UNICODE CACHEDIR_UNICODE[] = { static long pyc_magic = MAGIC; static const char *pyc_tag = TAG; static const Py_UNICODE PYC_TAG_UNICODE[] = { - 'c', 'p', 'y', 't', 'h', 'o', 'n', '-', '3', '2', '\0'}; + 'c', 'p', 'y', 't', 'h', 'o', 'n', '-', PY_MAJOR_VERSION + 48, PY_MINOR_VERSION + 48, '\0'}; +#undef QUOTE +#undef STRIFY +#undef MAJOR +#undef MINOR /* See _PyImport_FixupExtensionObject() below */ static PyObject *extensions = NULL; -- cgit v1.2.1 From 18d2c7a85fe49dbac1177fffb6517b8093f8e23c Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 19 Jun 2011 19:54:45 -0500 Subject: bump magic for super closure change --- Python/import.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index aaf649f858..9ec8cd8b84 100644 --- a/Python/import.c +++ b/Python/import.c @@ -102,6 +102,7 @@ typedef unsigned short mode_t; Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR) tag: cpython-32 Python 3.2a2 3180 (add DELETE_DEREF) + Python 3.3a0 3190 __class__ super closure changed */ /* MAGIC must change whenever the bytecode emitted by the compiler may no @@ -114,7 +115,7 @@ typedef unsigned short mode_t; #define STRIFY(name) QUOTE(name) #define MAJOR STRIFY(PY_MAJOR_VERSION) #define MINOR STRIFY(PY_MINOR_VERSION) -#define MAGIC (3180 | ((long)'\r'<<16) | ((long)'\n'<<24)) +#define MAGIC (3190 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" static const Py_UNICODE CACHEDIR_UNICODE[] = { -- cgit v1.2.1 From 1a17e5818c9c58989ffdd4d0e2931db9c66579a0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 20 Jun 2011 15:01:10 +0200 Subject: find_module_path_list() fails if _Py_fopen() failed and raised an exception (UnicodeEncodeError). --- Python/import.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 9ec8cd8b84..de5dc0ca41 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1908,6 +1908,10 @@ find_module_path_list(PyObject *fullname, PyObject *name, fp = _Py_fopen(filename, filemode); if (fp == NULL) { Py_DECREF(filename); + if (PyErr_Occurred()) { + Py_DECREF(prefix); + return NULL; + } continue; } match = case_ok(filename, -(Py_ssize_t)strlen(fdp->suffix), name); -- cgit v1.2.1 From fab579dcbba91dc35f5f2b25fcf221736fff5e4b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 20 Jun 2011 15:16:55 +0200 Subject: call_find_module() handles dup() failure: raise an OSError exception --- Python/import.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index de5dc0ca41..19e975a593 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3567,6 +3567,10 @@ call_find_module(PyObject *name, PyObject *path_list) if (fd != -1) fd = dup(fd); fclose(fp); + if (fd == -1) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } fp = NULL; } if (fd != -1) { -- cgit v1.2.1 From 9ce343f33c35b14fe616a15cea33f8f84ccbff85 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 15 Sep 2011 19:50:01 +0200 Subject: import.c: remove now useless arbitrary limit --- Python/import.c | 6 ------ 1 file changed, 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 7902721a92..adfd2cc5df 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1980,12 +1980,6 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, if (p_loader != NULL) *p_loader = NULL; - if (PyUnicode_GET_SIZE(name) > MAXPATHLEN) { - PyErr_SetString(PyExc_OverflowError, - "module name is too long"); - return NULL; - } - /* sys.meta_path import hook */ if (p_loader != NULL) { PyObject *meta_path; -- cgit v1.2.1 From d1d013c01c268d869597b35cbcd8b5d7c5baf2ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Wed, 28 Sep 2011 07:41:54 +0200 Subject: Implement PEP 393. --- Python/import.c | 335 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 190 insertions(+), 145 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 24df985d46..4afdb7058c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -118,12 +118,12 @@ typedef unsigned short mode_t; #define MAGIC (3190 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" -static const Py_UNICODE CACHEDIR_UNICODE[] = { +static const Py_UCS4 CACHEDIR_UNICODE[] = { '_', '_', 'p', 'y', 'c', 'a', 'c', 'h', 'e', '_', '_', '\0'}; /* Current magic word and string tag as globals. */ static long pyc_magic = MAGIC; static const char *pyc_tag = TAG; -static const Py_UNICODE PYC_TAG_UNICODE[] = { +static const Py_UCS4 PYC_TAG_UNICODE[] = { 'c', 'p', 'y', 't', 'h', 'o', 'n', '-', PY_MAJOR_VERSION + 48, PY_MINOR_VERSION + 48, '\0'}; #undef QUOTE #undef STRIFY @@ -762,7 +762,7 @@ remove_module(PyObject *name) static PyObject * get_sourcefile(PyObject *filename); static PyObject *make_source_pathname(PyObject *pathname); -static PyObject* make_compiled_pathname(Py_UNICODE *pathname, int debug); +static PyObject* make_compiled_pathname(PyObject *pathname, int debug); /* Execute a code object in a module and return the module object * WITH INCREMENTED REFERENCE COUNT. If an error occurs, name is @@ -886,10 +886,10 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, /* Like strrchr(string, '/') but searches for the rightmost of either SEP or ALTSEP, if the latter is defined. */ -static Py_UNICODE* -rightmost_sep(Py_UNICODE *s) +static Py_UCS4* +rightmost_sep(Py_UCS4 *s) { - Py_UNICODE *found, c; + Py_UCS4 *found, c; for (found = NULL; (c = *s); s++) { if (c == SEP #ifdef ALTSEP @@ -912,15 +912,21 @@ rightmost_sep(Py_UNICODE *s) foo.py -> __pycache__/foo..pyc */ static PyObject* -make_compiled_pathname(Py_UNICODE *pathname, int debug) +make_compiled_pathname(PyObject *pathstr, int debug) { - Py_UNICODE buf[MAXPATHLEN]; + Py_UCS4 *pathname; + Py_UCS4 buf[MAXPATHLEN]; size_t buflen = (size_t)MAXPATHLEN; - size_t len = Py_UNICODE_strlen(pathname); + size_t len; size_t i, save; - Py_UNICODE *pos; + Py_UCS4 *pos; int sep = SEP; + pathname = PyUnicode_AsUCS4Copy(pathstr); + if (!pathname) + return NULL; + len = Py_UCS4_strlen(pathname); + /* Sanity check that the buffer has roughly enough space to hold what will eventually be the full path to the compiled file. The 5 extra bytes include the slash afer __pycache__, the two extra dots, the @@ -930,8 +936,10 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug) sanity check before writing the extension to ensure we do not overflow the buffer. */ - if (len + Py_UNICODE_strlen(CACHEDIR_UNICODE) + Py_UNICODE_strlen(PYC_TAG_UNICODE) + 5 > buflen) + if (len + Py_UCS4_strlen(CACHEDIR_UNICODE) + Py_UCS4_strlen(PYC_TAG_UNICODE) + 5 > buflen) { + PyMem_Free(pathname); return NULL; + } /* Find the last path separator and copy everything from the start of the source string up to and including the separator. @@ -943,24 +951,28 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug) else { sep = *pos; i = pos - pathname + 1; - Py_UNICODE_strncpy(buf, pathname, i); + Py_UCS4_strncpy(buf, pathname, i); } save = i; buf[i++] = '\0'; /* Add __pycache__/ */ - Py_UNICODE_strcat(buf, CACHEDIR_UNICODE); - i += Py_UNICODE_strlen(CACHEDIR_UNICODE) - 1; + Py_UCS4_strcat(buf, CACHEDIR_UNICODE); + i += Py_UCS4_strlen(CACHEDIR_UNICODE) - 1; buf[i++] = sep; buf[i] = '\0'; /* Add the base filename, but remove the .py or .pyw extension, since the tag name must go before the extension. */ - Py_UNICODE_strcat(buf, pathname + save); - pos = Py_UNICODE_strrchr(buf + i, '.'); + Py_UCS4_strcat(buf, pathname + save); + pos = Py_UCS4_strrchr(buf + i, '.'); if (pos != NULL) *++pos = '\0'; - Py_UNICODE_strcat(buf, PYC_TAG_UNICODE); + + /* pathname is not used from here on. */ + PyMem_Free(pathname); + + Py_UCS4_strcat(buf, PYC_TAG_UNICODE); /* The length test above assumes that we're only adding one character to the end of what would normally be the extension. What if there is no extension, or the string ends in '.' or '.p', and otherwise @@ -1010,7 +1022,7 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug) #if 0 printf("strlen(buf): %d; buflen: %d\n", (int)strlen(buf), (int)buflen); #endif - len = Py_UNICODE_strlen(buf); + len = Py_UCS4_strlen(buf); if (len + 5 > buflen) return NULL; buf[len] = '.'; len++; @@ -1018,7 +1030,7 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug) buf[len] = 'y'; len++; buf[len] = debug ? 'c' : 'o'; len++; assert(len <= buflen); - return PyUnicode_FromUnicode(buf, len); + return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, len); } @@ -1033,14 +1045,16 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug) static PyObject* make_source_pathname(PyObject *pathobj) { - Py_UNICODE buf[MAXPATHLEN]; - Py_UNICODE *pathname; - Py_UNICODE *left, *right, *dot0, *dot1, sep; + Py_UCS4 buf[MAXPATHLEN]; + Py_UCS4 *pathname; + Py_UCS4 *left, *right, *dot0, *dot1, sep; size_t i, j; - if (PyUnicode_GET_SIZE(pathobj) > MAXPATHLEN) + if (PyUnicode_GET_LENGTH(pathobj) > MAXPATHLEN) + return NULL; + pathname = PyUnicode_AsUCS4Copy(pathobj); + if (!pathname) return NULL; - pathname = PyUnicode_AS_UNICODE(pathobj); /* Look back two slashes from the end. In between these two slashes must be the string __pycache__ or this is not a PEP 3147 style @@ -1057,31 +1071,35 @@ make_source_pathname(PyObject *pathobj) left = pathname; else left++; - if (right-left != Py_UNICODE_strlen(CACHEDIR_UNICODE) || - Py_UNICODE_strncmp(left, CACHEDIR_UNICODE, right-left) != 0) - return NULL; + if (right-left != Py_UCS4_strlen(CACHEDIR_UNICODE) || + Py_UCS4_strncmp(left, CACHEDIR_UNICODE, right-left) != 0) + goto error; /* Now verify that the path component to the right of the last slash has two dots in it. */ - if ((dot0 = Py_UNICODE_strchr(right + 1, '.')) == NULL) - return NULL; - if ((dot1 = Py_UNICODE_strchr(dot0 + 1, '.')) == NULL) - return NULL; + if ((dot0 = Py_UCS4_strchr(right + 1, '.')) == NULL) + goto error; + if ((dot1 = Py_UCS4_strchr(dot0 + 1, '.')) == NULL) + goto error; /* Too many dots? */ - if (Py_UNICODE_strchr(dot1 + 1, '.') != NULL) - return NULL; + if (Py_UCS4_strchr(dot1 + 1, '.') != NULL) + goto error; /* This is a PEP 3147 path. Start by copying everything from the start of pathname up to and including the leftmost slash. Then copy the file's basename, removing the magic tag and adding a .py suffix. */ - Py_UNICODE_strncpy(buf, pathname, (i=left-pathname)); - Py_UNICODE_strncpy(buf+i, right+1, (j=dot0-right)); + Py_UCS4_strncpy(buf, pathname, (i=left-pathname)); + Py_UCS4_strncpy(buf+i, right+1, (j=dot0-right)); buf[i+j] = 'p'; buf[i+j+1] = 'y'; - return PyUnicode_FromUnicode(buf, i+j+2); + PyMem_Free(pathname); + return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, i+j+2); + error: + PyMem_Free(pathname); + return NULL; } /* Given a pathname for a Python source file, its time of last @@ -1250,6 +1268,7 @@ static void write_compiled_module(PyCodeObject *co, PyObject *cpathname, struct stat *srcstat) { + Py_UCS4 *cpathname_ucs4; FILE *fp; time_t mtime = srcstat->st_mtime; #ifdef MS_WINDOWS /* since Windows uses different permissions */ @@ -1267,18 +1286,23 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, PyObject *cpathbytes; #endif PyObject *dirname; - Py_UNICODE *dirsep; + Py_UCS4 *dirsep; int res, ok; /* Ensure that the __pycache__ directory exists. */ - dirsep = rightmost_sep(PyUnicode_AS_UNICODE(cpathname)); + cpathname_ucs4 = PyUnicode_AsUCS4Copy(cpathname); + if (!cpathname_ucs4) + return; + dirsep = rightmost_sep(cpathname_ucs4); if (dirsep == NULL) { if (Py_VerboseFlag) PySys_FormatStderr("# no %s path found %R\n", CACHEDIR, cpathname); return; } - dirname = PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(cpathname), - dirsep - PyUnicode_AS_UNICODE(cpathname)); + dirname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + cpathname_ucs4, + dirsep - cpathname_ucs4); + PyMem_Free(cpathname_ucs4); if (dirname == NULL) { PyErr_Clear(); return; @@ -1461,9 +1485,7 @@ load_source_module(PyObject *name, PyObject *pathname, FILE *fp) goto error; } #endif - cpathname = make_compiled_pathname( - PyUnicode_AS_UNICODE(pathname), - !Py_OptimizeFlag); + cpathname = make_compiled_pathname(pathname, !Py_OptimizeFlag); if (cpathname != NULL) fpc = check_compiled_module(pathname, st.st_mtime, cpathname); @@ -1512,16 +1534,18 @@ static PyObject * get_sourcefile(PyObject *filename) { Py_ssize_t len; - Py_UNICODE *fileuni; + Py_UCS4 *fileuni; PyObject *py; struct stat statbuf; - len = PyUnicode_GET_SIZE(filename); + len = PyUnicode_GET_LENGTH(filename); if (len == 0) Py_RETURN_NONE; /* don't match *.pyc or *.pyo? */ - fileuni = PyUnicode_AS_UNICODE(filename); + fileuni = PyUnicode_AsUCS4Copy(filename); + if (!fileuni) + return NULL; if (len < 5 || fileuni[len-4] != '.' || (fileuni[len-3] != 'p' && fileuni[len-3] != 'P') @@ -1535,7 +1559,7 @@ get_sourcefile(PyObject *filename) py = make_source_pathname(filename); if (py == NULL) { PyErr_Clear(); - py = PyUnicode_FromUnicode(fileuni, len - 1); + py = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, fileuni, len - 1); } if (py == NULL) goto error; @@ -1548,6 +1572,7 @@ get_sourcefile(PyObject *filename) error: PyErr_Clear(); unchanged: + PyMem_Free(fileuni); Py_INCREF(filename); return filename; } @@ -1739,8 +1764,7 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, PyObject *path_hooks, PyObject *path_importer_cache, PyObject **p_path, PyObject **p_loader, struct filedescr **p_fd) { - Py_UNICODE buf[MAXPATHLEN+1]; - Py_ssize_t buflen = MAXPATHLEN+1; + Py_UCS4 buf[MAXPATHLEN+1]; PyObject *path_unicode, *filename; Py_ssize_t len; struct stat statbuf; @@ -1759,15 +1783,15 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, else return 0; - len = PyUnicode_GET_SIZE(path_unicode); - if (len + 2 + PyUnicode_GET_SIZE(name) + MAXSUFFIXSIZE >= buflen) { + len = PyUnicode_GET_LENGTH(path_unicode); + if (!PyUnicode_AsUCS4(path_unicode, buf, PY_ARRAY_LENGTH(buf), 1)) { Py_DECREF(path_unicode); - return 0; /* Too long */ + PyErr_Clear(); + return 0; } - Py_UNICODE_strcpy(buf, PyUnicode_AS_UNICODE(path_unicode)); Py_DECREF(path_unicode); - if (Py_UNICODE_strlen(buf) != len) + if (Py_UCS4_strlen(buf) != len) return 0; /* path contains '\0' */ /* sys.path_hooks import hook */ @@ -1804,10 +1828,14 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, #endif ) buf[len++] = SEP; - Py_UNICODE_strcpy(buf+len, PyUnicode_AS_UNICODE(name)); - len += PyUnicode_GET_SIZE(name); + if (!PyUnicode_AsUCS4(name, buf+len, PY_ARRAY_LENGTH(buf)-len, 1)) { + PyErr_Clear(); + return 0; + } + len += PyUnicode_GET_LENGTH(name); - filename = PyUnicode_FromUnicode(buf, len); + filename = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buf, len); if (filename == NULL) return -1; @@ -1989,6 +2017,12 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, if (p_loader != NULL) *p_loader = NULL; + if (PyUnicode_GET_LENGTH(name) > MAXPATHLEN) { + PyErr_SetString(PyExc_OverflowError, + "module name is too long"); + return NULL; + } + /* sys.meta_path import hook */ if (p_loader != NULL) { PyObject *meta_path; @@ -2704,7 +2738,7 @@ static PyObject *get_parent(PyObject *globals, int level); static PyObject *load_next(PyObject *mod, PyObject *altmod, PyObject *inputname, PyObject **p_outputname, - Py_UNICODE *buf, Py_ssize_t *p_buflen, + Py_UCS4 *buf, Py_ssize_t *p_buflen, Py_ssize_t bufsize); static int mark_miss(PyObject *name); static int ensure_fromlist(PyObject *mod, PyObject *fromlist, @@ -2718,37 +2752,47 @@ static PyObject * import_module_level(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) { - Py_UNICODE buf[MAXPATHLEN+1]; + Py_UCS4 buf[MAXPATHLEN+1]; Py_ssize_t buflen; Py_ssize_t bufsize = MAXPATHLEN+1; PyObject *parent, *head, *next, *tail, *inputname, *outputname; PyObject *parent_name, *ensure_name; - const Py_UNICODE *nameunicode; + Py_ssize_t sep, altsep; - nameunicode = PyUnicode_AS_UNICODE(name); + if (PyUnicode_READY(name)) + return NULL; - if (Py_UNICODE_strchr(nameunicode, SEP) != NULL + sep = PyUnicode_FindChar(name, SEP, 0, PyUnicode_GET_LENGTH(name), 1); + if (sep == -2) + return NULL; #ifdef ALTSEP - || Py_UNICODE_strchr(nameunicode, ALTSEP) != NULL + altsep = PyUnicode_FindChar(name, ALTSEP, 0, PyUnicode_GET_LENGTH(name), 1); + if (altsep == -2) + return NULL; +#else + altsep = -1; #endif - ) { + if (sep != -1 || altsep != -1) + { PyErr_SetString(PyExc_ImportError, "Import by filename is not supported."); return NULL; } parent = get_parent(globals, &parent_name, level); - if (parent == NULL) + if (parent == NULL) { return NULL; + } - buflen = PyUnicode_GET_SIZE(parent_name); - if (buflen+1 > bufsize) { + if (PyUnicode_READY(parent_name)) + return NULL; + buflen = PyUnicode_GET_LENGTH(parent_name); + if (!PyUnicode_AsUCS4(parent_name, buf, PY_ARRAY_LENGTH(buf), 1)) { Py_DECREF(parent_name); PyErr_SetString(PyExc_ValueError, "Module name too long"); return NULL; } - Py_UNICODE_strcpy(buf, PyUnicode_AS_UNICODE(parent_name)); Py_DECREF(parent_name); head = load_next(parent, level < 0 ? Py_None : parent, name, &outputname, @@ -2799,7 +2843,8 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, Py_DECREF(head); - ensure_name = PyUnicode_FromUnicode(buf, Py_UNICODE_strlen(buf)); + ensure_name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buf, Py_UCS4_strlen(buf)); if (ensure_name == NULL) { Py_DECREF(tail); return NULL; @@ -2859,8 +2904,6 @@ PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals static PyObject * get_parent(PyObject *globals, PyObject **p_name, int level) { - Py_UNICODE name[MAXPATHLEN+1]; - const Py_ssize_t bufsize = MAXPATHLEN+1; PyObject *nameobj; static PyObject *namestr = NULL; @@ -2897,7 +2940,7 @@ get_parent(PyObject *globals, PyObject **p_name, int level) "__package__ set to non-string"); return NULL; } - if (PyUnicode_GET_SIZE(pkgname) == 0) { + if (PyUnicode_GET_LENGTH(pkgname) == 0) { if (level > 0) { PyErr_SetString(PyExc_ValueError, "Attempted relative import in non-package"); @@ -2905,12 +2948,8 @@ get_parent(PyObject *globals, PyObject **p_name, int level) } goto return_none; } - if (PyUnicode_GET_SIZE(pkgname)+1 > bufsize) { - PyErr_SetString(PyExc_ValueError, - "Package name too long"); - return NULL; - } - Py_UNICODE_strcpy(name, PyUnicode_AS_UNICODE(pkgname)); + Py_INCREF(pkgname); + nameobj = pkgname; } else { /* __package__ not set, so figure it out and set it */ modname = PyDict_GetItem(globals, namestr); @@ -2922,74 +2961,71 @@ get_parent(PyObject *globals, PyObject **p_name, int level) /* __path__ is set, so modname is already the package name */ int error; - if (PyUnicode_GET_SIZE(modname)+1 > bufsize) { - PyErr_SetString(PyExc_ValueError, - "Module name too long"); - return NULL; - } - Py_UNICODE_strcpy(name, PyUnicode_AS_UNICODE(modname)); error = PyDict_SetItem(globals, pkgstr, modname); if (error) { PyErr_SetString(PyExc_ValueError, "Could not set __package__"); return NULL; } + Py_INCREF(modname); + nameobj = modname; } else { /* Normal module, so work out the package name if any */ - Py_UNICODE *start = PyUnicode_AS_UNICODE(modname); - Py_UNICODE *lastdot = Py_UNICODE_strrchr(start, '.'); Py_ssize_t len; - int error; - if (lastdot == NULL && level > 0) { - PyErr_SetString(PyExc_ValueError, - "Attempted relative import in non-package"); + len = PyUnicode_FindChar(modname, '.', + 0, PyUnicode_GET_LENGTH(modname), -1); + if (len == -2) return NULL; - } - if (lastdot == NULL) { - error = PyDict_SetItem(globals, pkgstr, Py_None); - if (error) { + if (len < 0) { + if (level > 0) { + PyErr_SetString(PyExc_ValueError, + "Attempted relative import in non-package"); + return NULL; + } + if (PyDict_SetItem(globals, pkgstr, Py_None)) { PyErr_SetString(PyExc_ValueError, "Could not set __package__"); return NULL; } goto return_none; } - len = lastdot - start; - if (len+1 > bufsize) { + pkgname = PyUnicode_Substring(modname, 0, len); + if (pkgname == NULL) + return NULL; + if (PyDict_SetItem(globals, pkgstr, pkgname)) { + Py_DECREF(pkgname); PyErr_SetString(PyExc_ValueError, - "Module name too long"); + "Could not set __package__"); return NULL; } - Py_UNICODE_strncpy(name, start, len); - name[len] = '\0'; - pkgname = PyUnicode_FromUnicode(name, len); - if (pkgname == NULL) { + nameobj = pkgname; + } + } + if (level > 1) { + Py_ssize_t dot, end = PyUnicode_GET_LENGTH(nameobj); + PyObject *newname; + while (--level > 0) { + dot = PyUnicode_FindChar(nameobj, '.', 0, end, -1); + if (dot == -2) { + Py_DECREF(nameobj); return NULL; } - error = PyDict_SetItem(globals, pkgstr, pkgname); - Py_DECREF(pkgname); - if (error) { + if (dot < 0) { + Py_DECREF(nameobj); PyErr_SetString(PyExc_ValueError, - "Could not set __package__"); + "Attempted relative import beyond " + "toplevel package"); return NULL; } + end = dot; } - } - while (--level > 0) { - Py_UNICODE *dot = Py_UNICODE_strrchr(name, '.'); - if (dot == NULL) { - PyErr_SetString(PyExc_ValueError, - "Attempted relative import beyond " - "toplevel package"); + newname = PyUnicode_Substring(nameobj, 0, end); + Py_DECREF(nameobj); + if (newname == NULL) return NULL; - } - *dot = '\0'; + nameobj = newname; } - nameobj = PyUnicode_FromUnicode(name, Py_UNICODE_strlen(name)); - if (nameobj == NULL) - return NULL; - modules = PyImport_GetModuleDict(); parent = PyDict_GetItem(modules, nameobj); if (parent == NULL) { @@ -3021,7 +3057,7 @@ get_parent(PyObject *globals, PyObject **p_name, int level) If this is violated... Who cares? */ return_none: - nameobj = PyUnicode_FromUnicode(NULL, 0); + nameobj = PyUnicode_New(0, 0); if (nameobj == NULL) return NULL; *p_name = nameobj; @@ -3032,28 +3068,28 @@ return_none: static PyObject * load_next(PyObject *mod, PyObject *altmod, PyObject *inputname, PyObject **p_outputname, - Py_UNICODE *buf, Py_ssize_t *p_buflen, Py_ssize_t bufsize) + Py_UCS4 *buf, Py_ssize_t *p_buflen, Py_ssize_t bufsize) { - const Py_UNICODE *dot; + Py_UCS4 *dot; Py_ssize_t len; - Py_UNICODE *p; + Py_UCS4 *p; PyObject *fullname, *name, *result, *mark_name; - const Py_UNICODE *nameuni; + const Py_UCS4 *nameuni; *p_outputname = NULL; - if (PyUnicode_GET_SIZE(inputname) == 0) { + if (PyUnicode_GET_LENGTH(inputname) == 0) { /* completely empty module name should only happen in 'from . import' (or '__import__("")')*/ Py_INCREF(mod); return mod; } - nameuni = PyUnicode_AS_UNICODE(inputname); + nameuni = PyUnicode_AsUCS4Copy(inputname); if (nameuni == NULL) return NULL; - dot = Py_UNICODE_strchr(nameuni, '.'); + dot = Py_UCS4_strchr(nameuni, '.'); if (dot != NULL) { len = dot - nameuni; if (len == 0) { @@ -3063,7 +3099,7 @@ load_next(PyObject *mod, PyObject *altmod, } } else - len = PyUnicode_GET_SIZE(inputname); + len = PyUnicode_GET_LENGTH(inputname); if (*p_buflen+len+1 >= bufsize) { PyErr_SetString(PyExc_ValueError, @@ -3076,14 +3112,16 @@ load_next(PyObject *mod, PyObject *altmod, *p++ = '.'; *p_buflen += 1; } - Py_UNICODE_strncpy(p, nameuni, len); + Py_UCS4_strncpy(p, nameuni, len); p[len] = '\0'; *p_buflen += len; - fullname = PyUnicode_FromUnicode(buf, *p_buflen); + fullname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buf, *p_buflen); if (fullname == NULL) return NULL; - name = PyUnicode_FromUnicode(p, len); + name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + p, len); if (name == NULL) { Py_DECREF(fullname); return NULL; @@ -3096,7 +3134,8 @@ load_next(PyObject *mod, PyObject *altmod, result = import_submodule(altmod, name, name); Py_DECREF(name); if (result != NULL && result != Py_None) { - mark_name = PyUnicode_FromUnicode(buf, *p_buflen); + mark_name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buf, *p_buflen); if (mark_name == NULL) { Py_DECREF(result); return NULL; @@ -3107,7 +3146,7 @@ load_next(PyObject *mod, PyObject *altmod, return NULL; } Py_DECREF(mark_name); - Py_UNICODE_strncpy(buf, nameuni, len); + Py_UCS4_strncpy(buf, nameuni, len); buf[len] = '\0'; *p_buflen = len; } @@ -3125,7 +3164,8 @@ load_next(PyObject *mod, PyObject *altmod, } if (dot != NULL) { - *p_outputname = PyUnicode_FromUnicode(dot+1, Py_UNICODE_strlen(dot+1)); + *p_outputname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + dot+1, Py_UCS4_strlen(dot+1)); if (*p_outputname == NULL) { Py_DECREF(result); return NULL; @@ -3166,7 +3206,7 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *name, Py_DECREF(item); return 0; } - if (PyUnicode_AS_UNICODE(item)[0] == '*') { + if (PyUnicode_READ_CHAR(item, 0) == '*') { PyObject *all; Py_DECREF(item); /* See if the package defines __all__ */ @@ -3304,7 +3344,7 @@ PyImport_ReloadModule(PyObject *m) PyObject *modules = PyImport_GetModuleDict(); PyObject *path_list = NULL, *loader = NULL, *existing_m = NULL; PyObject *nameobj, *bufobj, *subnameobj; - Py_UNICODE *name, *subname; + Py_UCS4 *name = NULL, *subname; struct filedescr *fdp; FILE *fp = NULL; PyObject *newm = NULL; @@ -3321,7 +3361,7 @@ PyImport_ReloadModule(PyObject *m) return NULL; } nameobj = PyModule_GetNameObject(m); - if (nameobj == NULL) + if (nameobj == NULL || PyUnicode_READY(nameobj) == -1) return NULL; if (m != PyDict_GetItem(modules, nameobj)) { PyErr_Format(PyExc_ImportError, @@ -3343,8 +3383,12 @@ PyImport_ReloadModule(PyObject *m) return NULL; } - name = PyUnicode_AS_UNICODE(nameobj); - subname = Py_UNICODE_strrchr(name, '.'); + name = PyUnicode_AsUCS4Copy(nameobj); + if (!name) { + Py_DECREF(nameobj); + return NULL; + } + subname = Py_UCS4_strrchr(name, '.'); if (subname == NULL) { Py_INCREF(nameobj); subnameobj = nameobj; @@ -3353,7 +3397,8 @@ PyImport_ReloadModule(PyObject *m) PyObject *parentname, *parent; Py_ssize_t len; len = subname - name; - parentname = PyUnicode_FromUnicode(name, len); + parentname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + name, len); if (parentname == NULL) { goto error; } @@ -3370,8 +3415,9 @@ PyImport_ReloadModule(PyObject *m) if (path_list == NULL) PyErr_Clear(); subname++; - len = PyUnicode_GET_SIZE(nameobj) - (len + 1); - subnameobj = PyUnicode_FromUnicode(subname, len); + len = PyUnicode_GET_LENGTH(nameobj) - (len + 1); + subnameobj = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + subname, len); } if (subnameobj == NULL) goto error; @@ -3403,6 +3449,7 @@ PyImport_ReloadModule(PyObject *m) error: imp_modules_reloading_clear(); Py_DECREF(nameobj); + PyMem_Free(name); return newm; } @@ -3910,9 +3957,7 @@ imp_cache_from_source(PyObject *self, PyObject *args, PyObject *kws) return NULL; } - cpathname = make_compiled_pathname( - PyUnicode_AS_UNICODE(pathname), - debug); + cpathname = make_compiled_pathname(pathname, debug); Py_DECREF(pathname); if (cpathname == NULL) { @@ -4105,7 +4150,7 @@ NullImporter_init(NullImporter *self, PyObject *args, PyObject *kwds) &pathobj)) return -1; - if (PyUnicode_GET_SIZE(pathobj) == 0) { + if (PyUnicode_GET_LENGTH(pathobj) == 0) { PyErr_SetString(PyExc_ImportError, "empty pathname"); return -1; } -- cgit v1.2.1 From 593041f3e6ac869b37c7da4355b69b1adc173b37 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Wed, 28 Sep 2011 21:49:49 +0200 Subject: Rename new macros to conform to naming rules (function macros have "Py" prefix, not "PY"). --- Python/import.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 4afdb7058c..9f38061723 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1784,7 +1784,7 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, return 0; len = PyUnicode_GET_LENGTH(path_unicode); - if (!PyUnicode_AsUCS4(path_unicode, buf, PY_ARRAY_LENGTH(buf), 1)) { + if (!PyUnicode_AsUCS4(path_unicode, buf, Py_ARRAY_LENGTH(buf), 1)) { Py_DECREF(path_unicode); PyErr_Clear(); return 0; @@ -1828,7 +1828,7 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, #endif ) buf[len++] = SEP; - if (!PyUnicode_AsUCS4(name, buf+len, PY_ARRAY_LENGTH(buf)-len, 1)) { + if (!PyUnicode_AsUCS4(name, buf+len, Py_ARRAY_LENGTH(buf)-len, 1)) { PyErr_Clear(); return 0; } @@ -2787,7 +2787,7 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, if (PyUnicode_READY(parent_name)) return NULL; buflen = PyUnicode_GET_LENGTH(parent_name); - if (!PyUnicode_AsUCS4(parent_name, buf, PY_ARRAY_LENGTH(buf), 1)) { + if (!PyUnicode_AsUCS4(parent_name, buf, Py_ARRAY_LENGTH(buf), 1)) { Py_DECREF(parent_name); PyErr_SetString(PyExc_ValueError, "Module name too long"); -- cgit v1.2.1 From 2ab1e534bf781097c59588983df13f737c912576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sat, 1 Oct 2011 16:35:40 +0200 Subject: Issue 13085: Fix some memory leaks. Patch by Stefan Krah. --- Python/import.c | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 9f38061723..901b733480 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1564,8 +1564,10 @@ get_sourcefile(PyObject *filename) if (py == NULL) goto error; - if (_Py_stat(py, &statbuf) == 0 && S_ISREG(statbuf.st_mode)) + if (_Py_stat(py, &statbuf) == 0 && S_ISREG(statbuf.st_mode)) { + PyMem_Free(fileuni); return py; + } Py_DECREF(py); goto unchanged; @@ -3074,7 +3076,7 @@ load_next(PyObject *mod, PyObject *altmod, Py_ssize_t len; Py_UCS4 *p; PyObject *fullname, *name, *result, *mark_name; - const Py_UCS4 *nameuni; + Py_UCS4 *nameuni; *p_outputname = NULL; @@ -3095,7 +3097,7 @@ load_next(PyObject *mod, PyObject *altmod, if (len == 0) { PyErr_SetString(PyExc_ValueError, "Empty module name"); - return NULL; + goto error; } } else @@ -3104,7 +3106,7 @@ load_next(PyObject *mod, PyObject *altmod, if (*p_buflen+len+1 >= bufsize) { PyErr_SetString(PyExc_ValueError, "Module name too long"); - return NULL; + goto error; } p = buf + *p_buflen; @@ -3119,12 +3121,12 @@ load_next(PyObject *mod, PyObject *altmod, fullname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, *p_buflen); if (fullname == NULL) - return NULL; + goto error; name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, p, len); if (name == NULL) { Py_DECREF(fullname); - return NULL; + goto error; } result = import_submodule(mod, name, fullname); Py_DECREF(fullname); @@ -3138,12 +3140,12 @@ load_next(PyObject *mod, PyObject *altmod, buf, *p_buflen); if (mark_name == NULL) { Py_DECREF(result); - return NULL; + goto error; } if (mark_miss(mark_name) != 0) { Py_DECREF(result); Py_DECREF(mark_name); - return NULL; + goto error; } Py_DECREF(mark_name); Py_UCS4_strncpy(buf, nameuni, len); @@ -3154,13 +3156,13 @@ load_next(PyObject *mod, PyObject *altmod, else Py_DECREF(name); if (result == NULL) - return NULL; + goto error; if (result == Py_None) { Py_DECREF(result); PyErr_Format(PyExc_ImportError, "No module named %R", inputname); - return NULL; + goto error; } if (dot != NULL) { @@ -3168,11 +3170,17 @@ load_next(PyObject *mod, PyObject *altmod, dot+1, Py_UCS4_strlen(dot+1)); if (*p_outputname == NULL) { Py_DECREF(result); - return NULL; + goto error; } } +out: + PyMem_Free(nameuni); return result; + +error: + PyMem_Free(nameuni); + return NULL; } static int -- cgit v1.2.1 From 7a98f5f5ced85e78dc79169bfaeaa1b3adfa0596 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 2 Oct 2011 13:19:16 -0400 Subject: remove unused label --- Python/import.c | 1 - 1 file changed, 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 901b733480..9f094c0da0 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3174,7 +3174,6 @@ load_next(PyObject *mod, PyObject *altmod, } } -out: PyMem_Free(nameuni); return result; -- cgit v1.2.1 From 0aeb34c31cc7bfc0c0fd5595b017e3f9dc0a93ef Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 6 Oct 2011 02:39:42 +0200 Subject: Fix find_module_path(): make the string ready --- Python/import.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 9f094c0da0..5f84ac2f74 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1785,6 +1785,9 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, else return 0; + if (PyUnicode_READY(path_unicode)) + return -1; + len = PyUnicode_GET_LENGTH(path_unicode); if (!PyUnicode_AsUCS4(path_unicode, buf, Py_ARRAY_LENGTH(buf), 1)) { Py_DECREF(path_unicode); -- cgit v1.2.1 From dcacb24f386ae1b107462181af42b6826ec15fb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 9 Oct 2011 10:38:36 +0200 Subject: =?UTF-8?q?Add=20API=20for=20static=20strings,=20primarily=20good?= =?UTF-8?q?=20for=20identifiers.=20Thanks=20to=20Konrad=20Sch=C3=B6bel=20a?= =?UTF-8?q?nd=20Jasper=20Schulz=20for=20helping=20with=20the=20mass-editin?= =?UTF-8?q?g.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Python/import.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 5f84ac2f74..6eca90a0f7 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1801,6 +1801,7 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, /* sys.path_hooks import hook */ if (p_loader != NULL) { + _Py_identifier(find_module); PyObject *importer; importer = get_path_importer(path_importer_cache, @@ -1811,8 +1812,8 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, /* Note: importer is a borrowed reference */ if (importer != Py_None) { PyObject *loader; - loader = PyObject_CallMethod(importer, - "find_module", "O", fullname); + loader = _PyObject_CallMethodId(importer, + &PyId_find_module, "O", fullname); if (loader == NULL) return -1; /* error */ if (loader != Py_None) { @@ -2030,6 +2031,7 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, /* sys.meta_path import hook */ if (p_loader != NULL) { + _Py_identifier(find_module); PyObject *meta_path; meta_path = PySys_GetObject("meta_path"); @@ -2044,7 +2046,7 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, for (i = 0; i < npath; i++) { PyObject *loader; PyObject *hook = PyList_GetItem(meta_path, i); - loader = PyObject_CallMethod(hook, "find_module", + loader = _PyObject_CallMethodId(hook, &PyId_find_module, "OO", fullname, search_path_list != NULL ? search_path_list : Py_None); @@ -2454,12 +2456,13 @@ load_module(PyObject *name, FILE *fp, PyObject *pathname, int type, PyObject *lo break; case IMP_HOOK: { + _Py_identifier(load_module); if (loader == NULL) { PyErr_SetString(PyExc_ImportError, "import hook without loader"); return NULL; } - m = PyObject_CallMethod(loader, "load_module", "O", name); + m = _PyObject_CallMethodId(loader, &PyId_load_module, "O", name); break; } -- cgit v1.2.1 From c3eb93fe1ce5f5ff2666fb9f34a6fd7c46279554 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Mon, 10 Oct 2011 18:11:30 +0200 Subject: Use identifier API for PyObject_GetAttrString. --- Python/import.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 6eca90a0f7..3d6c6ed1d4 100644 --- a/Python/import.c +++ b/Python/import.c @@ -154,7 +154,7 @@ static const struct filedescr _PyImport_StandardFiletab[] = { }; static PyObject *initstr = NULL; - +_Py_identifier(__path__); /* Initialize things */ @@ -248,8 +248,9 @@ _PyImportHooks_Init(void) PySys_WriteStderr("# can't import zipimport\n"); } else { - PyObject *zipimporter = PyObject_GetAttrString(zimpimport, - "zipimporter"); + _Py_identifier(zipimporter); + PyObject *zipimporter = _PyObject_GetAttrId(zimpimport, + &PyId_zipimporter); Py_DECREF(zimpimport); if (zipimporter == NULL) { PyErr_Clear(); /* No zipimporter object -- okay */ @@ -3203,7 +3204,7 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *name, PyObject *fullname; Py_ssize_t fromlist_len; - if (!PyObject_HasAttrString(mod, "__path__")) + if (!_PyObject_HasAttrId(mod, &PyId___path__)) return 1; fromlist_len = PySequence_Size(fromlist); @@ -3221,11 +3222,12 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *name, } if (PyUnicode_READ_CHAR(item, 0) == '*') { PyObject *all; + _Py_identifier(__all__); Py_DECREF(item); /* See if the package defines __all__ */ if (recursive) continue; /* Avoid endless recursion */ - all = PyObject_GetAttrString(mod, "__all__"); + all = _PyObject_GetAttrId(mod, &PyId___all__); if (all == NULL) PyErr_Clear(); else { @@ -3313,7 +3315,7 @@ import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) if (mod == Py_None) path_list = NULL; else { - path_list = PyObject_GetAttrString(mod, "__path__"); + path_list = _PyObject_GetAttrId(mod, &PyId___path__); if (path_list == NULL) { PyErr_Clear(); Py_INCREF(Py_None); @@ -3424,7 +3426,7 @@ PyImport_ReloadModule(PyObject *m) goto error; } Py_DECREF(parentname); - path_list = PyObject_GetAttrString(parent, "__path__"); + path_list = _PyObject_GetAttrId(parent, &PyId___path__); if (path_list == NULL) PyErr_Clear(); subname++; -- cgit v1.2.1 From 9fde12ef0a3cbe4260897de13f054ec630456992 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 11 Oct 2011 21:55:01 +0200 Subject: Use PyUnicode_AsUnicodeAndSize() instead of PyUnicode_GET_SIZE() --- Python/import.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 3d6c6ed1d4..6f564a65da 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2282,6 +2282,8 @@ case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) WIN32_FIND_DATAW data; HANDLE h; int cmp; + wchar_t *wname; + Py_ssizet wname_len; if (Py_GETENV("PYTHONCASEOK") != NULL) return 1; @@ -2294,9 +2296,12 @@ case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) return 0; } FindClose(h); - cmp = wcsncmp(data.cFileName, - PyUnicode_AS_UNICODE(name), - PyUnicode_GET_SIZE(name)); + + wname = PyUnicode_AsUnicodeAndSize(name, &wname_len); + if (wname == NULL) + return -1; + + cmp = wcsncmp(data.cFileName, wname, wname_len); return cmp == 0; #elif defined(USE_CASE_OK_BYTES) int match; -- cgit v1.2.1 From f716b4e95cf727dc9fe4a592c38e86b73ca16c74 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 11 Oct 2011 22:27:13 +0200 Subject: Fix typo in import.c --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 6f564a65da..d937cc165a 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2283,7 +2283,7 @@ case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) HANDLE h; int cmp; wchar_t *wname; - Py_ssizet wname_len; + Py_ssize_t wname_len; if (Py_GETENV("PYTHONCASEOK") != NULL) return 1; -- cgit v1.2.1 From f5ad3b280b43227eb0e3fa63d89490a58ba9c28b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Fri, 14 Oct 2011 10:20:37 +0200 Subject: Rename _Py_identifier to _Py_IDENTIFIER. --- Python/import.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index d937cc165a..6e3e0f181e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -154,7 +154,7 @@ static const struct filedescr _PyImport_StandardFiletab[] = { }; static PyObject *initstr = NULL; -_Py_identifier(__path__); +_Py_IDENTIFIER(__path__); /* Initialize things */ @@ -248,7 +248,7 @@ _PyImportHooks_Init(void) PySys_WriteStderr("# can't import zipimport\n"); } else { - _Py_identifier(zipimporter); + _Py_IDENTIFIER(zipimporter); PyObject *zipimporter = _PyObject_GetAttrId(zimpimport, &PyId_zipimporter); Py_DECREF(zimpimport); @@ -1802,7 +1802,7 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, /* sys.path_hooks import hook */ if (p_loader != NULL) { - _Py_identifier(find_module); + _Py_IDENTIFIER(find_module); PyObject *importer; importer = get_path_importer(path_importer_cache, @@ -2032,7 +2032,7 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, /* sys.meta_path import hook */ if (p_loader != NULL) { - _Py_identifier(find_module); + _Py_IDENTIFIER(find_module); PyObject *meta_path; meta_path = PySys_GetObject("meta_path"); @@ -2462,7 +2462,7 @@ load_module(PyObject *name, FILE *fp, PyObject *pathname, int type, PyObject *lo break; case IMP_HOOK: { - _Py_identifier(load_module); + _Py_IDENTIFIER(load_module); if (loader == NULL) { PyErr_SetString(PyExc_ImportError, "import hook without loader"); @@ -3227,7 +3227,7 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *name, } if (PyUnicode_READ_CHAR(item, 0) == '*') { PyObject *all; - _Py_identifier(__all__); + _Py_IDENTIFIER(__all__); Py_DECREF(item); /* See if the package defines __all__ */ if (recursive) -- cgit v1.2.1 From 5263d9e0be0bf742276426378f96f8b6e6687872 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 17 Oct 2011 19:28:44 +0200 Subject: Issue #13146: Writing a pyc file is now atomic under POSIX. --- Python/import.c | 45 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 6e3e0f181e..0db7e43d29 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1284,7 +1284,8 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, #ifdef MS_WINDOWS int fd; #else - PyObject *cpathbytes; + PyObject *cpathbytes, *cpathbytes_tmp; + Py_ssize_t cpathbytes_len; #endif PyObject *dirname; Py_UCS4 *dirsep; @@ -1345,13 +1346,25 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, else fp = NULL; #else + /* Under POSIX, we first write to a tmp file and then take advantage + of atomic renaming. */ cpathbytes = PyUnicode_EncodeFSDefault(cpathname); if (cpathbytes == NULL) { PyErr_Clear(); return; } + cpathbytes_len = PyBytes_GET_SIZE(cpathbytes); + cpathbytes_tmp = PyBytes_FromStringAndSize(NULL, cpathbytes_len + 4); + if (cpathbytes_tmp == NULL) { + Py_DECREF(cpathbytes); + PyErr_Clear(); + return; + } + memcpy(PyBytes_AS_STRING(cpathbytes_tmp), PyBytes_AS_STRING(cpathbytes), + cpathbytes_len); + memcpy(PyBytes_AS_STRING(cpathbytes_tmp) + cpathbytes_len, ".tmp", 4); - fp = open_exclusive(PyBytes_AS_STRING(cpathbytes), mode); + fp = open_exclusive(PyBytes_AS_STRING(cpathbytes_tmp), mode); #endif if (fp == NULL) { if (Py_VerboseFlag) @@ -1359,6 +1372,7 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, "# can't create %R\n", cpathname); #ifndef MS_WINDOWS Py_DECREF(cpathbytes); + Py_DECREF(cpathbytes_tmp); #endif return; } @@ -1366,6 +1380,11 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, /* First write a 0 for mtime */ PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION); PyMarshal_WriteObjectToFile((PyObject *)co, fp, Py_MARSHAL_VERSION); + fflush(fp); + /* Now write the true mtime */ + fseek(fp, 4L, 0); + assert(mtime < LONG_MAX); + PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION); if (fflush(fp) != 0 || ferror(fp)) { if (Py_VerboseFlag) PySys_FormatStderr("# can't write %R\n", cpathname); @@ -1374,20 +1393,28 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, #ifdef MS_WINDOWS (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname)); #else - (void) unlink(PyBytes_AS_STRING(cpathbytes)); + (void) unlink(PyBytes_AS_STRING(cpathbytes_tmp)); Py_DECREF(cpathbytes); + Py_DECREF(cpathbytes_tmp); #endif return; } + fclose(fp); + /* Under POSIX, do an atomic rename */ #ifndef MS_WINDOWS + if (rename(PyBytes_AS_STRING(cpathbytes_tmp), + PyBytes_AS_STRING(cpathbytes))) { + if (Py_VerboseFlag) + PySys_FormatStderr("# can't write %R\n", cpathname); + /* Don't keep tmp file */ + unlink(PyBytes_AS_STRING(cpathbytes_tmp)); + Py_DECREF(cpathbytes); + Py_DECREF(cpathbytes_tmp); + return; + } Py_DECREF(cpathbytes); + Py_DECREF(cpathbytes_tmp); #endif - /* Now write the true mtime */ - fseek(fp, 4L, 0); - assert(mtime < LONG_MAX); - PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION); - fflush(fp); - fclose(fp); if (Py_VerboseFlag) PySys_FormatStderr("# wrote %R\n", cpathname); } -- cgit v1.2.1 From 787a4931e7a65741deb81c532dfb184af13a0e20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 23 Oct 2011 17:29:08 +0200 Subject: Reformulate make_compiled_pathname in terms of unicode objects. --- Python/import.c | 177 +++++++++++++++++++------------------------------------- 1 file changed, 61 insertions(+), 116 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 0db7e43d29..1101c920f5 100644 --- a/Python/import.c +++ b/Python/import.c @@ -904,6 +904,25 @@ rightmost_sep(Py_UCS4 *s) return found; } +/* Like rightmost_sep, but operate on unicode objects. */ +static Py_ssize_t +rightmost_sep_obj(PyObject* o) +{ + Py_ssize_t found, i; + Py_UCS4 c; + for (found = -1, i = 0; i < PyUnicode_GET_LENGTH(o); i++) { + c = PyUnicode_READ_CHAR(o, i); + if (c == SEP +#ifdef ALTSEP + || c == ALTSEP +#endif + ) + { + found = i; + } + } + return found; +} /* Given a pathname for a Python source file, fill a buffer with the pathname for the corresponding compiled file. Return the pathname @@ -915,123 +934,49 @@ rightmost_sep(Py_UCS4 *s) static PyObject* make_compiled_pathname(PyObject *pathstr, int debug) { - Py_UCS4 *pathname; - Py_UCS4 buf[MAXPATHLEN]; - size_t buflen = (size_t)MAXPATHLEN; - size_t len; - size_t i, save; - Py_UCS4 *pos; - int sep = SEP; - - pathname = PyUnicode_AsUCS4Copy(pathstr); - if (!pathname) - return NULL; - len = Py_UCS4_strlen(pathname); - - /* Sanity check that the buffer has roughly enough space to hold what - will eventually be the full path to the compiled file. The 5 extra - bytes include the slash afer __pycache__, the two extra dots, the - extra trailing character ('c' or 'o') and null. This isn't exact - because the contents of the buffer can affect how many actual - characters of the string get into the buffer. We'll do a final - sanity check before writing the extension to ensure we do not - overflow the buffer. - */ - if (len + Py_UCS4_strlen(CACHEDIR_UNICODE) + Py_UCS4_strlen(PYC_TAG_UNICODE) + 5 > buflen) { - PyMem_Free(pathname); - return NULL; - } - - /* Find the last path separator and copy everything from the start of - the source string up to and including the separator. - */ - pos = rightmost_sep(pathname); - if (pos == NULL) { - i = 0; - } - else { - sep = *pos; - i = pos - pathname + 1; - Py_UCS4_strncpy(buf, pathname, i); - } - - save = i; - buf[i++] = '\0'; - /* Add __pycache__/ */ - Py_UCS4_strcat(buf, CACHEDIR_UNICODE); - i += Py_UCS4_strlen(CACHEDIR_UNICODE) - 1; - buf[i++] = sep; - buf[i] = '\0'; - /* Add the base filename, but remove the .py or .pyw extension, since - the tag name must go before the extension. - */ - Py_UCS4_strcat(buf, pathname + save); - pos = Py_UCS4_strrchr(buf + i, '.'); - if (pos != NULL) - *++pos = '\0'; - - /* pathname is not used from here on. */ - PyMem_Free(pathname); - - Py_UCS4_strcat(buf, PYC_TAG_UNICODE); - /* The length test above assumes that we're only adding one character - to the end of what would normally be the extension. What if there - is no extension, or the string ends in '.' or '.p', and otherwise - fills the buffer? By appending 4 more characters onto the string - here, we could overrun the buffer. - - As a simple example, let's say buflen=32 and the input string is - 'xxx.py'. strlen() would be 6 and the test above would yield: - - (6 + 11 + 10 + 5 == 32) > 32 - - which is false and so the name mangling would continue. This would - be fine because we'd end up with this string in buf: - - __pycache__/xxx.cpython-32.pyc\0 - - strlen(of that) == 30 + the nul fits inside a 32 character buffer. - We can even handle an input string of say 'xxxxx' above because - that's (5 + 11 + 10 + 5 == 31) > 32 which is also false. Name - mangling that yields: - - __pycache__/xxxxxcpython-32.pyc\0 - - which is 32 characters including the nul, and thus fits in the - buffer. However, an input string of 'xxxxxx' would yield a result - string of: - - __pycache__/xxxxxxcpython-32.pyc\0 - - which is 33 characters long (including the nul), thus overflowing - the buffer, even though the first test would fail, i.e.: the input - string is also 6 characters long, so 32 > 32 is false. - - The reason the first test fails but we still overflow the buffer is - that the test above only expects to add one extra character to be - added to the extension, and here we're adding three (pyc). We - don't add the first dot, so that reclaims one of expected - positions, leaving us overflowing by 1 byte (3 extra - 1 reclaimed - dot - 1 expected extra == 1 overflowed). - - The best we can do is ensure that we still have enough room in the - target buffer before we write the extension. Because it's always - only the extension that can cause the overflow, and never the other - path bytes we've written, it's sufficient to just do one more test - here. Still, the assertion that follows can't hurt. - */ -#if 0 - printf("strlen(buf): %d; buflen: %d\n", (int)strlen(buf), (int)buflen); -#endif - len = Py_UCS4_strlen(buf); - if (len + 5 > buflen) + PyObject *result; + Py_ssize_t fname, ext, len, i, pos, taglen; + Py_ssize_t pycache_len = sizeof("__pycache__/") - 1; + int kind; + void *data; + + /* Compute the output string size. */ + len = PyUnicode_GET_LENGTH(pathstr); + /* If there is no separator, this returns -1, so + lastsep will be 0. */ + fname = rightmost_sep_obj(pathstr) + 1; + ext = fname - 1; + for(i = fname; i < len; i++) + if (PyUnicode_READ_CHAR(pathstr, i) == '.') + ext = i + 1; + if (ext < fname) + /* No dot in filename; use entire filename */ + ext = len; + + /* result = pathstr[:fname] + "__pycache__" + SEP + + pathstr[fname:ext] + tag + ".py[co]" */ + taglen = strlen(pyc_tag); + result = PyUnicode_New(ext + pycache_len + taglen + 4, + PyUnicode_MAX_CHAR_VALUE(pathstr)); + if (!result) return NULL; - buf[len] = '.'; len++; - buf[len] = 'p'; len++; - buf[len] = 'y'; len++; - buf[len] = debug ? 'c' : 'o'; len++; - assert(len <= buflen); - return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, len); + kind = PyUnicode_KIND(result); + data = PyUnicode_DATA(result); + PyUnicode_CopyCharacters(result, 0, pathstr, 0, fname); + pos = fname; + for (i = 0; i < pycache_len - 1; i++) + PyUnicode_WRITE(kind, data, pos++, "__pycache__"[i]); + PyUnicode_WRITE(kind, data, pos++, SEP); + PyUnicode_CopyCharacters(result, pos, pathstr, + fname, ext - fname); + pos += ext - fname; + for (i = 0; pyc_tag[i]; i++) + PyUnicode_WRITE(kind, data, pos++, pyc_tag[i]); + PyUnicode_WRITE(kind, data, pos++, '.'); + PyUnicode_WRITE(kind, data, pos++, 'p'); + PyUnicode_WRITE(kind, data, pos++, 'y'); + PyUnicode_WRITE(kind, data, pos++, debug ? 'c' : 'o'); + return result; } -- cgit v1.2.1 From 17bab0f4a5891ac06360ae32cc1b1f9ab746cf52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 23 Oct 2011 17:35:46 +0200 Subject: Add ready checks for make_compiled_pathname. --- Python/import.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 1101c920f5..5dcaf3a40e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -929,7 +929,10 @@ rightmost_sep_obj(PyObject* o) for the compiled file, or NULL if there's no space in the buffer. Doesn't set an exception. - foo.py -> __pycache__/foo..pyc */ + foo.py -> __pycache__/foo..pyc + + pathstr is assumed to be "ready". +*/ static PyObject* make_compiled_pathname(PyObject *pathstr, int debug) @@ -1458,6 +1461,8 @@ load_source_module(PyObject *name, PyObject *pathname, FILE *fp) goto error; } #endif + if (PyUnicode_READY(pathname) < 0) + return NULL; cpathname = make_compiled_pathname(pathname, !Py_OptimizeFlag); if (cpathname != NULL) @@ -3949,6 +3954,9 @@ imp_cache_from_source(PyObject *self, PyObject *args, PyObject *kws) return NULL; } + if (PyUnicode_READY(pathname) < 0) + return NULL; + cpathname = make_compiled_pathname(pathname, debug); Py_DECREF(pathname); -- cgit v1.2.1 From ff4c4b6f318abc223cbae8920ee2286d111d8bce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 23 Oct 2011 18:05:06 +0200 Subject: Rewrite make_source_pathname using Unicode API. --- Python/import.c | 79 ++++++++++++++++++++++++++++++--------------------------- 1 file changed, 41 insertions(+), 38 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 5dcaf3a40e..a20871efcc 100644 --- a/Python/import.c +++ b/Python/import.c @@ -906,11 +906,11 @@ rightmost_sep(Py_UCS4 *s) /* Like rightmost_sep, but operate on unicode objects. */ static Py_ssize_t -rightmost_sep_obj(PyObject* o) +rightmost_sep_obj(PyObject* o, Py_ssize_t start, Py_ssize_t end) { Py_ssize_t found, i; Py_UCS4 c; - for (found = -1, i = 0; i < PyUnicode_GET_LENGTH(o); i++) { + for (found = -1, i = start; i < end; i++) { c = PyUnicode_READ_CHAR(o, i); if (c == SEP #ifdef ALTSEP @@ -947,7 +947,7 @@ make_compiled_pathname(PyObject *pathstr, int debug) len = PyUnicode_GET_LENGTH(pathstr); /* If there is no separator, this returns -1, so lastsep will be 0. */ - fname = rightmost_sep_obj(pathstr) + 1; + fname = rightmost_sep_obj(pathstr, 0, len) + 1; ext = fname - 1; for(i = fname; i < len; i++) if (PyUnicode_READ_CHAR(pathstr, i) == '.') @@ -992,63 +992,66 @@ make_compiled_pathname(PyObject *pathstr, int debug) (...)/__pycache__/foo..pyc -> (...)/foo.py */ static PyObject* -make_source_pathname(PyObject *pathobj) +make_source_pathname(PyObject *path) { - Py_UCS4 buf[MAXPATHLEN]; - Py_UCS4 *pathname; - Py_UCS4 *left, *right, *dot0, *dot1, sep; - size_t i, j; + Py_ssize_t left, right, dot0, dot1, len; + Py_ssize_t i, j; + PyObject *result; + int kind; + void *data; - if (PyUnicode_GET_LENGTH(pathobj) > MAXPATHLEN) - return NULL; - pathname = PyUnicode_AsUCS4Copy(pathobj); - if (!pathname) + len = PyUnicode_GET_LENGTH(path); + if (len > MAXPATHLEN) return NULL; /* Look back two slashes from the end. In between these two slashes must be the string __pycache__ or this is not a PEP 3147 style path. It's possible for there to be only one slash. */ - right = rightmost_sep(pathname); - if (right == NULL) + right = rightmost_sep_obj(path, 0, len); + if (right == -1) return NULL; - sep = *right; - *right = '\0'; - left = rightmost_sep(pathname); - *right = sep; - if (left == NULL) - left = pathname; + left = rightmost_sep_obj(path, 0, right); + if (left == -1) + left = 0; else left++; - if (right-left != Py_UCS4_strlen(CACHEDIR_UNICODE) || - Py_UCS4_strncmp(left, CACHEDIR_UNICODE, right-left) != 0) - goto error; + if (right-left != sizeof(CACHEDIR)-1) + return NULL; + for (i = 0; i < sizeof(CACHEDIR)-1; i++) + if (PyUnicode_READ_CHAR(path, left+i) != CACHEDIR[i]) + return NULL; /* Now verify that the path component to the right of the last slash has two dots in it. */ - if ((dot0 = Py_UCS4_strchr(right + 1, '.')) == NULL) - goto error; - if ((dot1 = Py_UCS4_strchr(dot0 + 1, '.')) == NULL) - goto error; + dot0 = PyUnicode_FindChar(path, '.', right+1, len, 1); + if (dot0 < 0) + return NULL; + dot1 = PyUnicode_FindChar(path, '.', dot0+1, len, 1); + if (dot1 < 0) + return NULL; /* Too many dots? */ - if (Py_UCS4_strchr(dot1 + 1, '.') != NULL) - goto error; + if (PyUnicode_FindChar(path, '.', dot1+1, len, 1) != -1) + return NULL; /* This is a PEP 3147 path. Start by copying everything from the start of pathname up to and including the leftmost slash. Then copy the file's basename, removing the magic tag and adding a .py suffix. */ - Py_UCS4_strncpy(buf, pathname, (i=left-pathname)); - Py_UCS4_strncpy(buf+i, right+1, (j=dot0-right)); - buf[i+j] = 'p'; - buf[i+j+1] = 'y'; - PyMem_Free(pathname); - return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, i+j+2); - error: - PyMem_Free(pathname); - return NULL; + result = PyUnicode_New(left + (dot0-right) + 2, + PyUnicode_MAX_CHAR_VALUE(path)); + if (!result) + return NULL; + kind = PyUnicode_KIND(result); + data = PyUnicode_DATA(result); + PyUnicode_CopyCharacters(result, 0, path, 0, (i = left)); + PyUnicode_CopyCharacters(result, left, path, right+1, + (j = dot0-right)); + PyUnicode_WRITE(kind, data, i+j, 'p'); + PyUnicode_WRITE(kind, data, i+j+1, 'y'); + return result; } /* Given a pathname for a Python source file, its time of last -- cgit v1.2.1 From c7e9dd65640704b9ef0174a6c5fc9d82fbebf492 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 23 Oct 2011 18:08:20 +0200 Subject: Drop unused macros. Use CACHEDIR consistently. --- Python/import.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index a20871efcc..ebaaeacebd 100644 --- a/Python/import.c +++ b/Python/import.c @@ -108,8 +108,8 @@ typedef unsigned short mode_t; /* MAGIC must change whenever the bytecode emitted by the compiler may no longer be understood by older implementations of the eval loop (usually due to the addition of new opcodes) - TAG and PYC_TAG_UNICODE must change for each major Python release. The magic - number will take care of any bytecode changes that occur during development. + TAG must change for each major Python release. The magic number will take + care of any bytecode changes that occur during development. */ #define QUOTE(arg) #arg #define STRIFY(name) QUOTE(name) @@ -118,13 +118,9 @@ typedef unsigned short mode_t; #define MAGIC (3190 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" -static const Py_UCS4 CACHEDIR_UNICODE[] = { - '_', '_', 'p', 'y', 'c', 'a', 'c', 'h', 'e', '_', '_', '\0'}; /* Current magic word and string tag as globals. */ static long pyc_magic = MAGIC; static const char *pyc_tag = TAG; -static const Py_UCS4 PYC_TAG_UNICODE[] = { - 'c', 'p', 'y', 't', 'h', 'o', 'n', '-', PY_MAJOR_VERSION + 48, PY_MINOR_VERSION + 48, '\0'}; #undef QUOTE #undef STRIFY #undef MAJOR @@ -939,7 +935,7 @@ make_compiled_pathname(PyObject *pathstr, int debug) { PyObject *result; Py_ssize_t fname, ext, len, i, pos, taglen; - Py_ssize_t pycache_len = sizeof("__pycache__/") - 1; + Py_ssize_t pycache_len = sizeof(CACHEDIR) - 1; int kind; void *data; @@ -968,7 +964,7 @@ make_compiled_pathname(PyObject *pathstr, int debug) PyUnicode_CopyCharacters(result, 0, pathstr, 0, fname); pos = fname; for (i = 0; i < pycache_len - 1; i++) - PyUnicode_WRITE(kind, data, pos++, "__pycache__"[i]); + PyUnicode_WRITE(kind, data, pos++, CACHEDIR[i]); PyUnicode_WRITE(kind, data, pos++, SEP); PyUnicode_CopyCharacters(result, pos, pathstr, fname, ext - fname); -- cgit v1.2.1 From 012400c9fe4281ff97e7dee256cd2a05649a54ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 23 Oct 2011 18:41:56 +0200 Subject: Fix off-by-one error. --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index ebaaeacebd..736ce7717c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -955,7 +955,7 @@ make_compiled_pathname(PyObject *pathstr, int debug) /* result = pathstr[:fname] + "__pycache__" + SEP + pathstr[fname:ext] + tag + ".py[co]" */ taglen = strlen(pyc_tag); - result = PyUnicode_New(ext + pycache_len + taglen + 4, + result = PyUnicode_New(ext + pycache_len + 1 + taglen + 4, PyUnicode_MAX_CHAR_VALUE(pathstr)); if (!result) return NULL; @@ -963,7 +963,7 @@ make_compiled_pathname(PyObject *pathstr, int debug) data = PyUnicode_DATA(result); PyUnicode_CopyCharacters(result, 0, pathstr, 0, fname); pos = fname; - for (i = 0; i < pycache_len - 1; i++) + for (i = 0; i < pycache_len; i++) PyUnicode_WRITE(kind, data, pos++, CACHEDIR[i]); PyUnicode_WRITE(kind, data, pos++, SEP); PyUnicode_CopyCharacters(result, pos, pathstr, -- cgit v1.2.1 From 9d5ef173055b3857a637b716f029b0e81d172c29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 23 Oct 2011 18:45:16 +0200 Subject: Rewrite find_module_path using unicode API. --- Python/import.c | 83 +++++++++++++++++++++++++++++++++------------------------ 1 file changed, 48 insertions(+), 35 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 736ce7717c..e21ae39359 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1743,11 +1743,11 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, PyObject *path_hooks, PyObject *path_importer_cache, PyObject **p_path, PyObject **p_loader, struct filedescr **p_fd) { - Py_UCS4 buf[MAXPATHLEN+1]; - PyObject *path_unicode, *filename; - Py_ssize_t len; + PyObject *path_unicode, *filename = NULL; + Py_ssize_t len, pos; struct stat statbuf; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; + int result, addsep; if (PyUnicode_Check(path)) { Py_INCREF(path); @@ -1766,15 +1766,10 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, return -1; len = PyUnicode_GET_LENGTH(path_unicode); - if (!PyUnicode_AsUCS4(path_unicode, buf, Py_ARRAY_LENGTH(buf), 1)) { - Py_DECREF(path_unicode); - PyErr_Clear(); - return 0; + if (PyUnicode_FindChar(path_unicode, 0, 0, len, 1) != -1) { + result = 0; + goto out; /* path contains '\0' */ } - Py_DECREF(path_unicode); - - if (Py_UCS4_strlen(buf) != len) - return 0; /* path contains '\0' */ /* sys.path_hooks import hook */ if (p_loader != NULL) { @@ -1784,43 +1779,54 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, importer = get_path_importer(path_importer_cache, path_hooks, path); if (importer == NULL) { - return -1; + result = -1; + goto out; } /* Note: importer is a borrowed reference */ if (importer != Py_None) { PyObject *loader; loader = _PyObject_CallMethodId(importer, &PyId_find_module, "O", fullname); - if (loader == NULL) - return -1; /* error */ + if (loader == NULL) { + result = -1; /* error */ + goto out; + } if (loader != Py_None) { /* a loader was found */ *p_loader = loader; *p_fd = &importhookdescr; - return 2; + result = 2; + goto out; } Py_DECREF(loader); - return 0; + result = 0; + goto out; } } /* no hook was found, use builtin import */ - if (len > 0 && buf[len-1] != SEP + addsep = 0; + if (len > 0 && PyUnicode_READ_CHAR(path_unicode, len-1) != SEP #ifdef ALTSEP - && buf[len-1] != ALTSEP + && PyUnicode_READ_CHAR(path_unicode, len-1) != ALTSEP #endif ) - buf[len++] = SEP; - if (!PyUnicode_AsUCS4(name, buf+len, Py_ARRAY_LENGTH(buf)-len, 1)) { - PyErr_Clear(); - return 0; - } - len += PyUnicode_GET_LENGTH(name); - - filename = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - buf, len); - if (filename == NULL) - return -1; + addsep = 1; + filename = PyUnicode_New(len + PyUnicode_GET_LENGTH(name) + addsep, + Py_MAX(PyUnicode_MAX_CHAR_VALUE(path_unicode), + PyUnicode_MAX_CHAR_VALUE(name))); + if (filename == NULL) { + result = -1; + goto out; + } + PyUnicode_CopyCharacters(filename, 0, path_unicode, 0, len); + pos = len; + if (addsep) + PyUnicode_WRITE(PyUnicode_KIND(filename), + PyUnicode_DATA(filename), + pos++, SEP); + PyUnicode_CopyCharacters(filename, pos, name, 0, + PyUnicode_GET_LENGTH(name)); /* Check for package import (buf holds a directory name, and there's an __init__ module in that directory */ @@ -1832,14 +1838,16 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, match = case_ok(filename, 0, name); if (match < 0) { - Py_DECREF(filename); - return -1; + result = -1; + goto out; } if (match) { /* case matches */ if (find_init_module(filename)) { /* and has __init__.py */ *p_path = filename; + filename = NULL; *p_fd = &fd_package; - return 2; + result = 2; + goto out; } else { int err; @@ -1847,15 +1855,20 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, "Not importing directory %R: missing __init__.py", filename); if (err) { - Py_DECREF(filename); - return -1; + result = -1; + goto out; } } } } #endif *p_path = filename; - return 1; + filename = NULL; + result = 1; + out: + Py_DECREF(path_unicode); + Py_XDECREF(filename); + return result; } /* Find a module in search_path_list. For each path, try -- cgit v1.2.1 From 9ab76944255d2498c50789b1d0582aec1e417ad9 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 25 Oct 2011 00:21:02 +0200 Subject: Fix test_imp failure under Windows --- Python/import.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index e21ae39359..3842c63c2e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -938,12 +938,19 @@ make_compiled_pathname(PyObject *pathstr, int debug) Py_ssize_t pycache_len = sizeof(CACHEDIR) - 1; int kind; void *data; + Py_UCS4 lastsep; /* Compute the output string size. */ len = PyUnicode_GET_LENGTH(pathstr); /* If there is no separator, this returns -1, so - lastsep will be 0. */ + fname will be 0. */ fname = rightmost_sep_obj(pathstr, 0, len) + 1; + /* Windows: re-use the last separator character (/ or \\) when + appending the __pycache__ path. */ + if (fname > 0) + lastsep = PyUnicode_READ_CHAR(pathstr, fname -1); + else + lastsep = SEP; ext = fname - 1; for(i = fname; i < len; i++) if (PyUnicode_READ_CHAR(pathstr, i) == '.') @@ -965,7 +972,7 @@ make_compiled_pathname(PyObject *pathstr, int debug) pos = fname; for (i = 0; i < pycache_len; i++) PyUnicode_WRITE(kind, data, pos++, CACHEDIR[i]); - PyUnicode_WRITE(kind, data, pos++, SEP); + PyUnicode_WRITE(kind, data, pos++, lastsep); PyUnicode_CopyCharacters(result, pos, pathstr, fname, ext - fname); pos += ext - fname; -- cgit v1.2.1 From 7336fba07c5e34404a0d72c4b24e78a3d83ac1cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 30 Oct 2011 09:07:07 +0100 Subject: Port PyImport_ReloadModule to Unicode API. --- Python/import.c | 60 +++++++++++++++++++++++++-------------------------------- 1 file changed, 26 insertions(+), 34 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 3842c63c2e..294eb2bb33 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3359,8 +3359,8 @@ PyImport_ReloadModule(PyObject *m) PyObject *modules_reloading = interp->modules_reloading; PyObject *modules = PyImport_GetModuleDict(); PyObject *path_list = NULL, *loader = NULL, *existing_m = NULL; - PyObject *nameobj, *bufobj, *subnameobj; - Py_UCS4 *name = NULL, *subname; + PyObject *name, *bufobj, *subname; + Py_ssize_t subname_start; struct filedescr *fdp; FILE *fp = NULL; PyObject *newm = NULL; @@ -3376,45 +3376,39 @@ PyImport_ReloadModule(PyObject *m) "reload() argument must be module"); return NULL; } - nameobj = PyModule_GetNameObject(m); - if (nameobj == NULL || PyUnicode_READY(nameobj) == -1) + name = PyModule_GetNameObject(m); + if (name == NULL || PyUnicode_READY(name) == -1) return NULL; - if (m != PyDict_GetItem(modules, nameobj)) { + if (m != PyDict_GetItem(modules, name)) { PyErr_Format(PyExc_ImportError, "reload(): module %R not in sys.modules", - nameobj); - Py_DECREF(nameobj); + name); + Py_DECREF(name); return NULL; } - existing_m = PyDict_GetItem(modules_reloading, nameobj); + existing_m = PyDict_GetItem(modules_reloading, name); if (existing_m != NULL) { /* Due to a recursive reload, this module is already being reloaded. */ - Py_DECREF(nameobj); + Py_DECREF(name); Py_INCREF(existing_m); return existing_m; } - if (PyDict_SetItem(modules_reloading, nameobj, m) < 0) { - Py_DECREF(nameobj); + if (PyDict_SetItem(modules_reloading, name, m) < 0) { + Py_DECREF(name); return NULL; } - name = PyUnicode_AsUCS4Copy(nameobj); - if (!name) { - Py_DECREF(nameobj); - return NULL; - } - subname = Py_UCS4_strrchr(name, '.'); - if (subname == NULL) { - Py_INCREF(nameobj); - subnameobj = nameobj; + subname_start = PyUnicode_FindChar(name, '.', 0, + PyUnicode_GET_LENGTH(name), -1); + if (subname_start == -1) { + Py_INCREF(name); + subname = name; } else { PyObject *parentname, *parent; Py_ssize_t len; - len = subname - name; - parentname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - name, len); + parentname = PyUnicode_Substring(name, 0, subname_start); if (parentname == NULL) { goto error; } @@ -3430,16 +3424,15 @@ PyImport_ReloadModule(PyObject *m) path_list = _PyObject_GetAttrId(parent, &PyId___path__); if (path_list == NULL) PyErr_Clear(); - subname++; - len = PyUnicode_GET_LENGTH(nameobj) - (len + 1); - subnameobj = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - subname, len); + subname_start++; + len = PyUnicode_GET_LENGTH(name) - (subname_start + 1); + subname = PyUnicode_Substring(name, subname_start, len); } - if (subnameobj == NULL) + if (subname == NULL) goto error; - fdp = find_module(nameobj, subnameobj, path_list, + fdp = find_module(name, subname, path_list, &bufobj, &fp, &loader); - Py_DECREF(subnameobj); + Py_DECREF(subname); Py_XDECREF(path_list); if (fdp == NULL) { @@ -3447,7 +3440,7 @@ PyImport_ReloadModule(PyObject *m) goto error; } - newm = load_module(nameobj, fp, bufobj, fdp->type, loader); + newm = load_module(name, fp, bufobj, fdp->type, loader); Py_XDECREF(bufobj); Py_XDECREF(loader); @@ -3459,13 +3452,12 @@ PyImport_ReloadModule(PyObject *m) * going to return NULL in this case regardless of whether * replacing name succeeds, so the return value is ignored. */ - PyDict_SetItem(modules, nameobj, m); + PyDict_SetItem(modules, name, m); } error: imp_modules_reloading_clear(); - Py_DECREF(nameobj); - PyMem_Free(name); + Py_DECREF(name); return newm; } -- cgit v1.2.1 From 2593ff3b46728f698ea5a5307fcc6263012b358b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 30 Oct 2011 23:50:02 +0100 Subject: Port import_module_level to Unicode API. --- Python/import.c | 164 +++++++++++++++++++++----------------------------------- 1 file changed, 61 insertions(+), 103 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 8e3aa4b921..03e9986126 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2753,8 +2753,7 @@ static PyObject *get_parent(PyObject *globals, int level); static PyObject *load_next(PyObject *mod, PyObject *altmod, PyObject *inputname, PyObject **p_outputname, - Py_UCS4 *buf, Py_ssize_t *p_buflen, - Py_ssize_t bufsize); + PyObject **p_prefix); static int mark_miss(PyObject *name); static int ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *buf, int recursive); @@ -2767,11 +2766,11 @@ static PyObject * import_module_level(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) { - Py_UCS4 buf[MAXPATHLEN+1]; - Py_ssize_t buflen; - Py_ssize_t bufsize = MAXPATHLEN+1; - PyObject *parent, *head, *next, *tail, *inputname, *outputname; - PyObject *parent_name, *ensure_name; + PyObject *parent, *next, *inputname, *outputname; + PyObject *head = NULL; + PyObject *tail = NULL; + PyObject *prefix = NULL; + PyObject *result = NULL; Py_ssize_t sep, altsep; if (PyUnicode_READY(name)) @@ -2794,26 +2793,18 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, return NULL; } - parent = get_parent(globals, &parent_name, level); + parent = get_parent(globals, &prefix, level); if (parent == NULL) { return NULL; } - if (PyUnicode_READY(parent_name)) + if (PyUnicode_READY(prefix)) return NULL; - buflen = PyUnicode_GET_LENGTH(parent_name); - if (!PyUnicode_AsUCS4(parent_name, buf, Py_ARRAY_LENGTH(buf), 1)) { - Py_DECREF(parent_name); - PyErr_SetString(PyExc_ValueError, - "Module name too long"); - return NULL; - } - Py_DECREF(parent_name); head = load_next(parent, level < 0 ? Py_None : parent, name, &outputname, - buf, &buflen, bufsize); + &prefix); if (head == NULL) - return NULL; + goto out; tail = head; Py_INCREF(tail); @@ -2822,13 +2813,11 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, while (1) { inputname = outputname; next = load_next(tail, tail, inputname, &outputname, - buf, &buflen, bufsize); - Py_DECREF(tail); - Py_DECREF(inputname); - if (next == NULL) { - Py_DECREF(head); - return NULL; - } + &prefix); + Py_CLEAR(tail); + Py_CLEAR(inputname); + if (next == NULL) + goto out; tail = next; if (outputname == NULL) { @@ -2840,10 +2829,8 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, /* If tail is Py_None, both get_parent and load_next found an empty module name: someone called __import__("") or doctored faulty bytecode */ - Py_DECREF(tail); - Py_DECREF(head); PyErr_SetString(PyExc_ValueError, "Empty module name"); - return NULL; + goto out; } if (fromlist != NULL) { @@ -2852,26 +2839,21 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, } if (fromlist == NULL) { - Py_DECREF(tail); - return head; - } - - Py_DECREF(head); - - ensure_name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - buf, Py_UCS4_strlen(buf)); - if (ensure_name == NULL) { - Py_DECREF(tail); - return NULL; - } - if (!ensure_fromlist(tail, fromlist, ensure_name, 0)) { - Py_DECREF(tail); - Py_DECREF(ensure_name); - return NULL; + result = head; + Py_INCREF(result); + goto out; } - Py_DECREF(ensure_name); - return tail; + if (!ensure_fromlist(tail, fromlist, prefix, 0)) + goto out; + + result = tail; + Py_INCREF(result); + out: + Py_XDECREF(head); + Py_XDECREF(tail); + Py_XDECREF(prefix); + return result; } PyObject * @@ -3083,91 +3065,67 @@ return_none: static PyObject * load_next(PyObject *mod, PyObject *altmod, PyObject *inputname, PyObject **p_outputname, - Py_UCS4 *buf, Py_ssize_t *p_buflen, Py_ssize_t bufsize) + PyObject **p_prefix) { - Py_UCS4 *dot; + Py_ssize_t dot; Py_ssize_t len; - Py_UCS4 *p; - PyObject *fullname, *name, *result, *mark_name; - Py_UCS4 *nameuni; + PyObject *fullname, *name = NULL, *result; *p_outputname = NULL; - if (PyUnicode_GET_LENGTH(inputname) == 0) { + len = PyUnicode_GET_LENGTH(inputname); + if (len == 0) { /* completely empty module name should only happen in 'from . import' (or '__import__("")')*/ Py_INCREF(mod); return mod; } - nameuni = PyUnicode_AsUCS4Copy(inputname); - if (nameuni == NULL) - return NULL; - dot = Py_UCS4_strchr(nameuni, '.'); - if (dot != NULL) { - len = dot - nameuni; + dot = PyUnicode_FindChar(inputname, '.', 0, len, 1); + if (dot >= 0) { + len = dot; if (len == 0) { PyErr_SetString(PyExc_ValueError, "Empty module name"); goto error; } } - else - len = PyUnicode_GET_LENGTH(inputname); - if (*p_buflen+len+1 >= bufsize) { - PyErr_SetString(PyExc_ValueError, - "Module name too long"); + /* name = inputname[:len] */ + name = PyUnicode_Substring(inputname, 0, len); + if (name == NULL) goto error; - } - p = buf + *p_buflen; - if (p != buf) { - *p++ = '.'; - *p_buflen += 1; + if (PyUnicode_GET_LENGTH(*p_prefix)) { + /* fullname = prefix + "." + name */ + fullname = PyUnicode_FromFormat("%U.%U", *p_prefix, name); + if (fullname == NULL) + goto error; } - Py_UCS4_strncpy(p, nameuni, len); - p[len] = '\0'; - *p_buflen += len; - - fullname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - buf, *p_buflen); - if (fullname == NULL) - goto error; - name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - p, len); - if (name == NULL) { - Py_DECREF(fullname); - goto error; + else { + fullname = name; + Py_INCREF(fullname); } + result = import_submodule(mod, name, fullname); - Py_DECREF(fullname); + Py_DECREF(*p_prefix); + /* Transfer reference. */ + *p_prefix = fullname; if (result == Py_None && altmod != mod) { Py_DECREF(result); /* Here, altmod must be None and mod must not be None */ result = import_submodule(altmod, name, name); - Py_DECREF(name); if (result != NULL && result != Py_None) { - mark_name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - buf, *p_buflen); - if (mark_name == NULL) { + if (mark_miss(*p_prefix) != 0) { Py_DECREF(result); goto error; } - if (mark_miss(mark_name) != 0) { - Py_DECREF(result); - Py_DECREF(mark_name); - goto error; - } - Py_DECREF(mark_name); - Py_UCS4_strncpy(buf, nameuni, len); - buf[len] = '\0'; - *p_buflen = len; + Py_DECREF(*p_prefix); + *p_prefix = name; + Py_INCREF(*p_prefix); } } - else - Py_DECREF(name); if (result == NULL) goto error; @@ -3178,20 +3136,20 @@ load_next(PyObject *mod, PyObject *altmod, goto error; } - if (dot != NULL) { - *p_outputname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - dot+1, Py_UCS4_strlen(dot+1)); + if (dot >= 0) { + *p_outputname = PyUnicode_Substring(inputname, dot+1, + PyUnicode_GET_LENGTH(inputname)); if (*p_outputname == NULL) { Py_DECREF(result); goto error; } } - PyMem_Free(nameuni); + Py_DECREF(name); return result; error: - PyMem_Free(nameuni); + Py_XDECREF(name); return NULL; } -- cgit v1.2.1 From 0c820a9c9584c838de7ba3abea17c76a03bd405b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Mon, 31 Oct 2011 20:47:31 +0100 Subject: Issue #13303: Fix a race condition in the bytecode file creation. --- Python/import.c | 50 ++++++++------------------------------------------ 1 file changed, 8 insertions(+), 42 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 03e9986126..701041b2f2 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1183,42 +1183,6 @@ parse_source_module(PyObject *pathname, FILE *fp) return co; } -/* Helper to open a bytecode file for writing in exclusive mode */ - -#ifndef MS_WINDOWS -static FILE * -open_exclusive(char *filename, mode_t mode) -{ -#if defined(O_EXCL)&&defined(O_CREAT)&&defined(O_WRONLY)&&defined(O_TRUNC) - /* Use O_EXCL to avoid a race condition when another process tries to - write the same file. When that happens, our open() call fails, - which is just fine (since it's only a cache). - XXX If the file exists and is writable but the directory is not - writable, the file will never be written. Oh well. - */ - int fd; - (void) unlink(filename); - fd = open(filename, O_EXCL|O_CREAT|O_WRONLY|O_TRUNC -#ifdef O_BINARY - |O_BINARY /* necessary for Windows */ -#endif -#ifdef __VMS - , mode, "ctxt=bin", "shr=nil" -#else - , mode -#endif - ); - if (fd < 0) - return NULL; - return fdopen(fd, "wb"); -#else - /* Best we can do -- on Windows this can't happen anyway */ - return fopen(filename, "wb"); -#endif -} -#endif - - /* Write a compiled module to a file, placing the time of last modification of its source into the header. Errors are ignored, if a write error occurs an attempt is made to @@ -1234,15 +1198,13 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, #ifdef MS_WINDOWS /* since Windows uses different permissions */ mode_t mode = srcstat->st_mode & ~S_IEXEC; #else - mode_t mode = srcstat->st_mode & ~S_IXUSR & ~S_IXGRP & ~S_IXOTH; mode_t dirmode = (srcstat->st_mode | S_IXUSR | S_IXGRP | S_IXOTH | S_IWUSR | S_IWGRP | S_IWOTH); PyObject *dirbytes; #endif -#ifdef MS_WINDOWS int fd; -#else +#ifndef MS_WINDOWS PyObject *cpathbytes, *cpathbytes_tmp; Py_ssize_t cpathbytes_len; #endif @@ -1313,7 +1275,7 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, return; } cpathbytes_len = PyBytes_GET_SIZE(cpathbytes); - cpathbytes_tmp = PyBytes_FromStringAndSize(NULL, cpathbytes_len + 4); + cpathbytes_tmp = PyBytes_FromStringAndSize(NULL, cpathbytes_len + 6); if (cpathbytes_tmp == NULL) { Py_DECREF(cpathbytes); PyErr_Clear(); @@ -1321,9 +1283,13 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, } memcpy(PyBytes_AS_STRING(cpathbytes_tmp), PyBytes_AS_STRING(cpathbytes), cpathbytes_len); - memcpy(PyBytes_AS_STRING(cpathbytes_tmp) + cpathbytes_len, ".tmp", 4); + memcpy(PyBytes_AS_STRING(cpathbytes_tmp) + cpathbytes_len, "XXXXXX", 6); - fp = open_exclusive(PyBytes_AS_STRING(cpathbytes_tmp), mode); + fd = mkstemp(PyBytes_AS_STRING(cpathbytes_tmp)); + if (0 <= fd) + fp = fdopen(fd, "wb"); + else + fp = NULL; #endif if (fp == NULL) { if (Py_VerboseFlag) -- cgit v1.2.1 From 17ec0be2a18cb0c8d6dc40f0bf0205642e47c31d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Thu, 10 Nov 2011 19:12:29 +0100 Subject: Issue #13303: Fix bytecode file default permission. --- Python/import.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 5c33b6be57..ae1101e4ec 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1202,12 +1202,10 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, S_IXUSR | S_IXGRP | S_IXOTH | S_IWUSR | S_IWGRP | S_IWOTH); PyObject *dirbytes; -#endif - int fd; -#ifndef MS_WINDOWS PyObject *cpathbytes, *cpathbytes_tmp; Py_ssize_t cpathbytes_len; #endif + int fd; PyObject *dirname; Py_UCS4 *dirsep; int res, ok; @@ -1275,7 +1273,7 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, return; } cpathbytes_len = PyBytes_GET_SIZE(cpathbytes); - cpathbytes_tmp = PyBytes_FromStringAndSize(NULL, cpathbytes_len + 6); + cpathbytes_tmp = PyBytes_FromStringAndSize(NULL, cpathbytes_len + 4); if (cpathbytes_tmp == NULL) { Py_DECREF(cpathbytes); PyErr_Clear(); @@ -1283,9 +1281,10 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, } memcpy(PyBytes_AS_STRING(cpathbytes_tmp), PyBytes_AS_STRING(cpathbytes), cpathbytes_len); - memcpy(PyBytes_AS_STRING(cpathbytes_tmp) + cpathbytes_len, "XXXXXX", 6); + memcpy(PyBytes_AS_STRING(cpathbytes_tmp) + cpathbytes_len, ".tmp", 4); - fd = mkstemp(PyBytes_AS_STRING(cpathbytes_tmp)); + fd = open(PyBytes_AS_STRING(cpathbytes_tmp), + O_CREAT | O_EXCL | O_WRONLY, 0666); if (0 <= fd) fp = fdopen(fd, "wb"); else -- cgit v1.2.1 From 1a238237f0304a03e73e464b069a2a4aec4beb3c Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 15 Nov 2011 19:15:19 +0100 Subject: Issue #13392: Writing a pyc file should now be atomic under Windows as well. --- Python/import.c | 48 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 10 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index ae1101e4ec..3c3e504e28 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1197,6 +1197,8 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, time_t mtime = srcstat->st_mtime; #ifdef MS_WINDOWS /* since Windows uses different permissions */ mode_t mode = srcstat->st_mode & ~S_IEXEC; + PyObject *cpathname_tmp; + Py_ssize_t cpathname_len; #else mode_t dirmode = (srcstat->st_mode | S_IXUSR | S_IXGRP | S_IXOTH | @@ -1255,18 +1257,29 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, } Py_DECREF(dirname); + /* We first write to a tmp file and then take advantage + of atomic renaming (which *should* be true even under Windows). */ #ifdef MS_WINDOWS - (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname)); - fd = _wopen(PyUnicode_AS_UNICODE(cpathname), - O_EXCL | O_CREAT | O_WRONLY | O_TRUNC | O_BINARY, - mode); + cpathname_len = PyUnicode_GET_LENGTH(cpathname); + cpathname_tmp = PyUnicode_New(cpathname_len + 4, + PyUnicode_MAX_CHAR_VALUE(cpathname)); + if (cpathname_tmp == NULL) { + PyErr_Clear(); + return; + } + PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 0, '.'); + PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 1, 't'); + PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 2, 'm'); + PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 3, 'p'); + (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname_tmp)); + fd = _wopen(PyUnicode_AS_UNICODE(cpathname_tmp), + O_EXCL | O_CREAT | O_WRONLY | O_BINARY, + mode); if (0 <= fd) fp = fdopen(fd, "wb"); else fp = NULL; #else - /* Under POSIX, we first write to a tmp file and then take advantage - of atomic renaming. */ cpathbytes = PyUnicode_EncodeFSDefault(cpathname); if (cpathbytes == NULL) { PyErr_Clear(); @@ -1294,7 +1307,9 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, if (Py_VerboseFlag) PySys_FormatStderr( "# can't create %R\n", cpathname); -#ifndef MS_WINDOWS +#ifdef MS_WINDOWS + Py_DECREF(cpathname_tmp); +#else Py_DECREF(cpathbytes); Py_DECREF(cpathbytes_tmp); #endif @@ -1315,7 +1330,8 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, /* Don't keep partial file */ fclose(fp); #ifdef MS_WINDOWS - (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname)); + (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname_tmp)); + Py_DECREF(cpathname_tmp); #else (void) unlink(PyBytes_AS_STRING(cpathbytes_tmp)); Py_DECREF(cpathbytes); @@ -1324,8 +1340,20 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, return; } fclose(fp); - /* Under POSIX, do an atomic rename */ -#ifndef MS_WINDOWS + /* Do a (hopefully) atomic rename */ +#ifdef MS_WINDOWS + if (!MoveFileExW(PyUnicode_AS_UNICODE(cpathname_tmp), + PyUnicode_AS_UNICODE(cpathname), + MOVEFILE_REPLACE_EXISTING)) { + if (Py_VerboseFlag) + PySys_FormatStderr("# can't write %R\n", cpathname); + /* Don't keep tmp file */ + (void) DeleteFileW(PyUnicode_AS_UNICODE(cpathname_tmp)); + Py_DECREF(cpathname_tmp); + return; + } + Py_DECREF(cpathname_tmp); +#else if (rename(PyBytes_AS_STRING(cpathbytes_tmp), PyBytes_AS_STRING(cpathbytes))) { if (Py_VerboseFlag) -- cgit v1.2.1 From 1a11faf52590fae1e61f304835b27c42ed21b1dd Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 15 Nov 2011 20:40:55 +0100 Subject: Fix regression under Windows following b75b41237380 (from issue #13392) --- Python/import.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 3c3e504e28..64197d7a48 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1267,6 +1267,11 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, PyErr_Clear(); return; } + if (PyUnicode_CopyCharacters(cpathname_tmp, 0, + cpathname, 0, cpathname_len) < 0) { + PyErr_Clear(); + return; + } PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 0, '.'); PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 1, 't'); PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 2, 'm'); -- cgit v1.2.1 From 23ad9bf93bf81d89142e55ca1cfb04f9c3ffc837 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 15 Nov 2011 22:27:32 +0100 Subject: Use PyUnicode_FromFormat() to create the temporary file name. Also, as in importlib, append the id of an object to make the file name pseudo-random. --- Python/import.c | 41 ++++++++++++++--------------------------- 1 file changed, 14 insertions(+), 27 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 64197d7a48..b1eca64f4d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1195,17 +1195,15 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, Py_UCS4 *cpathname_ucs4; FILE *fp; time_t mtime = srcstat->st_mtime; + PyObject *cpathname_tmp; #ifdef MS_WINDOWS /* since Windows uses different permissions */ mode_t mode = srcstat->st_mode & ~S_IEXEC; - PyObject *cpathname_tmp; - Py_ssize_t cpathname_len; #else mode_t dirmode = (srcstat->st_mode | S_IXUSR | S_IXGRP | S_IXOTH | S_IWUSR | S_IWGRP | S_IWOTH); PyObject *dirbytes; PyObject *cpathbytes, *cpathbytes_tmp; - Py_ssize_t cpathbytes_len; #endif int fd; PyObject *dirname; @@ -1258,24 +1256,18 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, Py_DECREF(dirname); /* We first write to a tmp file and then take advantage - of atomic renaming (which *should* be true even under Windows). */ -#ifdef MS_WINDOWS - cpathname_len = PyUnicode_GET_LENGTH(cpathname); - cpathname_tmp = PyUnicode_New(cpathname_len + 4, - PyUnicode_MAX_CHAR_VALUE(cpathname)); + of atomic renaming (which *should* be true even under Windows). + As in importlib, we use id(something) to generate a pseudo-random + filename. mkstemp() can't be used since it doesn't allow specifying + the file access permissions. + */ + cpathname_tmp = PyUnicode_FromFormat("%U.%zd", + cpathname, (Py_ssize_t) co); if (cpathname_tmp == NULL) { PyErr_Clear(); return; } - if (PyUnicode_CopyCharacters(cpathname_tmp, 0, - cpathname, 0, cpathname_len) < 0) { - PyErr_Clear(); - return; - } - PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 0, '.'); - PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 1, 't'); - PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 2, 'm'); - PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 3, 'p'); +#ifdef MS_WINDOWS (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname_tmp)); fd = _wopen(PyUnicode_AS_UNICODE(cpathname_tmp), O_EXCL | O_CREAT | O_WRONLY | O_BINARY, @@ -1285,22 +1277,17 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, else fp = NULL; #else - cpathbytes = PyUnicode_EncodeFSDefault(cpathname); - if (cpathbytes == NULL) { + cpathbytes_tmp = PyUnicode_EncodeFSDefault(cpathname_tmp); + Py_DECREF(cpathname_tmp); + if (cpathbytes_tmp == NULL) { PyErr_Clear(); return; } - cpathbytes_len = PyBytes_GET_SIZE(cpathbytes); - cpathbytes_tmp = PyBytes_FromStringAndSize(NULL, cpathbytes_len + 4); - if (cpathbytes_tmp == NULL) { - Py_DECREF(cpathbytes); + cpathbytes = PyUnicode_EncodeFSDefault(cpathname); + if (cpathbytes == NULL) { PyErr_Clear(); return; } - memcpy(PyBytes_AS_STRING(cpathbytes_tmp), PyBytes_AS_STRING(cpathbytes), - cpathbytes_len); - memcpy(PyBytes_AS_STRING(cpathbytes_tmp) + cpathbytes_len, ".tmp", 4); - fd = open(PyBytes_AS_STRING(cpathbytes_tmp), O_CREAT | O_EXCL | O_WRONLY, 0666); if (0 <= fd) -- cgit v1.2.1 From 84acdc7c2fbb43c63eab2518445c5a7771755037 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 17 Nov 2011 00:45:54 +0100 Subject: Catch PyUnicode_AS_UNICODE() errors --- Python/import.c | 47 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 14 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index b1eca64f4d..6c6d1f697a 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1198,6 +1198,7 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, PyObject *cpathname_tmp; #ifdef MS_WINDOWS /* since Windows uses different permissions */ mode_t mode = srcstat->st_mode & ~S_IEXEC; + wchar_t *wdirname, *wpathname, *wpathname_tmp; #else mode_t dirmode = (srcstat->st_mode | S_IXUSR | S_IXGRP | S_IXOTH | @@ -1230,7 +1231,12 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, } #ifdef MS_WINDOWS - res = CreateDirectoryW(PyUnicode_AS_UNICODE(dirname), NULL); + wdirname = PyUnicode_AsUnicode(dirname); + if (wdirname == NULL) { + PyErr_Clear(); + return; + } + res = CreateDirectoryW(wdirname, NULL); ok = (res != 0); if (!ok && GetLastError() == ERROR_ALREADY_EXISTS) ok = 1; @@ -1268,8 +1274,19 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, return; } #ifdef MS_WINDOWS - (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname_tmp)); - fd = _wopen(PyUnicode_AS_UNICODE(cpathname_tmp), + wpathname = PyUnicode_AsUnicode(cpathname); + if (wpathname == NULL) { + PyErr_Clear(); + return; + } + wpathname_tmp = PyUnicode_AsUnicode(cpathname_tmp); + if (wpathname_tmp == NULL) { + PyErr_Clear(); + return; + } + + (void)DeleteFileW(wpathname_tmp); + fd = _wopen(wpathname_tmp, O_EXCL | O_CREAT | O_WRONLY | O_BINARY, mode); if (0 <= fd) @@ -1322,7 +1339,7 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, /* Don't keep partial file */ fclose(fp); #ifdef MS_WINDOWS - (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname_tmp)); + (void)DeleteFileW(wpathname_tmp); Py_DECREF(cpathname_tmp); #else (void) unlink(PyBytes_AS_STRING(cpathbytes_tmp)); @@ -1334,13 +1351,11 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, fclose(fp); /* Do a (hopefully) atomic rename */ #ifdef MS_WINDOWS - if (!MoveFileExW(PyUnicode_AS_UNICODE(cpathname_tmp), - PyUnicode_AS_UNICODE(cpathname), - MOVEFILE_REPLACE_EXISTING)) { + if (!MoveFileExW(wpathname_tmp, wpathname, MOVEFILE_REPLACE_EXISTING)) { if (Py_VerboseFlag) PySys_FormatStderr("# can't write %R\n", cpathname); /* Don't keep tmp file */ - (void) DeleteFileW(PyUnicode_AS_UNICODE(cpathname_tmp)); + (void) DeleteFileW(wpathname_tmp); Py_DECREF(cpathname_tmp); return; } @@ -1819,10 +1834,10 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, PyUnicode_CopyCharacters(filename, 0, path_unicode, 0, len); pos = len; if (addsep) - PyUnicode_WRITE(PyUnicode_KIND(filename), + PyUnicode_WRITE(PyUnicode_KIND(filename), PyUnicode_DATA(filename), pos++, SEP); - PyUnicode_CopyCharacters(filename, pos, name, 0, + PyUnicode_CopyCharacters(filename, pos, name, 0, PyUnicode_GET_LENGTH(name)); /* Check for package import (buf holds a directory name, @@ -2268,18 +2283,22 @@ case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) WIN32_FIND_DATAW data; HANDLE h; int cmp; - wchar_t *wname; + wchar_t *wfilename, *wname; Py_ssize_t wname_len; if (Py_GETENV("PYTHONCASEOK") != NULL) return 1; - h = FindFirstFileW(PyUnicode_AS_UNICODE(filename), &data); + wfilename = PyUnicode_AsUnicode(filename); + if (wfilename == NULL) + return -1; + + h = FindFirstFileW(wfilename, &data); if (h == INVALID_HANDLE_VALUE) { PyErr_Format(PyExc_NameError, "Can't find file for module %R\n(filename %R)", name, filename); - return 0; + return -1; } FindClose(h); @@ -2831,7 +2850,7 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, if (!ensure_fromlist(tail, fromlist, prefix, 0)) goto out; - + result = tail; Py_INCREF(result); out: -- cgit v1.2.1 From 949c3f094c436fea35325980527d63d0d9eabd21 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 25 Nov 2011 18:56:07 +0100 Subject: PEP 3155 / issue #13448: Qualified name for classes and functions. --- Python/import.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 6c6d1f697a..7199230098 100644 --- a/Python/import.c +++ b/Python/import.c @@ -103,6 +103,7 @@ typedef unsigned short mode_t; tag: cpython-32 Python 3.2a2 3180 (add DELETE_DEREF) Python 3.3a0 3190 __class__ super closure changed + Python 3.3a0 3200 (__qualname__ added) */ /* MAGIC must change whenever the bytecode emitted by the compiler may no @@ -115,7 +116,7 @@ typedef unsigned short mode_t; #define STRIFY(name) QUOTE(name) #define MAJOR STRIFY(PY_MAJOR_VERSION) #define MINOR STRIFY(PY_MINOR_VERSION) -#define MAGIC (3190 | ((long)'\r'<<16) | ((long)'\n'<<24)) +#define MAGIC (3200 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" /* Current magic word and string tag as globals. */ -- cgit v1.2.1 From 07c717edd69dd5e05901335996d83fe11d6a24e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Wed, 7 Dec 2011 19:16:01 +0100 Subject: Issue #11051: Reduce the number of syscalls per import. --- Python/import.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 7199230098..9773687f62 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1944,8 +1944,7 @@ find_module_path_list(PyObject *fullname, PyObject *name, if (Py_VerboseFlag > 1) PySys_FormatStderr("# trying %R\n", filename); - if (_Py_stat(filename, &statbuf) == 0 && /* it exists */ - S_ISDIR(statbuf.st_mode)) /* it's a directory */ + if (_Py_stat(filename, &statbuf) != 0 || S_ISDIR(statbuf.st_mode)) { Py_DECREF(filename); continue; -- cgit v1.2.1 From d07179846c02fb4a745fd29f7ddfcfc8f51e366b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Wed, 7 Dec 2011 23:17:58 +0100 Subject: Followup to a541bda2f5e2: Add a short comment. --- Python/import.c | 1 + 1 file changed, 1 insertion(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 9773687f62..47839c543d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1946,6 +1946,7 @@ find_module_path_list(PyObject *fullname, PyObject *name, if (_Py_stat(filename, &statbuf) != 0 || S_ISDIR(statbuf.st_mode)) { + /* it doesn't exist, or it's a directory */ Py_DECREF(filename); continue; } -- cgit v1.2.1 From 219d85b51cf4eed90b0cd74706d6b55b3753c604 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 18 Dec 2011 20:47:30 +0100 Subject: import.c now catchs _Py_stat() exceptions _Py_stat() now returns -2 if an exception was raised. --- Python/import.c | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 47839c543d..34d1a26dc7 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1527,6 +1527,7 @@ get_sourcefile(PyObject *filename) Py_UCS4 *fileuni; PyObject *py; struct stat statbuf; + int err; len = PyUnicode_GET_LENGTH(filename); if (len == 0) @@ -1554,7 +1555,10 @@ get_sourcefile(PyObject *filename) if (py == NULL) goto error; - if (_Py_stat(py, &statbuf) == 0 && S_ISREG(statbuf.st_mode)) { + err = _Py_stat(py, &statbuf); + if (err == -2) + goto error; + if (err == 0 && S_ISREG(statbuf.st_mode)) { PyMem_Free(fileuni); return py; } @@ -1760,7 +1764,7 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, Py_ssize_t len, pos; struct stat statbuf; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; - int result, addsep; + int err, result, addsep; if (PyUnicode_Check(path)) { Py_INCREF(path); @@ -1844,7 +1848,12 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, /* Check for package import (buf holds a directory name, and there's an __init__ module in that directory */ #ifdef HAVE_STAT - if (_Py_stat(filename, &statbuf) == 0 && /* it exists */ + err = _Py_stat(filename, &statbuf); + if (err == -2) { + result = -1; + goto out; + } + if (err == 0 && /* it exists */ S_ISDIR(statbuf.st_mode)) /* it's a directory */ { int match; @@ -1905,6 +1914,7 @@ find_module_path_list(PyObject *fullname, PyObject *name, FILE *fp = NULL; PyObject *prefix, *filename; int match; + int err; npath = PyList_Size(search_path_list); for (i = 0; i < npath; i++) { @@ -1944,8 +1954,13 @@ find_module_path_list(PyObject *fullname, PyObject *name, if (Py_VerboseFlag > 1) PySys_FormatStderr("# trying %R\n", filename); - if (_Py_stat(filename, &statbuf) != 0 || S_ISDIR(statbuf.st_mode)) - { + err = _Py_stat(filename, &statbuf); + if (err == -2) { + Py_DECREF(prefix); + Py_DECREF(filename); + return NULL; + } + if (err != 0 || S_ISDIR(statbuf.st_mode)) { /* it doesn't exist, or it's a directory */ Py_DECREF(filename); continue; @@ -2345,11 +2360,15 @@ find_init_module(PyObject *directory) struct stat statbuf; PyObject *filename; int match; + int err; filename = PyUnicode_FromFormat("%U%c__init__.py", directory, SEP); if (filename == NULL) return -1; - if (_Py_stat(filename, &statbuf) == 0) { + err = _Py_stat(filename, &statbuf); + if (err == -2) + return -1; + if (err == 0) { /* 3=len(".py") */ match = case_ok(filename, -3, initstr); if (match < 0) { @@ -2367,7 +2386,12 @@ find_init_module(PyObject *directory) directory, SEP, Py_OptimizeFlag ? 'o' : 'c'); if (filename == NULL) return -1; - if (_Py_stat(filename, &statbuf) == 0) { + err = _Py_stat(filename, &statbuf); + if (err == -2) { + Py_DECREF(filename); + return -1; + } + if (err == 0) { /* 4=len(".pyc") */ match = case_ok(filename, -4, initstr); if (match < 0) { -- cgit v1.2.1 From 1434e995657f5525a55584535d023dbe0ce2af24 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 13 Jan 2012 18:52:16 +0100 Subject: Issue #13645: pyc files now contain the size of the corresponding source code, to avoid timestamp collisions (especially on filesystems with a low timestamp resolution) when checking for freshness of the bytecode. --- Python/import.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index c7541316b5..4efc36923a 100644 --- a/Python/import.c +++ b/Python/import.c @@ -104,6 +104,7 @@ typedef unsigned short mode_t; Python 3.2a2 3180 (add DELETE_DEREF) Python 3.3a0 3190 __class__ super closure changed Python 3.3a0 3200 (__qualname__ added) + 3210 (added size modulo 2**32 to the pyc header) */ /* MAGIC must change whenever the bytecode emitted by the compiler may no @@ -116,7 +117,7 @@ typedef unsigned short mode_t; #define STRIFY(name) QUOTE(name) #define MAJOR STRIFY(PY_MAJOR_VERSION) #define MINOR STRIFY(PY_MINOR_VERSION) -#define MAGIC (3200 | ((long)'\r'<<16) | ((long)'\n'<<24)) +#define MAGIC (3210 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" /* Current magic word and string tag as globals. */ @@ -1071,11 +1072,12 @@ make_source_pathname(PyObject *path) Doesn't set an exception. */ static FILE * -check_compiled_module(PyObject *pathname, time_t mtime, PyObject *cpathname) +check_compiled_module(PyObject *pathname, struct stat *srcstat, PyObject *cpathname) { FILE *fp; long magic; long pyc_mtime; + long pyc_size; fp = _Py_fopen(cpathname, "rb"); if (fp == NULL) @@ -1088,12 +1090,19 @@ check_compiled_module(PyObject *pathname, time_t mtime, PyObject *cpathname) return NULL; } pyc_mtime = PyMarshal_ReadLongFromFile(fp); - if (pyc_mtime != mtime) { + if (pyc_mtime != srcstat->st_mtime) { if (Py_VerboseFlag) PySys_FormatStderr("# %R has bad mtime\n", cpathname); fclose(fp); return NULL; } + pyc_size = PyMarshal_ReadLongFromFile(fp); + if (pyc_size != (srcstat->st_size & 0xFFFFFFFF)) { + if (Py_VerboseFlag) + PySys_FormatStderr("# %R has bad size\n", cpathname); + fclose(fp); + return NULL; + } if (Py_VerboseFlag) PySys_FormatStderr("# %R matches %R\n", cpathname, pathname); return fp; @@ -1136,6 +1145,8 @@ load_compiled_module(PyObject *name, PyObject *cpathname, FILE *fp) "Bad magic number in %R", cpathname); return NULL; } + /* Skip mtime and size */ + (void) PyMarshal_ReadLongFromFile(fp); (void) PyMarshal_ReadLongFromFile(fp); co = read_compiled_module(cpathname, fp); if (co == NULL) @@ -1196,6 +1207,7 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, Py_UCS4 *cpathname_ucs4; FILE *fp; time_t mtime = srcstat->st_mtime; + long size = srcstat->st_size & 0xFFFFFFFF; PyObject *cpathname_tmp; #ifdef MS_WINDOWS /* since Windows uses different permissions */ mode_t mode = srcstat->st_mode & ~S_IEXEC; @@ -1326,14 +1338,16 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, return; } PyMarshal_WriteLongToFile(pyc_magic, fp, Py_MARSHAL_VERSION); - /* First write a 0 for mtime */ + /* First write a 0 for mtime and size */ + PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION); PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION); PyMarshal_WriteObjectToFile((PyObject *)co, fp, Py_MARSHAL_VERSION); fflush(fp); - /* Now write the true mtime */ + /* Now write the true mtime and size */ fseek(fp, 4L, 0); assert(mtime < LONG_MAX); PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION); + PyMarshal_WriteLongToFile(size, fp, Py_MARSHAL_VERSION); if (fflush(fp) != 0 || ferror(fp)) { if (Py_VerboseFlag) PySys_FormatStderr("# can't write %R\n", cpathname); @@ -1478,7 +1492,7 @@ load_source_module(PyObject *name, PyObject *pathname, FILE *fp) cpathname = make_compiled_pathname(pathname, !Py_OptimizeFlag); if (cpathname != NULL) - fpc = check_compiled_module(pathname, st.st_mtime, cpathname); + fpc = check_compiled_module(pathname, &st, cpathname); else fpc = NULL; -- cgit v1.2.1 From c6d22e93db4b5e0cdb94ae023a4b7e43693dfed2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 26 Jan 2012 00:31:49 +0100 Subject: Remove now useless arbitrary limit of module name length --- Python/import.c | 6 ------ 1 file changed, 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index d5b89d5966..8bd7a61722 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2052,12 +2052,6 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, if (p_loader != NULL) *p_loader = NULL; - if (PyUnicode_GET_LENGTH(name) > MAXPATHLEN) { - PyErr_SetString(PyExc_OverflowError, - "module name is too long"); - return NULL; - } - /* sys.meta_path import hook */ if (p_loader != NULL) { _Py_IDENTIFIER(find_module); -- cgit v1.2.1 From 0ea564cff248528b899a26f75b173f35b55d505a Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Thu, 15 Mar 2012 15:37:39 -0500 Subject: perform yield from delegation by repeating YIELD_FROM opcode (closes #14230) This allows generators that are using yield from to be seen by debuggers. It also kills the f_yieldfrom field on frame objects. Patch mostly from Mark Shannon with a few tweaks by me. --- Python/import.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 4871b99a93..14148c7a1c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -104,7 +104,8 @@ typedef unsigned short mode_t; Python 3.2a2 3180 (add DELETE_DEREF) Python 3.3a0 3190 __class__ super closure changed Python 3.3a0 3200 (__qualname__ added) - 3210 (added size modulo 2**32 to the pyc header) + Python 3.3a1 3210 (added size modulo 2**32 to the pyc header) + 3220 (changed PEP 380 implementation) */ /* MAGIC must change whenever the bytecode emitted by the compiler may no -- cgit v1.2.1 From 06b08c0eb2d160a859326db57c12ea099d74a13c Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Thu, 15 Mar 2012 15:40:37 -0500 Subject: fix comment --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 14148c7a1c..c0f89aaa1b 100644 --- a/Python/import.c +++ b/Python/import.c @@ -104,8 +104,8 @@ typedef unsigned short mode_t; Python 3.2a2 3180 (add DELETE_DEREF) Python 3.3a0 3190 __class__ super closure changed Python 3.3a0 3200 (__qualname__ added) - Python 3.3a1 3210 (added size modulo 2**32 to the pyc header) - 3220 (changed PEP 380 implementation) + 3210 (added size modulo 2**32 to the pyc header) + Python 3.3a1 3220 (changed PEP 380 implementation) */ /* MAGIC must change whenever the bytecode emitted by the compiler may no -- cgit v1.2.1 From d8c00dc5e03601e3959e3147dbf9c527e1d3b648 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Fri, 16 Mar 2012 09:39:12 -0500 Subject: in 72556ff86828, I should have updated the magic as well as the comment (#14230) --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index c0f89aaa1b..144ca71117 100644 --- a/Python/import.c +++ b/Python/import.c @@ -118,7 +118,7 @@ typedef unsigned short mode_t; #define STRIFY(name) QUOTE(name) #define MAJOR STRIFY(PY_MAJOR_VERSION) #define MINOR STRIFY(PY_MINOR_VERSION) -#define MAGIC (3210 | ((long)'\r'<<16) | ((long)'\n'<<24)) +#define MAGIC (3220 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" /* Current magic word and string tag as globals. */ -- cgit v1.2.1 From e6ecc6d13e7b3c963d6cabfdacca8138f968e970 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sat, 14 Apr 2012 14:10:13 -0400 Subject: Issue #2377: Make importlib the implementation of __import__(). importlib._bootstrap is now frozen into Python/importlib.h and stored as _frozen_importlib in sys.modules. Py_Initialize() loads the frozen code along with sys and imp and then uses _frozen_importlib._install() to set builtins.__import__() w/ _frozen_importlib.__import__(). --- Python/import.c | 766 ++++++++++++++++++-------------------------------------- 1 file changed, 244 insertions(+), 522 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 144ca71117..87c1214c7e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -203,17 +203,13 @@ _PyImport_Init(void) void _PyImportHooks_Init(void) { - PyObject *v, *path_hooks = NULL, *zimpimport; + PyObject *v, *path_hooks = NULL; int err = 0; - /* adding sys.path_hooks and sys.path_importer_cache, setting up - zipimport */ if (PyType_Ready(&PyNullImporter_Type) < 0) goto error; - if (Py_VerboseFlag) - PySys_WriteStderr("# installing zipimport hook\n"); - + /* adding sys.path_hooks and sys.path_importer_cache */ v = PyList_New(0); if (v == NULL) goto error; @@ -234,11 +230,26 @@ _PyImportHooks_Init(void) err = PySys_SetObject("path_hooks", path_hooks); if (err) { error: - PyErr_Print(); - Py_FatalError("initializing sys.meta_path, sys.path_hooks, " - "path_importer_cache, or NullImporter failed" - ); + PyErr_Print(); + Py_FatalError("initializing sys.meta_path, sys.path_hooks, " + "path_importer_cache, or NullImporter failed" + ); } + Py_DECREF(path_hooks); +} + +void +_PyImportZip_Init(void) +{ + PyObject *path_hooks, *zimpimport; + int err = 0; + + path_hooks = PySys_GetObject("path_hooks"); + if (path_hooks == NULL) + goto error; + + if (Py_VerboseFlag) + PySys_WriteStderr("# installing zipimport hook\n"); zimpimport = PyImport_ImportModule("zipimport"); if (zimpimport == NULL) { @@ -261,14 +272,20 @@ _PyImportHooks_Init(void) /* sys.path_hooks.append(zipimporter) */ err = PyList_Append(path_hooks, zipimporter); Py_DECREF(zipimporter); - if (err) + if (err < 0) { goto error; + } if (Py_VerboseFlag) PySys_WriteStderr( "# installed zipimport hook\n"); } } - Py_DECREF(path_hooks); + + return; + + error: + PyErr_Print(); + Py_FatalError("initializing zipimport or NullImporter failed"); } /* Locking primitives to prevent parallel imports of the same module @@ -2542,8 +2559,6 @@ init_builtin(PyObject *name) name); return -1; } - if (Py_VerboseFlag) - PySys_FormatStderr("import %U # builtin\n", name); mod = (*p->initfunc)(); if (mod == 0) return -1; @@ -2654,9 +2669,6 @@ PyImport_ImportFrozenModuleObject(PyObject *name) ispackage = (size < 0); if (ispackage) size = -size; - if (Py_VerboseFlag) - PySys_FormatStderr("import %U # frozen%s\n", - name, ispackage ? " package" : ""); co = PyMarshal_ReadObjectFromString((char *)p->code, size); if (co == NULL) return -1; @@ -2786,568 +2798,278 @@ PyImport_ImportModuleNoBlock(const char *name) return result; } -/* Forward declarations for helper routines */ -static PyObject *get_parent(PyObject *globals, - PyObject **p_name, - int level); -static PyObject *load_next(PyObject *mod, PyObject *altmod, - PyObject *inputname, PyObject **p_outputname, - PyObject **p_prefix); -static int mark_miss(PyObject *name); -static int ensure_fromlist(PyObject *mod, PyObject *fromlist, - PyObject *buf, int recursive); -static PyObject * import_submodule(PyObject *mod, PyObject *name, - PyObject *fullname); - -/* The Magnum Opus of dotted-name import :-) */ -static PyObject * -import_module_level(PyObject *name, PyObject *globals, PyObject *locals, - PyObject *fromlist, int level) +PyObject * +PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, + PyObject *locals, PyObject *given_fromlist, + int level) { - PyObject *parent, *next, *inputname, *outputname; - PyObject *head = NULL; - PyObject *tail = NULL; - PyObject *prefix = NULL; - PyObject *result = NULL; - Py_ssize_t sep, altsep; - - if (PyUnicode_READY(name)) - return NULL; - - sep = PyUnicode_FindChar(name, SEP, 0, PyUnicode_GET_LENGTH(name), 1); - if (sep == -2) - return NULL; -#ifdef ALTSEP - altsep = PyUnicode_FindChar(name, ALTSEP, 0, PyUnicode_GET_LENGTH(name), 1); - if (altsep == -2) - return NULL; -#else - altsep = -1; -#endif - if (sep != -1 || altsep != -1) - { - PyErr_SetString(PyExc_ImportError, - "Import by filename is not supported."); - return NULL; - } - - parent = get_parent(globals, &prefix, level); - if (parent == NULL) { - return NULL; - } - - if (PyUnicode_READY(prefix)) - return NULL; - - head = load_next(parent, level < 0 ? Py_None : parent, name, &outputname, - &prefix); - if (head == NULL) - goto out; - - tail = head; - Py_INCREF(tail); - - if (outputname != NULL) { - while (1) { - inputname = outputname; - next = load_next(tail, tail, inputname, &outputname, - &prefix); - Py_CLEAR(tail); - Py_CLEAR(inputname); - if (next == NULL) - goto out; - tail = next; + _Py_IDENTIFIER(__import__); + _Py_IDENTIFIER(__package__); + _Py_IDENTIFIER(__path__); + _Py_IDENTIFIER(__name__); + _Py_IDENTIFIER(_find_and_load); + _Py_IDENTIFIER(_handle_fromlist); + _Py_static_string(single_dot, "."); + PyObject *abs_name = NULL; + PyObject *builtins_import = NULL; + PyObject *final_mod = NULL; + PyObject *mod = NULL; + PyObject *package = NULL; + PyObject *globals = NULL; + PyObject *fromlist = NULL; + PyInterpreterState *interp = PyThreadState_GET()->interp; - if (outputname == NULL) { - break; - } + /* Make sure to use default values so as to not have + PyObject_CallMethodObjArgs() truncate the parameter list because of a + NULL argument. */ + if (given_globals == NULL) { + globals = PyDict_New(); + if (globals == NULL) { + goto error; } } - if (tail == Py_None) { - /* If tail is Py_None, both get_parent and load_next found - an empty module name: someone called __import__("") or - doctored faulty bytecode */ - PyErr_SetString(PyExc_ValueError, "Empty module name"); - goto out; + else { + /* Only have to care what given_globals is if it will be used + fortsomething. */ + if (level > 0 && !PyDict_Check(given_globals)) { + PyErr_SetString(PyExc_TypeError, "globals must be a dict"); + goto error; + } + globals = given_globals; + Py_INCREF(globals); } - if (fromlist != NULL) { - if (fromlist == Py_None || !PyObject_IsTrue(fromlist)) - fromlist = NULL; + if (given_fromlist == NULL) { + fromlist = PyList_New(0); + if (fromlist == NULL) { + goto error; + } } - - if (fromlist == NULL) { - result = head; - Py_INCREF(result); - goto out; + else { + fromlist = given_fromlist; + Py_INCREF(fromlist); } - - if (!ensure_fromlist(tail, fromlist, prefix, 0)) - goto out; - - result = tail; - Py_INCREF(result); - out: - Py_XDECREF(head); - Py_XDECREF(tail); - Py_XDECREF(prefix); - return result; -} - -PyObject * -PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, - PyObject *locals, PyObject *fromlist, - int level) -{ - PyObject *mod; - _PyImport_AcquireLock(); - mod = import_module_level(name, globals, locals, fromlist, level); - if (_PyImport_ReleaseLock() < 0) { - Py_XDECREF(mod); - PyErr_SetString(PyExc_RuntimeError, - "not holding the import lock"); - return NULL; + if (name == NULL) { + PyErr_SetString(PyExc_ValueError, "Empty module name"); + goto error; } - return mod; -} -PyObject * -PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals, - PyObject *fromlist, int level) -{ - PyObject *nameobj, *mod; - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - mod = PyImport_ImportModuleLevelObject(nameobj, globals, locals, - fromlist, level); - Py_DECREF(nameobj); - return mod; -} + /* The below code is importlib.__import__() & _gcd_import(), ported to C + for added performance. */ - -/* Return the package that an import is being performed in. If globals comes - from the module foo.bar.bat (not itself a package), this returns the - sys.modules entry for foo.bar. If globals is from a package's __init__.py, - the package's entry in sys.modules is returned, as a borrowed reference. - - The name of the returned package is returned in *p_name. - - If globals doesn't come from a package or a module in a package, or a - corresponding entry is not found in sys.modules, Py_None is returned. -*/ -static PyObject * -get_parent(PyObject *globals, PyObject **p_name, int level) -{ - PyObject *nameobj; - - static PyObject *namestr = NULL; - static PyObject *pathstr = NULL; - static PyObject *pkgstr = NULL; - PyObject *pkgname, *modname, *modpath, *modules, *parent; - int orig_level = level; - - if (globals == NULL || !PyDict_Check(globals) || !level) - goto return_none; - - if (namestr == NULL) { - namestr = PyUnicode_InternFromString("__name__"); - if (namestr == NULL) - return NULL; + if (!PyUnicode_Check(name)) { + PyErr_SetString(PyExc_TypeError, "module name must be a string"); + goto error; } - if (pathstr == NULL) { - pathstr = PyUnicode_InternFromString("__path__"); - if (pathstr == NULL) - return NULL; + else if (PyUnicode_READY(name) < 0) { + goto error; } - if (pkgstr == NULL) { - pkgstr = PyUnicode_InternFromString("__package__"); - if (pkgstr == NULL) - return NULL; + if (level < 0) { + PyErr_SetString(PyExc_ValueError, "level must be >= 0"); + goto error; } - - pkgname = PyDict_GetItem(globals, pkgstr); - - if ((pkgname != NULL) && (pkgname != Py_None)) { - /* __package__ is set, so use it */ - if (!PyUnicode_Check(pkgname)) { - PyErr_SetString(PyExc_ValueError, - "__package__ set to non-string"); - return NULL; - } - if (PyUnicode_GET_LENGTH(pkgname) == 0) { - if (level > 0) { - PyErr_SetString(PyExc_ValueError, - "Attempted relative import in non-package"); - return NULL; + else if (level > 0) { + package = _PyDict_GetItemId(globals, &PyId___package__); + if (package != NULL && package != Py_None) { + Py_INCREF(package); + if (!PyUnicode_Check(package)) { + PyErr_SetString(PyExc_TypeError, "package must be a string"); + goto error; } - goto return_none; } - Py_INCREF(pkgname); - nameobj = pkgname; - } else { - /* __package__ not set, so figure it out and set it */ - modname = PyDict_GetItem(globals, namestr); - if (modname == NULL || !PyUnicode_Check(modname)) - goto return_none; - - modpath = PyDict_GetItem(globals, pathstr); - if (modpath != NULL) { - /* __path__ is set, so modname is already the package name */ - int error; - - error = PyDict_SetItem(globals, pkgstr, modname); - if (error) { - PyErr_SetString(PyExc_ValueError, - "Could not set __package__"); - return NULL; + else { + package = _PyDict_GetItemIdWithError(globals, &PyId___name__); + if (package == NULL) { + goto error; } - Py_INCREF(modname); - nameobj = modname; - } else { - /* Normal module, so work out the package name if any */ - Py_ssize_t len; - len = PyUnicode_FindChar(modname, '.', - 0, PyUnicode_GET_LENGTH(modname), -1); - if (len == -2) - return NULL; - if (len < 0) { - if (level > 0) { - PyErr_SetString(PyExc_ValueError, - "Attempted relative import in non-package"); - return NULL; + else if (!PyUnicode_Check(package)) { + PyErr_SetString(PyExc_TypeError, "__name__ must be a string"); + } + Py_INCREF(package); + + if (_PyDict_GetItemId(globals, &PyId___path__) == NULL) { + PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot); + if (borrowed_dot == NULL) { + goto error; } - if (PyDict_SetItem(globals, pkgstr, Py_None)) { - PyErr_SetString(PyExc_ValueError, - "Could not set __package__"); - return NULL; + PyObject *partition = PyUnicode_RPartition(package, + borrowed_dot); + Py_DECREF(package); + if (partition == NULL) { + goto error; } - goto return_none; + package = PyTuple_GET_ITEM(partition, 0); + Py_INCREF(package); + Py_DECREF(partition); } - pkgname = PyUnicode_Substring(modname, 0, len); - if (pkgname == NULL) - return NULL; - if (PyDict_SetItem(globals, pkgstr, pkgname)) { - Py_DECREF(pkgname); - PyErr_SetString(PyExc_ValueError, - "Could not set __package__"); - return NULL; - } - nameobj = pkgname; } - } - if (level > 1) { - Py_ssize_t dot, end = PyUnicode_GET_LENGTH(nameobj); - PyObject *newname; - while (--level > 0) { - dot = PyUnicode_FindChar(nameobj, '.', 0, end, -1); - if (dot == -2) { - Py_DECREF(nameobj); - return NULL; - } - if (dot < 0) { - Py_DECREF(nameobj); - PyErr_SetString(PyExc_ValueError, - "Attempted relative import beyond " - "toplevel package"); - return NULL; - } - end = dot; - } - newname = PyUnicode_Substring(nameobj, 0, end); - Py_DECREF(nameobj); - if (newname == NULL) - return NULL; - nameobj = newname; - } - modules = PyImport_GetModuleDict(); - parent = PyDict_GetItem(modules, nameobj); - if (parent == NULL) { - int err; - - if (orig_level >= 1) { + if (PyDict_GetItem(interp->modules, package) == NULL) { PyErr_Format(PyExc_SystemError, - "Parent module %R not loaded, " - "cannot perform relative import", nameobj); - Py_DECREF(nameobj); - return NULL; + "Parent module %R not loaded, cannot perform relative " + "import", package); + goto error; } - - err = PyErr_WarnFormat( - PyExc_RuntimeWarning, 1, - "Parent module %R not found while handling absolute import", - nameobj); - Py_DECREF(nameobj); - if (err) - return NULL; - - goto return_none; } - *p_name = nameobj; - return parent; - /* We expect, but can't guarantee, if parent != None, that: - - parent.__name__ == name - - parent.__dict__ is globals - If this is violated... Who cares? */ - -return_none: - nameobj = PyUnicode_New(0, 0); - if (nameobj == NULL) - return NULL; - *p_name = nameobj; - return Py_None; -} - -/* altmod is either None or same as mod */ -static PyObject * -load_next(PyObject *mod, PyObject *altmod, - PyObject *inputname, PyObject **p_outputname, - PyObject **p_prefix) -{ - Py_ssize_t dot; - Py_ssize_t len; - PyObject *fullname, *name = NULL, *result; - - *p_outputname = NULL; - - len = PyUnicode_GET_LENGTH(inputname); - if (len == 0) { - /* completely empty module name should only happen in - 'from . import' (or '__import__("")')*/ - Py_INCREF(mod); - return mod; + else { /* level == 0 */ + if (PyUnicode_GET_LENGTH(name) == 0) { + PyErr_SetString(PyExc_ValueError, "Empty module name"); + goto error; + } + package = Py_None; + Py_INCREF(package); } + if (level > 0) { + Py_ssize_t last_dot = PyUnicode_GET_LENGTH(package); + PyObject *base = NULL; + int level_up = 1; - dot = PyUnicode_FindChar(inputname, '.', 0, len, 1); - if (dot >= 0) { - len = dot; - if (len == 0) { - PyErr_SetString(PyExc_ValueError, - "Empty module name"); - goto error; + for (level_up = 1; level_up < level; level_up += 1) { + last_dot = PyUnicode_FindChar(package, '.', 0, last_dot, -1); + if (last_dot == -2) { + goto error; + } + else if (last_dot == -1) { + PyErr_SetString(PyExc_ValueError, + "attempted relative import beyond top-level " + "package"); + goto error; + } } - } + base = PyUnicode_Substring(package, 0, last_dot); + if (PyUnicode_GET_LENGTH(name) > 0) { + PyObject *borrowed_dot = NULL; + PyObject *seq = PyTuple_Pack(2, base, name); - /* name = inputname[:len] */ - name = PyUnicode_Substring(inputname, 0, len); - if (name == NULL) - goto error; + borrowed_dot = _PyUnicode_FromId(&single_dot); + if (borrowed_dot == NULL || seq == NULL) { + goto error; + } - if (PyUnicode_GET_LENGTH(*p_prefix)) { - /* fullname = prefix + "." + name */ - fullname = PyUnicode_FromFormat("%U.%U", *p_prefix, name); - if (fullname == NULL) - goto error; - } - else { - fullname = name; - Py_INCREF(fullname); - } - - result = import_submodule(mod, name, fullname); - Py_DECREF(*p_prefix); - /* Transfer reference. */ - *p_prefix = fullname; - if (result == Py_None && altmod != mod) { - Py_DECREF(result); - /* Here, altmod must be None and mod must not be None */ - result = import_submodule(altmod, name, name); - if (result != NULL && result != Py_None) { - if (mark_miss(*p_prefix) != 0) { - Py_DECREF(result); + abs_name = PyUnicode_Join(borrowed_dot, seq); + Py_DECREF(seq); + if (abs_name == NULL) { goto error; } - Py_DECREF(*p_prefix); - *p_prefix = name; - Py_INCREF(*p_prefix); + } + else { + abs_name = base; } } - if (result == NULL) - goto error; - - if (result == Py_None) { - Py_DECREF(result); - PyErr_Format(PyExc_ImportError, - "No module named %R", inputname); - goto error; + else { + abs_name = name; + Py_INCREF(abs_name); } - if (dot >= 0) { - *p_outputname = PyUnicode_Substring(inputname, dot+1, - PyUnicode_GET_LENGTH(inputname)); - if (*p_outputname == NULL) { - Py_DECREF(result); - goto error; +#if WITH_THREAD + _PyImport_AcquireLock(); +#endif + /* From this point forward, goto error_with_unlock! */ + if (PyDict_Check(globals)) { + builtins_import = _PyDict_GetItemId(globals, &PyId___import__); + } + if (builtins_import == NULL) { + builtins_import = _PyDict_GetItemId(interp->builtins, &PyId___import__); + if (builtins_import == NULL) { + Py_FatalError("__import__ missing"); } } + Py_INCREF(builtins_import); - Py_DECREF(name); - return result; - -error: - Py_XDECREF(name); - return NULL; -} + mod = PyDict_GetItem(interp->modules, abs_name); + if (mod == Py_None) { + PyErr_Format(PyExc_ImportError, + "import of %R halted; None in sys.modules", abs_name); + goto error_with_unlock; + } + else if (mod != NULL) { + Py_INCREF(mod); + } + else { + mod = _PyObject_CallMethodObjIdArgs(interp->importlib, + &PyId__find_and_load, abs_name, + builtins_import, NULL); + if (mod == NULL) { + goto error_with_unlock; + } + } -static int -mark_miss(PyObject *name) -{ - PyObject *modules = PyImport_GetModuleDict(); - return PyDict_SetItem(modules, name, Py_None); -} + if (PyObject_Not(fromlist)) { + if (level == 0 || PyUnicode_GET_LENGTH(name) > 0) { + PyObject *front = NULL; + PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot); -static int -ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *name, - int recursive) -{ - int i; - PyObject *fullname; - Py_ssize_t fromlist_len; + if (borrowed_dot == NULL) { + goto error_with_unlock; + } - if (!_PyObject_HasAttrId(mod, &PyId___path__)) - return 1; + PyObject *partition = PyUnicode_Partition(name, borrowed_dot); + if (partition == NULL) { + goto error_with_unlock; + } - fromlist_len = PySequence_Size(fromlist); + front = PyTuple_GET_ITEM(partition, 0); + Py_INCREF(front); + Py_DECREF(partition); - for (i = 0; i < fromlist_len; i++) { - PyObject *item = PySequence_GetItem(fromlist, i); - int hasit; - if (item == NULL) - return 0; - if (!PyUnicode_Check(item)) { - PyErr_SetString(PyExc_TypeError, - "Item in ``from list'' not a string"); - Py_DECREF(item); - return 0; - } - if (PyUnicode_READ_CHAR(item, 0) == '*') { - PyObject *all; - _Py_IDENTIFIER(__all__); - Py_DECREF(item); - /* See if the package defines __all__ */ - if (recursive) - continue; /* Avoid endless recursion */ - all = _PyObject_GetAttrId(mod, &PyId___all__); - if (all == NULL) - PyErr_Clear(); + if (level == 0) { + final_mod = PyDict_GetItemWithError(interp->modules, front); + Py_DECREF(front); + Py_XINCREF(final_mod); + } else { - int ret = ensure_fromlist(mod, all, name, 1); - Py_DECREF(all); - if (!ret) - return 0; + Py_ssize_t cut_off = PyUnicode_GetLength(name) - + PyUnicode_GetLength(front); + Py_ssize_t abs_name_len = PyUnicode_GetLength(abs_name); + PyObject *to_return = PyUnicode_Substring(name, 0, + abs_name_len - cut_off); + + final_mod = PyDict_GetItem(interp->modules, to_return); + Py_INCREF(final_mod); + Py_DECREF(to_return); } - continue; } - hasit = PyObject_HasAttr(mod, item); - if (!hasit) { - PyObject *submod; - fullname = PyUnicode_FromFormat("%U.%U", name, item); - if (fullname != NULL) { - submod = import_submodule(mod, item, fullname); - Py_DECREF(fullname); - } - else - submod = NULL; - Py_XDECREF(submod); - if (submod == NULL) { - Py_DECREF(item); - return 0; - } + else { + final_mod = mod; + Py_INCREF(mod); } - Py_DECREF(item); - } - - return 1; -} - -static int -add_submodule(PyObject *mod, PyObject *submod, PyObject *fullname, - PyObject *subname, PyObject *modules) -{ - if (mod == Py_None) - return 1; - /* Irrespective of the success of this load, make a - reference to it in the parent package module. A copy gets - saved in the modules dictionary under the full name, so get a - reference from there, if need be. (The exception is when the - load failed with a SyntaxError -- then there's no trace in - sys.modules. In that case, of course, do nothing extra.) */ - if (submod == NULL) { - submod = PyDict_GetItem(modules, fullname); - if (submod == NULL) - return 1; - } - if (PyModule_Check(mod)) { - /* We can't use setattr here since it can give a - * spurious warning if the submodule name shadows a - * builtin name */ - PyObject *dict = PyModule_GetDict(mod); - if (!dict) - return 0; - if (PyDict_SetItem(dict, subname, submod) < 0) - return 0; } else { - if (PyObject_SetAttr(mod, subname, submod) < 0) - return 0; + final_mod = _PyObject_CallMethodObjIdArgs(interp->importlib, + &PyId__handle_fromlist, mod, + fromlist, builtins_import, + NULL); } - return 1; + error_with_unlock: +#if WITH_THREAD + if (_PyImport_ReleaseLock() < 0) { + PyErr_SetString(PyExc_RuntimeError, "not holding the import lock"); + } +#endif + error: + Py_XDECREF(abs_name); + Py_XDECREF(builtins_import); + Py_XDECREF(mod); + Py_XDECREF(package); + Py_XDECREF(globals); + Py_XDECREF(fromlist); + return final_mod; } -static PyObject * -import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) +PyObject * +PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals, + PyObject *fromlist, int level) { - PyObject *modules = PyImport_GetModuleDict(); - PyObject *m = NULL, *bufobj, *path_list, *loader; - struct filedescr *fdp; - FILE *fp; - - /* Require: - if mod == None: subname == fullname - else: mod.__name__ + "." + subname == fullname - */ - - if ((m = PyDict_GetItem(modules, fullname)) != NULL) { - Py_INCREF(m); - return m; - } - - if (mod == Py_None) - path_list = NULL; - else { - path_list = _PyObject_GetAttrId(mod, &PyId___path__); - if (path_list == NULL) { - PyErr_Clear(); - Py_INCREF(Py_None); - return Py_None; - } - } - - fdp = find_module(fullname, subname, path_list, - &bufobj, &fp, &loader); - Py_XDECREF(path_list); - if (fdp == NULL) { - if (!PyErr_ExceptionMatches(PyExc_ImportError)) - return NULL; - PyErr_Clear(); - Py_INCREF(Py_None); - return Py_None; - } - m = load_module(fullname, fp, bufobj, fdp->type, loader); - Py_XDECREF(bufobj); - Py_XDECREF(loader); - if (fp) - fclose(fp); - if (m == NULL) - return NULL; - if (!add_submodule(mod, m, fullname, subname, modules)) { - Py_XDECREF(m); + PyObject *nameobj, *mod; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) return NULL; - } - return m; + mod = PyImport_ImportModuleLevelObject(nameobj, globals, locals, + fromlist, level); + Py_DECREF(nameobj); + return mod; } -- cgit v1.2.1 From d23854c2d242ca58e31af9d37638e8a20d85ad58 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sat, 14 Apr 2012 14:23:49 -0400 Subject: Undo a C99 idiom. --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 87c1214c7e..083146b155 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2889,12 +2889,12 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, Py_INCREF(package); if (_PyDict_GetItemId(globals, &PyId___path__) == NULL) { + PyObject *partition = NULL; PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot); if (borrowed_dot == NULL) { goto error; } - PyObject *partition = PyUnicode_RPartition(package, - borrowed_dot); + partition = PyUnicode_RPartition(package, borrowed_dot); Py_DECREF(package); if (partition == NULL) { goto error; -- cgit v1.2.1 From 29f1fb195df1165ebc3116bc3e9807622d68fa56 Mon Sep 17 00:00:00 2001 From: Brian Curtin Date: Sat, 14 Apr 2012 14:19:33 -0500 Subject: Fix Windows build --- Python/import.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 083146b155..be83ab4e1c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2963,7 +2963,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, Py_INCREF(abs_name); } -#if WITH_THREAD +#ifdef WITH_THREAD _PyImport_AcquireLock(); #endif /* From this point forward, goto error_with_unlock! */ @@ -2999,13 +2999,14 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, if (PyObject_Not(fromlist)) { if (level == 0 || PyUnicode_GET_LENGTH(name) > 0) { PyObject *front = NULL; + PyObject *partition = NULL; PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot); if (borrowed_dot == NULL) { goto error_with_unlock; } - PyObject *partition = PyUnicode_Partition(name, borrowed_dot); + partition = PyUnicode_Partition(name, borrowed_dot); if (partition == NULL) { goto error_with_unlock; } @@ -3043,7 +3044,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, NULL); } error_with_unlock: -#if WITH_THREAD +#ifdef WITH_THREAD if (_PyImport_ReleaseLock() < 0) { PyErr_SetString(PyExc_RuntimeError, "not holding the import lock"); } -- cgit v1.2.1 From 312c8a1bbef75dc3656917e7a3565437e3062146 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sat, 14 Apr 2012 21:50:00 -0400 Subject: Handle importing pkg.mod by executing __import__('mod', {'__packaging__': 'pkg', level=1) w/o properly (and thus not segfaulting). --- Python/import.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index be83ab4e1c..5136d98ac7 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3016,20 +3016,33 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, Py_DECREF(partition); if (level == 0) { - final_mod = PyDict_GetItemWithError(interp->modules, front); + final_mod = PyDict_GetItem(interp->modules, front); Py_DECREF(front); - Py_XINCREF(final_mod); + if (final_mod == NULL) { + PyErr_Format(PyExc_KeyError, + "%R not in sys.modules as expected", front); + } + else { + Py_INCREF(final_mod); + } } else { Py_ssize_t cut_off = PyUnicode_GetLength(name) - PyUnicode_GetLength(front); Py_ssize_t abs_name_len = PyUnicode_GetLength(abs_name); - PyObject *to_return = PyUnicode_Substring(name, 0, + PyObject *to_return = PyUnicode_Substring(abs_name, 0, abs_name_len - cut_off); final_mod = PyDict_GetItem(interp->modules, to_return); - Py_INCREF(final_mod); Py_DECREF(to_return); + if (final_mod == NULL) { + PyErr_Format(PyExc_KeyError, + "%R not in sys.modules as expected", + to_return); + } + else { + Py_INCREF(final_mod); + } } } else { -- cgit v1.2.1 From 1d6e335a95def2f9b4c7855ad0cd84c0c387ae45 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 15 Apr 2012 14:15:31 -0400 Subject: Set ImportError.name when raising the exception in the case of None found in sys.modules. --- Python/import.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 5136d98ac7..d4f57835d3 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2980,8 +2980,11 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, mod = PyDict_GetItem(interp->modules, abs_name); if (mod == Py_None) { - PyErr_Format(PyExc_ImportError, - "import of %R halted; None in sys.modules", abs_name); + PyObject *msg = PyUnicode_FromFormat("import of %R halted; " + "None in sys.modules", abs_name); + if (msg != NULL) { + PyErr_SetFromImportErrorWithName(msg, abs_name); + } goto error_with_unlock; } else if (mod != NULL) { -- cgit v1.2.1 From 4d6e5e7766952c0591f3a9cc0cdbc0b400e5e457 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 15 Apr 2012 15:24:04 -0400 Subject: Issue #14582: Import returns the module returned by a loader instead of sys.modules when possible. This is being done for two reasons. One is to gain a little bit of performance by skipping an unnecessary dict lookup in sys.modules. But the other (and main) reason is to be a little bit more clear in how things should work from the perspective of import's interactions with loaders. Otherwise loaders can easily forget to return the module even though PEP 302 explicitly states they are expected to return the module they loaded. --- Python/import.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index d4f57835d3..e96c7e4bc6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3019,15 +3019,22 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, Py_DECREF(partition); if (level == 0) { - final_mod = PyDict_GetItem(interp->modules, front); - Py_DECREF(front); - if (final_mod == NULL) { - PyErr_Format(PyExc_KeyError, - "%R not in sys.modules as expected", front); + if (PyUnicode_GET_LENGTH(name) == + PyUnicode_GET_LENGTH(front)) { + final_mod = mod; } else { - Py_INCREF(final_mod); + final_mod = PyDict_GetItem(interp->modules, front); + if (final_mod == NULL) { + PyErr_Format(PyExc_KeyError, + "%R not in sys.modules as expected", front); + } + } + Py_DECREF(front); + if (final_mod == NULL) { + goto error_with_unlock; } + Py_INCREF(final_mod); } else { Py_ssize_t cut_off = PyUnicode_GetLength(name) - -- cgit v1.2.1 From 6d17cfc162c054f5c39bd2c7dbde2a6513924722 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 15 Apr 2012 16:08:47 -0400 Subject: Issue #13959: Rename imp to _imp and add Lib/imp.py and begin rewriting functionality in pure Python. To start, imp.new_module() has been rewritten in pure Python, put into importlib (privately) and then publicly exposed in imp. --- Python/import.c | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index e96c7e4bc6..a930382fc7 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3684,15 +3684,6 @@ imp_load_package(PyObject *self, PyObject *args) return ret; } -static PyObject * -imp_new_module(PyObject *self, PyObject *args) -{ - PyObject *name; - if (!PyArg_ParseTuple(args, "U:new_module", &name)) - return NULL; - return PyModule_NewObject(name); -} - static PyObject * imp_reload(PyObject *self, PyObject *v) { @@ -3781,8 +3772,7 @@ does not conform to PEP 3147 format, ValueError will be raised."); /* Doc strings */ PyDoc_STRVAR(doc_imp, -"This module provides the components needed to build your own\n\ -__import__ function. Undocumented functions are obsolete."); +"(Extremely) low-level import machinery bits as used by importlib and imp."); PyDoc_STRVAR(doc_find_module, "find_module(name, [path]) -> (file, filename, (suffix, mode, type))\n\ @@ -3809,11 +3799,6 @@ PyDoc_STRVAR(doc_get_suffixes, Return a list of (suffix, mode, type) tuples describing the files\n\ that find_module() looks for."); -PyDoc_STRVAR(doc_new_module, -"new_module(name) -> module\n\ -Create a new module. Do not enter it in sys.modules.\n\ -The module name must include the full package name, if any."); - PyDoc_STRVAR(doc_lock_held, "lock_held() -> boolean\n\ Return True if the import lock is currently held, else False.\n\ @@ -3837,7 +3822,6 @@ static PyMethodDef imp_methods[] = { {"get_tag", imp_get_tag, METH_NOARGS, doc_get_tag}, {"get_suffixes", imp_get_suffixes, METH_NOARGS, doc_get_suffixes}, {"load_module", imp_load_module, METH_VARARGS, doc_load_module}, - {"new_module", imp_new_module, METH_VARARGS, doc_new_module}, {"lock_held", imp_lock_held, METH_NOARGS, doc_lock_held}, {"acquire_lock", imp_acquire_lock, METH_NOARGS, doc_acquire_lock}, {"release_lock", imp_release_lock, METH_NOARGS, doc_release_lock}, @@ -4005,7 +3989,7 @@ PyTypeObject PyNullImporter_Type = { static struct PyModuleDef impmodule = { PyModuleDef_HEAD_INIT, - "imp", + "_imp", doc_imp, 0, imp_methods, -- cgit v1.2.1 From accab4318fbf81aed2975225dd45fb3b2950d67f Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 15 Apr 2012 17:56:09 -0400 Subject: Issue #13959: Simplify imp.reload() by relying on a module's __loader__. Since import now sets __loader__ on all modules it creates and imp.reload() already relied on the attribute for modules that import didn't create, the only potential compatibility issue is if people were deleting the attribute on modules and expecting imp.reload() to continue to work. --- Python/import.c | 44 ++++++++++---------------------------------- 1 file changed, 10 insertions(+), 34 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index a930382fc7..747b0ddff4 100644 --- a/Python/import.c +++ b/Python/import.c @@ -153,7 +153,6 @@ static const struct filedescr _PyImport_StandardFiletab[] = { }; static PyObject *initstr = NULL; -_Py_IDENTIFIER(__path__); /* Initialize things */ @@ -3106,12 +3105,12 @@ PyImport_ReloadModule(PyObject *m) PyInterpreterState *interp = PyThreadState_Get()->interp; PyObject *modules_reloading = interp->modules_reloading; PyObject *modules = PyImport_GetModuleDict(); - PyObject *path_list = NULL, *loader = NULL, *existing_m = NULL; - PyObject *name, *bufobj, *subname; + PyObject *loader = NULL, *existing_m = NULL; + PyObject *name; Py_ssize_t subname_start; - struct filedescr *fdp; - FILE *fp = NULL; PyObject *newm = NULL; + _Py_IDENTIFIER(__loader__); + _Py_IDENTIFIER(load_module); if (modules_reloading == NULL) { Py_FatalError("PyImport_ReloadModule: " @@ -3149,51 +3148,28 @@ PyImport_ReloadModule(PyObject *m) subname_start = PyUnicode_FindChar(name, '.', 0, PyUnicode_GET_LENGTH(name), -1); - if (subname_start == -1) { - Py_INCREF(name); - subname = name; - } - else { + if (subname_start != -1) { PyObject *parentname, *parent; - Py_ssize_t len; parentname = PyUnicode_Substring(name, 0, subname_start); if (parentname == NULL) { goto error; } parent = PyDict_GetItem(modules, parentname); + Py_XDECREF(parent); if (parent == NULL) { PyErr_Format(PyExc_ImportError, "reload(): parent %R not in sys.modules", parentname); - Py_DECREF(parentname); goto error; } - Py_DECREF(parentname); - path_list = _PyObject_GetAttrId(parent, &PyId___path__); - if (path_list == NULL) - PyErr_Clear(); - subname_start++; - len = PyUnicode_GET_LENGTH(name) - (subname_start + 1); - subname = PyUnicode_Substring(name, subname_start, len); } - if (subname == NULL) - goto error; - fdp = find_module(name, subname, path_list, - &bufobj, &fp, &loader); - Py_DECREF(subname); - Py_XDECREF(path_list); - if (fdp == NULL) { - Py_XDECREF(loader); + loader = _PyObject_GetAttrId(m, &PyId___loader__); + if (loader == NULL) { goto error; } - - newm = load_module(name, fp, bufobj, fdp->type, loader); - Py_XDECREF(bufobj); - Py_XDECREF(loader); - - if (fp) - fclose(fp); + newm = _PyObject_CallMethodId(loader, &PyId_load_module, "O", name); + Py_DECREF(loader); if (newm == NULL) { /* load_module probably removed name from modules because of * the error. Put back the original module object. We're -- cgit v1.2.1 From 1aa20525c9c533e58932c23f8686f6b4e628965e Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 15 Apr 2012 20:25:23 -0400 Subject: Issue #13959: Re-implement imp.load_module() in imp.py. --- Python/import.c | 53 ----------------------------------------------------- 1 file changed, 53 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 747b0ddff4..06830fe2cb 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3600,53 +3600,6 @@ imp_load_source(PyObject *self, PyObject *args) return m; } -static PyObject * -imp_load_module(PyObject *self, PyObject *args) -{ - PyObject *name, *fob, *pathname, *pathname_obj, *ret; - char *suffix; /* Unused */ - char *mode; - int type; - FILE *fp; - - if (!PyArg_ParseTuple(args, "UOO(ssi):load_module", - &name, &fob, &pathname_obj, &suffix, &mode, &type)) - return NULL; - if (pathname_obj != Py_None) { - if (!PyUnicode_FSDecoder(pathname_obj, &pathname)) - return NULL; - } - else - pathname = NULL; - - if (*mode) { - /* Mode must start with 'r' or 'U' and must not contain '+'. - Implicit in this test is the assumption that the mode - may contain other modifiers like 'b' or 't'. */ - - if (!(*mode == 'r' || *mode == 'U') || strchr(mode, '+')) { - PyErr_Format(PyExc_ValueError, - "invalid file open mode %.200s", mode); - Py_XDECREF(pathname); - return NULL; - } - } - if (fob == Py_None) - fp = NULL; - else { - fp = get_file(NULL, fob, mode); - if (fp == NULL) { - Py_XDECREF(pathname); - return NULL; - } - } - ret = load_module(name, fp, pathname, type, NULL); - Py_XDECREF(pathname); - if (fp) - fclose(fp); - return ret; -} - static PyObject * imp_load_package(PyObject *self, PyObject *args) { @@ -3757,11 +3710,6 @@ built-in, frozen or special module and continue search in sys.path.\n\ The module name cannot contain '.'; to search for a submodule of a\n\ package, pass the submodule name and the package's __path__."); -PyDoc_STRVAR(doc_load_module, -"load_module(name, file, filename, (suffix, mode, type)) -> module\n\ -Load a module, given information returned by find_module().\n\ -The module name must include the full package name, if any."); - PyDoc_STRVAR(doc_get_magic, "get_magic() -> string\n\ Return the magic number for .pyc or .pyo files."); @@ -3797,7 +3745,6 @@ static PyMethodDef imp_methods[] = { {"get_magic", imp_get_magic, METH_NOARGS, doc_get_magic}, {"get_tag", imp_get_tag, METH_NOARGS, doc_get_tag}, {"get_suffixes", imp_get_suffixes, METH_NOARGS, doc_get_suffixes}, - {"load_module", imp_load_module, METH_VARARGS, doc_load_module}, {"lock_held", imp_lock_held, METH_NOARGS, doc_lock_held}, {"acquire_lock", imp_acquire_lock, METH_NOARGS, doc_acquire_lock}, {"release_lock", imp_release_lock, METH_NOARGS, doc_release_lock}, -- cgit v1.2.1 From b7a6769987b1f9cee26a973a603ce6d8b0205a5a Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 15 Apr 2012 22:28:28 -0400 Subject: Issue #13959: Re-implement imp.load_package() in imp.py. Thanks to Eric Snow for helping with imp.load_module() (previous commit) which led to the removal of a bunch of C code. --- Python/import.c | 171 -------------------------------------------------------- 1 file changed, 171 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 06830fe2cb..1b93b7c096 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1602,65 +1602,10 @@ unchanged: } /* Forward */ -static PyObject *load_module(PyObject *, FILE *, PyObject *, int, PyObject *); static struct filedescr *find_module(PyObject *, PyObject *, PyObject *, PyObject **, FILE **, PyObject **); static struct _frozen * find_frozen(PyObject *); -/* Load a package and return its module object WITH INCREMENTED - REFERENCE COUNT */ - -static PyObject * -load_package(PyObject *name, PyObject *pathname) -{ - PyObject *m, *d, *bufobj; - PyObject *file = NULL, *path_list = NULL; - int err; - FILE *fp = NULL; - struct filedescr *fdp; - - m = PyImport_AddModuleObject(name); - if (m == NULL) - return NULL; - if (Py_VerboseFlag) - PySys_FormatStderr("import %U # directory %R\n", - name, pathname); - file = get_sourcefile(pathname); - if (file == NULL) - return NULL; - path_list = Py_BuildValue("[O]", file); - if (path_list == NULL) { - Py_DECREF(file); - return NULL; - } - d = PyModule_GetDict(m); - err = PyDict_SetItemString(d, "__file__", file); - Py_DECREF(file); - if (err == 0) - err = PyDict_SetItemString(d, "__path__", path_list); - if (err != 0) { - Py_DECREF(path_list); - return NULL; - } - fdp = find_module(name, initstr, path_list, - &bufobj, &fp, NULL); - Py_DECREF(path_list); - if (fdp == NULL) { - if (PyErr_ExceptionMatches(PyExc_ImportError)) { - PyErr_Clear(); - Py_INCREF(m); - return m; - } - else - return NULL; - } - m = load_module(name, fp, bufobj, fdp->type, NULL); - Py_XDECREF(bufobj); - if (fp != NULL) - fclose(fp); - return m; -} - /* Helper to test for built-in module */ @@ -2434,108 +2379,6 @@ find_init_module(PyObject *directory) static int init_builtin(PyObject *); /* Forward */ -static PyObject* -load_builtin(PyObject *name, int type) -{ - PyObject *m, *modules; - int err; - - if (type == C_BUILTIN) - err = init_builtin(name); - else - err = PyImport_ImportFrozenModuleObject(name); - if (err < 0) - return NULL; - if (err == 0) { - PyErr_Format(PyExc_ImportError, - "Purported %s module %R not found", - type == C_BUILTIN ? "builtin" : "frozen", - name); - return NULL; - } - - modules = PyImport_GetModuleDict(); - m = PyDict_GetItem(modules, name); - if (m == NULL) { - PyErr_Format( - PyExc_ImportError, - "%s module %R not properly initialized", - type == C_BUILTIN ? "builtin" : "frozen", - name); - return NULL; - } - Py_INCREF(m); - return m; -} - -/* Load an external module using the default search path and return - its module object WITH INCREMENTED REFERENCE COUNT */ - -static PyObject * -load_module(PyObject *name, FILE *fp, PyObject *pathname, int type, PyObject *loader) -{ - PyObject *m; - - /* First check that there's an open file (if we need one) */ - switch (type) { - case PY_SOURCE: - case PY_COMPILED: - if (fp == NULL) { - PyErr_Format(PyExc_ValueError, - "file object required for import (type code %d)", - type); - return NULL; - } - } - - switch (type) { - - case PY_SOURCE: - m = load_source_module(name, pathname, fp); - break; - - case PY_COMPILED: - m = load_compiled_module(name, pathname, fp); - break; - -#ifdef HAVE_DYNAMIC_LOADING - case C_EXTENSION: - m = _PyImport_LoadDynamicModule(name, pathname, fp); - break; -#endif - - case PKG_DIRECTORY: - m = load_package(name, pathname); - break; - - case C_BUILTIN: - case PY_FROZEN: - m = load_builtin(name, type); - break; - - case IMP_HOOK: { - _Py_IDENTIFIER(load_module); - if (loader == NULL) { - PyErr_SetString(PyExc_ImportError, - "import hook without loader"); - return NULL; - } - m = _PyObject_CallMethodId(loader, &PyId_load_module, "O", name); - break; - } - - default: - PyErr_Format(PyExc_ImportError, - "Don't know how to import %R (type code %d)", - name, type); - m = NULL; - - } - - return m; -} - - /* Initialize a built-in module. Return 1 for success, 0 if the module is not found, and -1 with an exception set if the initialization failed. */ @@ -3600,19 +3443,6 @@ imp_load_source(PyObject *self, PyObject *args) return m; } -static PyObject * -imp_load_package(PyObject *self, PyObject *args) -{ - PyObject *name, *pathname; - PyObject * ret; - if (!PyArg_ParseTuple(args, "UO&:load_package", - &name, PyUnicode_FSDecoder, &pathname)) - return NULL; - ret = load_package(name, pathname); - Py_DECREF(pathname); - return ret; -} - static PyObject * imp_reload(PyObject *self, PyObject *v) { @@ -3764,7 +3594,6 @@ static PyMethodDef imp_methods[] = { #ifdef HAVE_DYNAMIC_LOADING {"load_dynamic", imp_load_dynamic, METH_VARARGS}, #endif - {"load_package", imp_load_package, METH_VARARGS}, {"load_source", imp_load_source, METH_VARARGS}, {"_fix_co_filename", imp_fix_co_filename, METH_VARARGS}, {NULL, NULL} /* sentinel */ -- cgit v1.2.1 From 7e1cc9ac0bcb3416e62b4c3a5841bfdf4f6318e9 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 16 Apr 2012 18:48:49 +0200 Subject: Fix reference loss on Py_None when None is encountered in sys.modules. --- Python/import.c | 1 + 1 file changed, 1 insertion(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 1b93b7c096..bd0d18ac31 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2827,6 +2827,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, if (msg != NULL) { PyErr_SetFromImportErrorWithName(msg, abs_name); } + mod = NULL; goto error_with_unlock; } else if (mod != NULL) { -- cgit v1.2.1 From 92e8b174fc2d8893cd6c710f4e4af397425aee73 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 16 Apr 2012 21:52:45 +0200 Subject: Fix a refleak --- Python/import.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index bd0d18ac31..a71bc3756d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2782,10 +2782,11 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, } base = PyUnicode_Substring(package, 0, last_dot); if (PyUnicode_GET_LENGTH(name) > 0) { - PyObject *borrowed_dot = NULL; - PyObject *seq = PyTuple_Pack(2, base, name); + PyObject *borrowed_dot, *seq = NULL; borrowed_dot = _PyUnicode_FromId(&single_dot); + seq = PyTuple_Pack(2, base, name); + Py_DECREF(base); if (borrowed_dot == NULL || seq == NULL) { goto error; } -- cgit v1.2.1 From 8f692771f066cef9711f5bb63f22254a85a33913 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 16 Apr 2012 22:06:21 +0200 Subject: Fix another refleak --- Python/import.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index a71bc3756d..1b232fc874 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2881,11 +2881,15 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, Py_INCREF(final_mod); } else { - Py_ssize_t cut_off = PyUnicode_GetLength(name) - - PyUnicode_GetLength(front); - Py_ssize_t abs_name_len = PyUnicode_GetLength(abs_name); + Py_ssize_t cut_off = PyUnicode_GET_LENGTH(name) - + PyUnicode_GET_LENGTH(front); + Py_ssize_t abs_name_len = PyUnicode_GET_LENGTH(abs_name); PyObject *to_return = PyUnicode_Substring(abs_name, 0, abs_name_len - cut_off); + Py_DECREF(front); + if (to_return == NULL) { + goto error_with_unlock; + } final_mod = PyDict_GetItem(interp->modules, to_return); Py_DECREF(to_return); -- cgit v1.2.1 From a045ed1a8ada767dc4cbce8168b5953238da8cbe Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 16 Apr 2012 22:11:25 -0400 Subject: Issue #13959: Re-implement imp.load_source() in imp.py. --- Python/import.c | 390 -------------------------------------------------------- 1 file changed, 390 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 1b232fc874..584f30eb38 100644 --- a/Python/import.c +++ b/Python/import.c @@ -904,26 +904,6 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, } -/* Like strrchr(string, '/') but searches for the rightmost of either SEP - or ALTSEP, if the latter is defined. -*/ -static Py_UCS4* -rightmost_sep(Py_UCS4 *s) -{ - Py_UCS4 *found, c; - for (found = NULL; (c = *s); s++) { - if (c == SEP -#ifdef ALTSEP - || c == ALTSEP -#endif - ) - { - found = s; - } - } - return found; -} - /* Like rightmost_sep, but operate on unicode objects. */ static Py_ssize_t rightmost_sep_obj(PyObject* o, Py_ssize_t start, Py_ssize_t end) @@ -1081,50 +1061,6 @@ make_source_pathname(PyObject *path) return result; } -/* Given a pathname for a Python source file, its time of last - modification, and a pathname for a compiled file, check whether the - compiled file represents the same version of the source. If so, - return a FILE pointer for the compiled file, positioned just after - the header; if not, return NULL. - Doesn't set an exception. */ - -static FILE * -check_compiled_module(PyObject *pathname, struct stat *srcstat, PyObject *cpathname) -{ - FILE *fp; - long magic; - long pyc_mtime; - long pyc_size; - - fp = _Py_fopen(cpathname, "rb"); - if (fp == NULL) - return NULL; - magic = PyMarshal_ReadLongFromFile(fp); - if (magic != pyc_magic) { - if (Py_VerboseFlag) - PySys_FormatStderr("# %R has bad magic\n", cpathname); - fclose(fp); - return NULL; - } - pyc_mtime = PyMarshal_ReadLongFromFile(fp); - if (pyc_mtime != srcstat->st_mtime) { - if (Py_VerboseFlag) - PySys_FormatStderr("# %R has bad mtime\n", cpathname); - fclose(fp); - return NULL; - } - pyc_size = PyMarshal_ReadLongFromFile(fp); - if (pyc_size != (srcstat->st_size & 0xFFFFFFFF)) { - if (Py_VerboseFlag) - PySys_FormatStderr("# %R has bad size\n", cpathname); - fclose(fp); - return NULL; - } - if (Py_VerboseFlag) - PySys_FormatStderr("# %R matches %R\n", cpathname, pathname); - return fp; -} - /* Read a code object from a file and check it for validity */ @@ -1178,238 +1114,6 @@ load_compiled_module(PyObject *name, PyObject *cpathname, FILE *fp) return m; } -/* Parse a source file and return the corresponding code object */ - -static PyCodeObject * -parse_source_module(PyObject *pathname, FILE *fp) -{ - PyCodeObject *co; - PyObject *pathbytes; - mod_ty mod; - PyCompilerFlags flags; - PyArena *arena; - - pathbytes = PyUnicode_EncodeFSDefault(pathname); - if (pathbytes == NULL) - return NULL; - - arena = PyArena_New(); - if (arena == NULL) { - Py_DECREF(pathbytes); - return NULL; - } - - flags.cf_flags = 0; - mod = PyParser_ASTFromFile(fp, PyBytes_AS_STRING(pathbytes), NULL, - Py_file_input, 0, 0, &flags, - NULL, arena); - if (mod != NULL) - co = PyAST_Compile(mod, PyBytes_AS_STRING(pathbytes), NULL, arena); - else - co = NULL; - Py_DECREF(pathbytes); - PyArena_Free(arena); - return co; -} - -/* Write a compiled module to a file, placing the time of last - modification of its source into the header. - Errors are ignored, if a write error occurs an attempt is made to - remove the file. */ - -static void -write_compiled_module(PyCodeObject *co, PyObject *cpathname, - struct stat *srcstat) -{ - Py_UCS4 *cpathname_ucs4; - FILE *fp; - time_t mtime = srcstat->st_mtime; - long size = srcstat->st_size & 0xFFFFFFFF; - PyObject *cpathname_tmp; -#ifdef MS_WINDOWS /* since Windows uses different permissions */ - mode_t mode = srcstat->st_mode & ~S_IEXEC; - wchar_t *wdirname, *wpathname, *wpathname_tmp; -#else - mode_t dirmode = (srcstat->st_mode | - S_IXUSR | S_IXGRP | S_IXOTH | - S_IWUSR | S_IWGRP | S_IWOTH); - PyObject *dirbytes; - PyObject *cpathbytes, *cpathbytes_tmp; -#endif - int fd; - PyObject *dirname; - Py_UCS4 *dirsep; - int res, ok; - - /* Ensure that the __pycache__ directory exists. */ - cpathname_ucs4 = PyUnicode_AsUCS4Copy(cpathname); - if (!cpathname_ucs4) - return; - dirsep = rightmost_sep(cpathname_ucs4); - if (dirsep == NULL) { - if (Py_VerboseFlag) - PySys_FormatStderr("# no %s path found %R\n", CACHEDIR, cpathname); - return; - } - dirname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - cpathname_ucs4, - dirsep - cpathname_ucs4); - PyMem_Free(cpathname_ucs4); - if (dirname == NULL) { - PyErr_Clear(); - return; - } - -#ifdef MS_WINDOWS - wdirname = PyUnicode_AsUnicode(dirname); - if (wdirname == NULL) { - PyErr_Clear(); - return; - } - res = CreateDirectoryW(wdirname, NULL); - ok = (res != 0); - if (!ok && GetLastError() == ERROR_ALREADY_EXISTS) - ok = 1; -#else - dirbytes = PyUnicode_EncodeFSDefault(dirname); - if (dirbytes == NULL) { - PyErr_Clear(); - return; - } - res = mkdir(PyBytes_AS_STRING(dirbytes), dirmode); - Py_DECREF(dirbytes); - if (0 <= res) - ok = 1; - else - ok = (errno == EEXIST); -#endif - if (!ok) { - if (Py_VerboseFlag) - PySys_FormatStderr("# cannot create cache dir %R\n", dirname); - Py_DECREF(dirname); - return; - } - Py_DECREF(dirname); - - /* We first write to a tmp file and then take advantage - of atomic renaming (which *should* be true even under Windows). - As in importlib, we use id(something) to generate a pseudo-random - filename. mkstemp() can't be used since it doesn't allow specifying - the file access permissions. - */ - cpathname_tmp = PyUnicode_FromFormat("%U.%zd", - cpathname, (Py_ssize_t) co); - if (cpathname_tmp == NULL) { - PyErr_Clear(); - return; - } -#ifdef MS_WINDOWS - wpathname = PyUnicode_AsUnicode(cpathname); - if (wpathname == NULL) { - PyErr_Clear(); - return; - } - wpathname_tmp = PyUnicode_AsUnicode(cpathname_tmp); - if (wpathname_tmp == NULL) { - PyErr_Clear(); - return; - } - - (void)DeleteFileW(wpathname_tmp); - fd = _wopen(wpathname_tmp, - O_EXCL | O_CREAT | O_WRONLY | O_BINARY, - mode); - if (0 <= fd) - fp = fdopen(fd, "wb"); - else - fp = NULL; -#else - cpathbytes_tmp = PyUnicode_EncodeFSDefault(cpathname_tmp); - Py_DECREF(cpathname_tmp); - if (cpathbytes_tmp == NULL) { - PyErr_Clear(); - return; - } - cpathbytes = PyUnicode_EncodeFSDefault(cpathname); - if (cpathbytes == NULL) { - PyErr_Clear(); - return; - } - fd = open(PyBytes_AS_STRING(cpathbytes_tmp), - O_CREAT | O_EXCL | O_WRONLY, 0666); - if (0 <= fd) - fp = fdopen(fd, "wb"); - else - fp = NULL; -#endif - if (fp == NULL) { - if (Py_VerboseFlag) - PySys_FormatStderr( - "# can't create %R\n", cpathname); -#ifdef MS_WINDOWS - Py_DECREF(cpathname_tmp); -#else - Py_DECREF(cpathbytes); - Py_DECREF(cpathbytes_tmp); -#endif - return; - } - PyMarshal_WriteLongToFile(pyc_magic, fp, Py_MARSHAL_VERSION); - /* First write a 0 for mtime and size */ - PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION); - PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION); - PyMarshal_WriteObjectToFile((PyObject *)co, fp, Py_MARSHAL_VERSION); - fflush(fp); - /* Now write the true mtime and size (as 32-bit fields) */ - fseek(fp, 4L, 0); - assert(mtime <= 0xFFFFFFFF); - PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION); - PyMarshal_WriteLongToFile(size, fp, Py_MARSHAL_VERSION); - if (fflush(fp) != 0 || ferror(fp)) { - if (Py_VerboseFlag) - PySys_FormatStderr("# can't write %R\n", cpathname); - /* Don't keep partial file */ - fclose(fp); -#ifdef MS_WINDOWS - (void)DeleteFileW(wpathname_tmp); - Py_DECREF(cpathname_tmp); -#else - (void) unlink(PyBytes_AS_STRING(cpathbytes_tmp)); - Py_DECREF(cpathbytes); - Py_DECREF(cpathbytes_tmp); -#endif - return; - } - fclose(fp); - /* Do a (hopefully) atomic rename */ -#ifdef MS_WINDOWS - if (!MoveFileExW(wpathname_tmp, wpathname, MOVEFILE_REPLACE_EXISTING)) { - if (Py_VerboseFlag) - PySys_FormatStderr("# can't write %R\n", cpathname); - /* Don't keep tmp file */ - (void) DeleteFileW(wpathname_tmp); - Py_DECREF(cpathname_tmp); - return; - } - Py_DECREF(cpathname_tmp); -#else - if (rename(PyBytes_AS_STRING(cpathbytes_tmp), - PyBytes_AS_STRING(cpathbytes))) { - if (Py_VerboseFlag) - PySys_FormatStderr("# can't write %R\n", cpathname); - /* Don't keep tmp file */ - unlink(PyBytes_AS_STRING(cpathbytes_tmp)); - Py_DECREF(cpathbytes); - Py_DECREF(cpathbytes_tmp); - return; - } - Py_DECREF(cpathbytes); - Py_DECREF(cpathbytes_tmp); -#endif - if (Py_VerboseFlag) - PySys_FormatStderr("# wrote %R\n", cpathname); -} - static void update_code_filenames(PyCodeObject *co, PyObject *oldname, PyObject *newname) { @@ -1474,76 +1178,6 @@ imp_fix_co_filename(PyObject *self, PyObject *args) Py_RETURN_NONE; } -/* Load a source module from a given file and return its module - object WITH INCREMENTED REFERENCE COUNT. If there's a matching - byte-compiled file, use that instead. */ - -static PyObject * -load_source_module(PyObject *name, PyObject *pathname, FILE *fp) -{ - struct stat st; - FILE *fpc; - PyObject *cpathname = NULL, *cpathbytes = NULL; - PyCodeObject *co; - PyObject *m = NULL; - - if (fstat(fileno(fp), &st) != 0) { - PyErr_Format(PyExc_RuntimeError, - "unable to get file status from %R", - pathname); - goto error; - } - if (sizeof st.st_mtime > 4) { - /* Python's .pyc timestamp handling presumes that the timestamp fits - in 4 bytes. Since the code only does an equality comparison, - ordering is not important and we can safely ignore the higher bits - (collisions are extremely unlikely). - */ - st.st_mtime &= 0xFFFFFFFF; - } - if (PyUnicode_READY(pathname) < 0) - return NULL; - cpathname = make_compiled_pathname(pathname, !Py_OptimizeFlag); - - if (cpathname != NULL) - fpc = check_compiled_module(pathname, &st, cpathname); - else - fpc = NULL; - - if (fpc) { - co = read_compiled_module(cpathname, fpc); - fclose(fpc); - if (co == NULL) - goto error; - update_compiled_module(co, pathname); - if (Py_VerboseFlag) - PySys_FormatStderr("import %U # precompiled from %R\n", - name, cpathname); - m = PyImport_ExecCodeModuleObject(name, (PyObject *)co, - cpathname, cpathname); - } - else { - co = parse_source_module(pathname, fp); - if (co == NULL) - goto error; - if (Py_VerboseFlag) - PySys_FormatStderr("import %U # from %R\n", - name, pathname); - if (cpathname != NULL) { - PyObject *ro = PySys_GetObject("dont_write_bytecode"); - if (ro == NULL || !PyObject_IsTrue(ro)) - write_compiled_module(co, cpathname, &st); - } - m = PyImport_ExecCodeModuleObject(name, (PyObject *)co, - pathname, cpathname); - } - Py_DECREF(co); - -error: - Py_XDECREF(cpathbytes); - Py_XDECREF(cpathname); - return m; -} /* Get source file -> unicode or None * Returns the path to the py file if available, else the given path @@ -3426,29 +3060,6 @@ imp_load_dynamic(PyObject *self, PyObject *args) #endif /* HAVE_DYNAMIC_LOADING */ -static PyObject * -imp_load_source(PyObject *self, PyObject *args) -{ - PyObject *name, *pathname; - PyObject *fob = NULL; - PyObject *m; - FILE *fp; - if (!PyArg_ParseTuple(args, "UO&|O:load_source", - &name, - PyUnicode_FSDecoder, &pathname, - &fob)) - return NULL; - fp = get_file(pathname, fob, "r"); - if (fp == NULL) { - Py_DECREF(pathname); - return NULL; - } - m = load_source_module(name, pathname, fp); - Py_DECREF(pathname); - fclose(fp); - return m; -} - static PyObject * imp_reload(PyObject *self, PyObject *v) { @@ -3600,7 +3211,6 @@ static PyMethodDef imp_methods[] = { #ifdef HAVE_DYNAMIC_LOADING {"load_dynamic", imp_load_dynamic, METH_VARARGS}, #endif - {"load_source", imp_load_source, METH_VARARGS}, {"_fix_co_filename", imp_fix_co_filename, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; -- cgit v1.2.1 From 5d8520121e1ea38bcfe37258449a946ee328cac1 Mon Sep 17 00:00:00 2001 From: Brian Curtin Date: Tue, 17 Apr 2012 16:57:09 -0500 Subject: Fix #14600. Correct reference handling and naming of ImportError convenience function --- Python/import.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 584f30eb38..f3de7d8262 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2460,7 +2460,8 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, PyObject *msg = PyUnicode_FromFormat("import of %R halted; " "None in sys.modules", abs_name); if (msg != NULL) { - PyErr_SetFromImportErrorWithName(msg, abs_name); + PyErr_SetImportError(msg, abs_name, NULL); + Py_DECREF(msg); } mod = NULL; goto error_with_unlock; -- cgit v1.2.1 From 9459c0f7879ddbb51a8e8da936c1c41113b077fa Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 17 Apr 2012 19:05:11 -0400 Subject: Issue #14592: A relative import will raise a KeyError if __package__ or __name__ are not set in globals. Thanks to Stefan Behnel for the bug report. --- Python/import.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index f3de7d8262..3e620b3706 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2355,8 +2355,9 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, } } else { - package = _PyDict_GetItemIdWithError(globals, &PyId___name__); + package = _PyDict_GetItemId(globals, &PyId___name__); if (package == NULL) { + PyErr_SetString(PyExc_KeyError, "'__name__' not in globals"); goto error; } else if (!PyUnicode_Check(package)) { -- cgit v1.2.1 From 8544db0e8a92dab860693980b6325b3cafe4d4af Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 17 Apr 2012 19:14:26 -0400 Subject: Issue #13959: Re-implement imp.load_compiled() in imp.py. --- Python/import.c | 76 --------------------------------------------------------- 1 file changed, 76 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 3e620b3706..07c3d6f40c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1062,58 +1062,6 @@ make_source_pathname(PyObject *path) } -/* Read a code object from a file and check it for validity */ - -static PyCodeObject * -read_compiled_module(PyObject *cpathname, FILE *fp) -{ - PyObject *co; - - co = PyMarshal_ReadLastObjectFromFile(fp); - if (co == NULL) - return NULL; - if (!PyCode_Check(co)) { - PyErr_Format(PyExc_ImportError, - "Non-code object in %R", cpathname); - Py_DECREF(co); - return NULL; - } - return (PyCodeObject *)co; -} - - -/* Load a module from a compiled file, execute it, and return its - module object WITH INCREMENTED REFERENCE COUNT */ - -static PyObject * -load_compiled_module(PyObject *name, PyObject *cpathname, FILE *fp) -{ - long magic; - PyCodeObject *co; - PyObject *m; - - magic = PyMarshal_ReadLongFromFile(fp); - if (magic != pyc_magic) { - PyErr_Format(PyExc_ImportError, - "Bad magic number in %R", cpathname); - return NULL; - } - /* Skip mtime and size */ - (void) PyMarshal_ReadLongFromFile(fp); - (void) PyMarshal_ReadLongFromFile(fp); - co = read_compiled_module(cpathname, fp); - if (co == NULL) - return NULL; - if (Py_VerboseFlag) - PySys_FormatStderr("import %U # precompiled from %R\n", - name, cpathname); - m = PyImport_ExecCodeModuleObject(name, (PyObject *)co, - cpathname, cpathname); - Py_DECREF(co); - - return m; -} - static void update_code_filenames(PyCodeObject *co, PyObject *oldname, PyObject *newname) { @@ -3010,29 +2958,6 @@ get_file(PyObject *pathname, PyObject *fob, char *mode) } } -static PyObject * -imp_load_compiled(PyObject *self, PyObject *args) -{ - PyObject *name, *pathname; - PyObject *fob = NULL; - PyObject *m; - FILE *fp; - if (!PyArg_ParseTuple(args, "UO&|O:load_compiled", - &name, - PyUnicode_FSDecoder, &pathname, - &fob)) - return NULL; - fp = get_file(pathname, fob, "rb"); - if (fp == NULL) { - Py_DECREF(pathname); - return NULL; - } - m = load_compiled_module(name, pathname, fp); - fclose(fp); - Py_DECREF(pathname); - return m; -} - #ifdef HAVE_DYNAMIC_LOADING static PyObject * @@ -3209,7 +3134,6 @@ static PyMethodDef imp_methods[] = { {"init_frozen", imp_init_frozen, METH_VARARGS}, {"is_builtin", imp_is_builtin, METH_VARARGS}, {"is_frozen", imp_is_frozen, METH_VARARGS}, - {"load_compiled", imp_load_compiled, METH_VARARGS}, #ifdef HAVE_DYNAMIC_LOADING {"load_dynamic", imp_load_dynamic, METH_VARARGS}, #endif -- cgit v1.2.1 From dfd572c80d2888ed84d050b9cbbaef1eacf7506f Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Wed, 18 Apr 2012 10:55:43 -0400 Subject: rollback 005fd1fe31ab (see #14609 and #14582) Being able to overload a sys.module entry during import of a module was broken by this changeset. --- Python/import.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 07c3d6f40c..11d58aefe4 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2447,22 +2447,15 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, Py_DECREF(partition); if (level == 0) { - if (PyUnicode_GET_LENGTH(name) == - PyUnicode_GET_LENGTH(front)) { - final_mod = mod; - } - else { - final_mod = PyDict_GetItem(interp->modules, front); - if (final_mod == NULL) { - PyErr_Format(PyExc_KeyError, - "%R not in sys.modules as expected", front); - } - } + final_mod = PyDict_GetItem(interp->modules, front); Py_DECREF(front); if (final_mod == NULL) { - goto error_with_unlock; + PyErr_Format(PyExc_KeyError, + "%R not in sys.modules as expected", front); + } + else { + Py_INCREF(final_mod); } - Py_INCREF(final_mod); } else { Py_ssize_t cut_off = PyUnicode_GET_LENGTH(name) - -- cgit v1.2.1 From 20a5054c215951276c9a2f74ef2da0468f5c6a6c Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 20 Apr 2012 21:44:46 -0400 Subject: Issue #13959: Re-implement imp.cache_from_source() in Lib/imp.py. --- Python/import.c | 111 -------------------------------------------------------- 1 file changed, 111 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 11d58aefe4..7298ca51e6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -783,7 +783,6 @@ remove_module(PyObject *name) static PyObject * get_sourcefile(PyObject *filename); static PyObject *make_source_pathname(PyObject *pathname); -static PyObject* make_compiled_pathname(PyObject *pathname, int debug); /* Execute a code object in a module and return the module object * WITH INCREMENTED REFERENCE COUNT. If an error occurs, name is @@ -924,71 +923,6 @@ rightmost_sep_obj(PyObject* o, Py_ssize_t start, Py_ssize_t end) return found; } -/* Given a pathname for a Python source file, fill a buffer with the - pathname for the corresponding compiled file. Return the pathname - for the compiled file, or NULL if there's no space in the buffer. - Doesn't set an exception. - - foo.py -> __pycache__/foo..pyc - - pathstr is assumed to be "ready". -*/ - -static PyObject* -make_compiled_pathname(PyObject *pathstr, int debug) -{ - PyObject *result; - Py_ssize_t fname, ext, len, i, pos, taglen; - Py_ssize_t pycache_len = sizeof(CACHEDIR) - 1; - int kind; - void *data; - Py_UCS4 lastsep; - - /* Compute the output string size. */ - len = PyUnicode_GET_LENGTH(pathstr); - /* If there is no separator, this returns -1, so - fname will be 0. */ - fname = rightmost_sep_obj(pathstr, 0, len) + 1; - /* Windows: re-use the last separator character (/ or \\) when - appending the __pycache__ path. */ - if (fname > 0) - lastsep = PyUnicode_READ_CHAR(pathstr, fname -1); - else - lastsep = SEP; - ext = fname - 1; - for(i = fname; i < len; i++) - if (PyUnicode_READ_CHAR(pathstr, i) == '.') - ext = i + 1; - if (ext < fname) - /* No dot in filename; use entire filename */ - ext = len; - - /* result = pathstr[:fname] + "__pycache__" + SEP + - pathstr[fname:ext] + tag + ".py[co]" */ - taglen = strlen(pyc_tag); - result = PyUnicode_New(ext + pycache_len + 1 + taglen + 4, - PyUnicode_MAX_CHAR_VALUE(pathstr)); - if (!result) - return NULL; - kind = PyUnicode_KIND(result); - data = PyUnicode_DATA(result); - PyUnicode_CopyCharacters(result, 0, pathstr, 0, fname); - pos = fname; - for (i = 0; i < pycache_len; i++) - PyUnicode_WRITE(kind, data, pos++, CACHEDIR[i]); - PyUnicode_WRITE(kind, data, pos++, lastsep); - PyUnicode_CopyCharacters(result, pos, pathstr, - fname, ext - fname); - pos += ext - fname; - for (i = 0; pyc_tag[i]; i++) - PyUnicode_WRITE(kind, data, pos++, pyc_tag[i]); - PyUnicode_WRITE(kind, data, pos++, '.'); - PyUnicode_WRITE(kind, data, pos++, 'p'); - PyUnicode_WRITE(kind, data, pos++, 'y'); - PyUnicode_WRITE(kind, data, pos++, debug ? 'c' : 'o'); - return result; -} - /* Given a pathname to a Python byte compiled file, return the path to the source file, if the path matches the PEP 3147 format. This does not check @@ -2991,49 +2925,6 @@ PyDoc_STRVAR(doc_reload, \n\ Reload the module. The module must have been successfully imported before."); -static PyObject * -imp_cache_from_source(PyObject *self, PyObject *args, PyObject *kws) -{ - static char *kwlist[] = {"path", "debug_override", NULL}; - - PyObject *pathname, *cpathname; - PyObject *debug_override = NULL; - int debug = !Py_OptimizeFlag; - - if (!PyArg_ParseTupleAndKeywords( - args, kws, "O&|O", kwlist, - PyUnicode_FSDecoder, &pathname, &debug_override)) - return NULL; - - if (debug_override != NULL && - (debug = PyObject_IsTrue(debug_override)) < 0) { - Py_DECREF(pathname); - return NULL; - } - - if (PyUnicode_READY(pathname) < 0) - return NULL; - - cpathname = make_compiled_pathname(pathname, debug); - Py_DECREF(pathname); - - if (cpathname == NULL) { - PyErr_Format(PyExc_SystemError, "path buffer too short"); - return NULL; - } - return cpathname; -} - -PyDoc_STRVAR(doc_cache_from_source, -"cache_from_source(path, [debug_override]) -> path\n\ -Given the path to a .py file, return the path to its .pyc/.pyo file.\n\ -\n\ -The .py file does not need to exist; this simply returns the path to the\n\ -.pyc/.pyo file calculated as if the .py file were imported. The extension\n\ -will be .pyc unless __debug__ is not defined, then it will be .pyo.\n\ -\n\ -If debug_override is not None, then it must be a boolean and is taken as\n\ -the value of __debug__ instead."); static PyObject * imp_source_from_cache(PyObject *self, PyObject *args, PyObject *kws) @@ -3116,8 +3007,6 @@ static PyMethodDef imp_methods[] = { {"acquire_lock", imp_acquire_lock, METH_NOARGS, doc_acquire_lock}, {"release_lock", imp_release_lock, METH_NOARGS, doc_release_lock}, {"reload", imp_reload, METH_O, doc_reload}, - {"cache_from_source", (PyCFunction)imp_cache_from_source, - METH_VARARGS | METH_KEYWORDS, doc_cache_from_source}, {"source_from_cache", (PyCFunction)imp_source_from_cache, METH_VARARGS | METH_KEYWORDS, doc_source_from_cache}, /* The rest are obsolete */ -- cgit v1.2.1 From c276765b01cb8a531e0204e439053a26691df495 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sat, 21 Apr 2012 18:52:52 -0400 Subject: Issue #13959: Re-implement imp.source_from_cache() in Lib/imp.py. --- Python/import.c | 33 --------------------------------- 1 file changed, 33 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 7298ca51e6..bcf6bd7622 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2926,36 +2926,6 @@ PyDoc_STRVAR(doc_reload, Reload the module. The module must have been successfully imported before."); -static PyObject * -imp_source_from_cache(PyObject *self, PyObject *args, PyObject *kws) -{ - static char *kwlist[] = {"path", NULL}; - PyObject *pathname, *source; - - if (!PyArg_ParseTupleAndKeywords( - args, kws, "O&", kwlist, - PyUnicode_FSDecoder, &pathname)) - return NULL; - - source = make_source_pathname(pathname); - if (source == NULL) { - PyErr_Format(PyExc_ValueError, "Not a PEP 3147 pyc path: %R", - pathname); - Py_DECREF(pathname); - return NULL; - } - Py_DECREF(pathname); - return source; -} - -PyDoc_STRVAR(doc_source_from_cache, -"source_from_cache(path) -> path\n\ -Given the path to a .pyc./.pyo file, return the path to its .py file.\n\ -\n\ -The .pyc/.pyo file does not need to exist; this simply returns the path to\n\ -the .py file calculated to correspond to the .pyc/.pyo file. If path\n\ -does not conform to PEP 3147 format, ValueError will be raised."); - /* Doc strings */ PyDoc_STRVAR(doc_imp, @@ -3007,9 +2977,6 @@ static PyMethodDef imp_methods[] = { {"acquire_lock", imp_acquire_lock, METH_NOARGS, doc_acquire_lock}, {"release_lock", imp_release_lock, METH_NOARGS, doc_release_lock}, {"reload", imp_reload, METH_O, doc_reload}, - {"source_from_cache", (PyCFunction)imp_source_from_cache, - METH_VARARGS | METH_KEYWORDS, doc_source_from_cache}, - /* The rest are obsolete */ {"get_frozen_object", imp_get_frozen_object, METH_VARARGS}, {"is_frozen_package", imp_is_frozen_package, METH_VARARGS}, {"init_builtin", imp_init_builtin, METH_VARARGS}, -- cgit v1.2.1 From 06fad5af9464da87657bf0815c832e8f5d835b43 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sat, 21 Apr 2012 21:09:46 -0400 Subject: Issue #13959: Re-implement imp.find_module() in Lib/imp.py. Thanks to Eric Snow for taking an initial stab at the implementation. --- Python/import.c | 751 -------------------------------------------------------- 1 file changed, 751 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index bcf6bd7622..ab320e523c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1118,8 +1118,6 @@ unchanged: } /* Forward */ -static struct filedescr *find_module(PyObject *, PyObject *, PyObject *, - PyObject **, FILE **, PyObject **); static struct _frozen * find_frozen(PyObject *); @@ -1220,678 +1218,12 @@ PyImport_GetImporter(PyObject *path) { return importer; } -/* Search the path (default sys.path) for a module. Return the - corresponding filedescr struct, and (via return arguments) the - pathname and an open file. Return NULL if the module is not found. */ #ifdef MS_COREDLL extern FILE *_PyWin_FindRegisteredModule(PyObject *, struct filedescr **, PyObject **p_path); #endif -/* Forward */ -static int case_ok(PyObject *, Py_ssize_t, PyObject *); -static int find_init_module(PyObject *); -static struct filedescr importhookdescr = {"", "", IMP_HOOK}; - -/* Get the path of a module: get its importer and call importer.find_module() - hook, or check if the module if a package (if path/__init__.py exists). - - -1: error: a Python error occurred - 0: ignore: an error occurred because of invalid data, but the error is not - important enough to be reported. - 1: get path: module not found, but *buf contains its path - 2: found: *p_fd is the file descriptor (IMP_HOOK or PKG_DIRECTORY) - and *buf is the path */ - -static int -find_module_path(PyObject *fullname, PyObject *name, PyObject *path, - PyObject *path_hooks, PyObject *path_importer_cache, - PyObject **p_path, PyObject **p_loader, struct filedescr **p_fd) -{ - PyObject *path_unicode, *filename = NULL; - Py_ssize_t len, pos; - struct stat statbuf; - static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; - int err, result, addsep; - - if (PyUnicode_Check(path)) { - Py_INCREF(path); - path_unicode = path; - } - else if (PyBytes_Check(path)) { - path_unicode = PyUnicode_DecodeFSDefaultAndSize( - PyBytes_AS_STRING(path), PyBytes_GET_SIZE(path)); - if (path_unicode == NULL) - return -1; - } - else - return 0; - - if (PyUnicode_READY(path_unicode)) - return -1; - - len = PyUnicode_GET_LENGTH(path_unicode); - if (PyUnicode_FindChar(path_unicode, 0, 0, len, 1) != -1) { - result = 0; - goto out; /* path contains '\0' */ - } - - /* sys.path_hooks import hook */ - if (p_loader != NULL) { - _Py_IDENTIFIER(find_module); - PyObject *importer; - - importer = get_path_importer(path_importer_cache, - path_hooks, path); - if (importer == NULL) { - result = -1; - goto out; - } - /* Note: importer is a borrowed reference */ - if (importer != Py_None) { - PyObject *loader; - loader = _PyObject_CallMethodId(importer, - &PyId_find_module, "O", fullname); - if (loader == NULL) { - result = -1; /* error */ - goto out; - } - if (loader != Py_None) { - /* a loader was found */ - *p_loader = loader; - *p_fd = &importhookdescr; - result = 2; - goto out; - } - Py_DECREF(loader); - result = 0; - goto out; - } - } - /* no hook was found, use builtin import */ - - addsep = 0; - if (len > 0 && PyUnicode_READ_CHAR(path_unicode, len-1) != SEP -#ifdef ALTSEP - && PyUnicode_READ_CHAR(path_unicode, len-1) != ALTSEP -#endif - ) - addsep = 1; - filename = PyUnicode_New(len + PyUnicode_GET_LENGTH(name) + addsep, - Py_MAX(PyUnicode_MAX_CHAR_VALUE(path_unicode), - PyUnicode_MAX_CHAR_VALUE(name))); - if (filename == NULL) { - result = -1; - goto out; - } - PyUnicode_CopyCharacters(filename, 0, path_unicode, 0, len); - pos = len; - if (addsep) - PyUnicode_WRITE(PyUnicode_KIND(filename), - PyUnicode_DATA(filename), - pos++, SEP); - PyUnicode_CopyCharacters(filename, pos, name, 0, - PyUnicode_GET_LENGTH(name)); - - /* Check for package import (buf holds a directory name, - and there's an __init__ module in that directory */ -#ifdef HAVE_STAT - err = _Py_stat(filename, &statbuf); - if (err == -2) { - result = -1; - goto out; - } - if (err == 0 && /* it exists */ - S_ISDIR(statbuf.st_mode)) /* it's a directory */ - { - int match; - - match = case_ok(filename, 0, name); - if (match < 0) { - result = -1; - goto out; - } - if (match) { /* case matches */ - if (find_init_module(filename)) { /* and has __init__.py */ - *p_path = filename; - filename = NULL; - *p_fd = &fd_package; - result = 2; - goto out; - } - else { - int err; - err = PyErr_WarnFormat(PyExc_ImportWarning, 1, - "Not importing directory %R: missing __init__.py", - filename); - if (err) { - result = -1; - goto out; - } - } - } - } -#endif - *p_path = filename; - filename = NULL; - result = 1; - out: - Py_DECREF(path_unicode); - Py_XDECREF(filename); - return result; -} - -/* Find a module in search_path_list. For each path, try - find_module_path() or try each _PyImport_Filetab suffix. - - If the module is found, return a file descriptor, write the path in - *p_filename, write the pointer to the file object into *p_fp, and (if - p_loader is not NULL) the loader into *p_loader. - - Otherwise, raise an exception and return NULL. */ - -static struct filedescr* -find_module_path_list(PyObject *fullname, PyObject *name, - PyObject *search_path_list, PyObject *path_hooks, - PyObject *path_importer_cache, - PyObject **p_path, FILE **p_fp, PyObject **p_loader) -{ - Py_ssize_t i, npath; - struct filedescr *fdp = NULL; - char *filemode; - FILE *fp = NULL; - PyObject *prefix, *filename; - int match; - int err; - - npath = PyList_Size(search_path_list); - for (i = 0; i < npath; i++) { - PyObject *path; - int ok; - - path = PyList_GetItem(search_path_list, i); - if (path == NULL) - return NULL; - - prefix = NULL; - ok = find_module_path(fullname, name, path, - path_hooks, path_importer_cache, - &prefix, p_loader, &fdp); - if (ok < 0) - return NULL; - if (ok == 0) - continue; - if (ok == 2) { - *p_path = prefix; - return fdp; - } - - for (fdp = _PyImport_Filetab; fdp->suffix != NULL; fdp++) { - struct stat statbuf; - - filemode = fdp->mode; - if (filemode[0] == 'U') - filemode = "r" PY_STDIOTEXTMODE; - - filename = PyUnicode_FromFormat("%U%s", prefix, fdp->suffix); - if (filename == NULL) { - Py_DECREF(prefix); - return NULL; - } - - if (Py_VerboseFlag > 1) - PySys_FormatStderr("# trying %R\n", filename); - - err = _Py_stat(filename, &statbuf); - if (err == -2) { - Py_DECREF(prefix); - Py_DECREF(filename); - return NULL; - } - if (err != 0 || S_ISDIR(statbuf.st_mode)) { - /* it doesn't exist, or it's a directory */ - Py_DECREF(filename); - continue; - } - - fp = _Py_fopen(filename, filemode); - if (fp == NULL) { - Py_DECREF(filename); - if (PyErr_Occurred()) { - Py_DECREF(prefix); - return NULL; - } - continue; - } - match = case_ok(filename, -(Py_ssize_t)strlen(fdp->suffix), name); - if (match < 0) { - Py_DECREF(prefix); - Py_DECREF(filename); - return NULL; - } - if (match) { - Py_DECREF(prefix); - *p_path = filename; - *p_fp = fp; - return fdp; - } - Py_DECREF(filename); - - fclose(fp); - fp = NULL; - } - Py_DECREF(prefix); - } - PyErr_Format(PyExc_ImportError, - "No module named %R", name); - return NULL; -} - -/* Find a module: - - - try find_module() of each sys.meta_path hook - - try find_frozen() - - try is_builtin() - - try _PyWin_FindRegisteredModule() (Windows only) - - otherwise, call find_module_path_list() with search_path_list (if not - NULL) or sys.path - - fullname can be NULL, but only if p_loader is NULL. - - Return: - - - &fd_builtin (C_BUILTIN) if it is a builtin - - &fd_frozen (PY_FROZEN) if it is frozen - - &fd_package (PKG_DIRECTORY) and write the filename into *p_path - if it is a package - - &importhookdescr (IMP_HOOK) and write the loader into *p_loader if a - importer loader was found - - a file descriptor (PY_SOURCE, PY_COMPILED, C_EXTENSION, PY_RESOURCE or - PY_CODERESOURCE: see _PyImport_Filetab), write the filename into - *p_path and the pointer to the open file into *p_fp - - NULL on error - - By default, *p_path, *p_fp and *p_loader (if set) are set to NULL. - Eg. *p_path is set to NULL for a builtin package. -*/ - -static struct filedescr * -find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, - PyObject **p_path, FILE **p_fp, PyObject **p_loader) -{ - Py_ssize_t i, npath; - static struct filedescr fd_frozen = {"", "", PY_FROZEN}; - static struct filedescr fd_builtin = {"", "", C_BUILTIN}; - PyObject *path_hooks, *path_importer_cache; - - *p_path = NULL; - *p_fp = NULL; - if (p_loader != NULL) - *p_loader = NULL; - - /* sys.meta_path import hook */ - if (p_loader != NULL) { - _Py_IDENTIFIER(find_module); - PyObject *meta_path; - - meta_path = PySys_GetObject("meta_path"); - if (meta_path == NULL || !PyList_Check(meta_path)) { - PyErr_SetString(PyExc_RuntimeError, - "sys.meta_path must be a list of " - "import hooks"); - return NULL; - } - Py_INCREF(meta_path); /* zap guard */ - npath = PyList_Size(meta_path); - for (i = 0; i < npath; i++) { - PyObject *loader; - PyObject *hook = PyList_GetItem(meta_path, i); - loader = _PyObject_CallMethodId(hook, &PyId_find_module, - "OO", fullname, - search_path_list != NULL ? - search_path_list : Py_None); - if (loader == NULL) { - Py_DECREF(meta_path); - return NULL; /* true error */ - } - if (loader != Py_None) { - /* a loader was found */ - *p_loader = loader; - Py_DECREF(meta_path); - return &importhookdescr; - } - Py_DECREF(loader); - } - Py_DECREF(meta_path); - } - - if (find_frozen(fullname) != NULL) - return &fd_frozen; - - if (search_path_list == NULL) { -#ifdef MS_COREDLL - FILE *fp; - struct filedescr *fdp; -#endif - if (is_builtin(name)) - return &fd_builtin; -#ifdef MS_COREDLL - fp = _PyWin_FindRegisteredModule(name, &fdp, p_path); - if (fp != NULL) { - *p_fp = fp; - return fdp; - } - else if (PyErr_Occurred()) - return NULL; -#endif - search_path_list = PySys_GetObject("path"); - } - - if (search_path_list == NULL || !PyList_Check(search_path_list)) { - PyErr_SetString(PyExc_RuntimeError, - "sys.path must be a list of directory names"); - return NULL; - } - - path_hooks = PySys_GetObject("path_hooks"); - if (path_hooks == NULL || !PyList_Check(path_hooks)) { - PyErr_SetString(PyExc_RuntimeError, - "sys.path_hooks must be a list of " - "import hooks"); - return NULL; - } - path_importer_cache = PySys_GetObject("path_importer_cache"); - if (path_importer_cache == NULL || - !PyDict_Check(path_importer_cache)) { - PyErr_SetString(PyExc_RuntimeError, - "sys.path_importer_cache must be a dict"); - return NULL; - } - - return find_module_path_list(fullname, name, - search_path_list, path_hooks, - path_importer_cache, - p_path, p_fp, p_loader); -} - -/* case_bytes(char* buf, Py_ssize_t len, Py_ssize_t namelen, char* name) - * The arguments here are tricky, best shown by example: - * /a/b/c/d/e/f/g/h/i/j/k/some_long_module_name.py\0 - * ^ ^ ^ ^ - * |--------------------- buf ---------------------| - * |------------------- len ------------------| - * |------ name -------| - * |----- namelen -----| - * buf is the full path, but len only counts up to (& exclusive of) the - * extension. name is the module name, also exclusive of extension. - * - * We've already done a successful stat() or fopen() on buf, so know that - * there's some match, possibly case-insensitive. - * - * case_bytes() is to return 1 if there's a case-sensitive match for - * name, else 0. case_bytes() is also to return 1 if envar PYTHONCASEOK - * exists. - * - * case_bytes() is used to implement case-sensitive import semantics even - * on platforms with case-insensitive filesystems. It's trivial to implement - * for case-sensitive filesystems. It's pretty much a cross-platform - * nightmare for systems with case-insensitive filesystems. - */ - -/* First we may need a pile of platform-specific header files; the sequence - * of #if's here should match the sequence in the body of case_bytes(). - */ -#if defined(MS_WINDOWS) -#include - -#elif defined(DJGPP) -#include - -#elif (defined(__MACH__) && defined(__APPLE__) || defined(__CYGWIN__)) && defined(HAVE_DIRENT_H) -#include -#include - -#elif defined(PYOS_OS2) -#define INCL_DOS -#define INCL_DOSERRORS -#define INCL_NOPMAPI -#include -#endif - -#if defined(DJGPP) \ - || ((defined(__MACH__) && defined(__APPLE__) || defined(__CYGWIN__)) \ - && defined(HAVE_DIRENT_H)) \ - || defined(PYOS_OS2) -# define USE_CASE_OK_BYTES -#endif - - -#ifdef USE_CASE_OK_BYTES -static int -case_bytes(char *buf, Py_ssize_t len, Py_ssize_t namelen, const char *name) -{ -/* Pick a platform-specific implementation; the sequence of #if's here should - * match the sequence just above. - */ - -/* DJGPP */ -#if defined(DJGPP) - struct ffblk ffblk; - int done; - - if (Py_GETENV("PYTHONCASEOK") != NULL) - return 1; - - done = findfirst(buf, &ffblk, FA_ARCH|FA_RDONLY|FA_HIDDEN|FA_DIREC); - if (done) { - PyErr_Format(PyExc_NameError, - "Can't find file for module %.100s\n(filename %.300s)", - name, buf); - return -1; - } - return strncmp(ffblk.ff_name, name, namelen) == 0; - -/* new-fangled macintosh (macosx) or Cygwin */ -#elif (defined(__MACH__) && defined(__APPLE__) || defined(__CYGWIN__)) && defined(HAVE_DIRENT_H) - DIR *dirp; - struct dirent *dp; - char dirname[MAXPATHLEN + 1]; - const int dirlen = len - namelen - 1; /* don't want trailing SEP */ - - if (Py_GETENV("PYTHONCASEOK") != NULL) - return 1; - - /* Copy the dir component into dirname; substitute "." if empty */ - if (dirlen <= 0) { - dirname[0] = '.'; - dirname[1] = '\0'; - } - else { - assert(dirlen <= MAXPATHLEN); - memcpy(dirname, buf, dirlen); - dirname[dirlen] = '\0'; - } - /* Open the directory and search the entries for an exact match. */ - dirp = opendir(dirname); - if (dirp) { - char *nameWithExt = buf + len - namelen; - while ((dp = readdir(dirp)) != NULL) { - const int thislen = -#ifdef _DIRENT_HAVE_D_NAMELEN - dp->d_namlen; -#else - strlen(dp->d_name); -#endif - if (thislen >= namelen && - strcmp(dp->d_name, nameWithExt) == 0) { - (void)closedir(dirp); - return 1; /* Found */ - } - } - (void)closedir(dirp); - } - return 0 ; /* Not found */ - -/* OS/2 */ -#elif defined(PYOS_OS2) - HDIR hdir = 1; - ULONG srchcnt = 1; - FILEFINDBUF3 ffbuf; - APIRET rc; - - if (Py_GETENV("PYTHONCASEOK") != NULL) - return 1; - - rc = DosFindFirst(buf, - &hdir, - FILE_READONLY | FILE_HIDDEN | FILE_SYSTEM | FILE_DIRECTORY, - &ffbuf, sizeof(ffbuf), - &srchcnt, - FIL_STANDARD); - if (rc != NO_ERROR) - return 0; - return strncmp(ffbuf.achName, name, namelen) == 0; - -/* assuming it's a case-sensitive filesystem, so there's nothing to do! */ -#else -# error "USE_CASE_OK_BYTES is not correctly defined" -#endif -} -#endif - -/* - * Check if a filename case matchs the name case. We've already done a - * successful stat() or fopen() on buf, so know that there's some match, - * possibly case-insensitive. - * - * case_ok() is to return 1 if there's a case-sensitive match for name, 0 if it - * the filename doesn't match, or -1 on error. case_ok() is also to return 1 - * if envar PYTHONCASEOK exists. - * - * case_ok() is used to implement case-sensitive import semantics even - * on platforms with case-insensitive filesystems. It's trivial to implement - * for case-sensitive filesystems. It's pretty much a cross-platform - * nightmare for systems with case-insensitive filesystems. - */ - -static int -case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) -{ -#ifdef MS_WINDOWS - WIN32_FIND_DATAW data; - HANDLE h; - int cmp; - wchar_t *wfilename, *wname; - Py_ssize_t wname_len; - - if (Py_GETENV("PYTHONCASEOK") != NULL) - return 1; - - wfilename = PyUnicode_AsUnicode(filename); - if (wfilename == NULL) - return -1; - - h = FindFirstFileW(wfilename, &data); - if (h == INVALID_HANDLE_VALUE) { - PyErr_Format(PyExc_NameError, - "Can't find file for module %R\n(filename %R)", - name, filename); - return -1; - } - FindClose(h); - - wname = PyUnicode_AsUnicodeAndSize(name, &wname_len); - if (wname == NULL) - return -1; - - cmp = wcsncmp(data.cFileName, wname, wname_len); - return cmp == 0; -#elif defined(USE_CASE_OK_BYTES) - int match; - PyObject *filebytes, *namebytes; - filebytes = PyUnicode_EncodeFSDefault(filename); - if (filebytes == NULL) - return -1; - namebytes = PyUnicode_EncodeFSDefault(name); - if (namebytes == NULL) { - Py_DECREF(filebytes); - return -1; - } - match = case_bytes( - PyBytes_AS_STRING(filebytes), - PyBytes_GET_SIZE(filebytes) + prefix_delta, - PyBytes_GET_SIZE(namebytes), - PyBytes_AS_STRING(namebytes)); - Py_DECREF(filebytes); - Py_DECREF(namebytes); - return match; -#else - /* assuming it's a case-sensitive filesystem, so there's nothing to do! */ - return 1; - -#endif -} - -#ifdef HAVE_STAT - -/* Helper to look for __init__.py or __init__.py[co] in potential package. - Return 1 if __init__ was found, 0 if not, or -1 on error. */ -static int -find_init_module(PyObject *directory) -{ - struct stat statbuf; - PyObject *filename; - int match; - int err; - - filename = PyUnicode_FromFormat("%U%c__init__.py", directory, SEP); - if (filename == NULL) - return -1; - err = _Py_stat(filename, &statbuf); - if (err == -2) - return -1; - if (err == 0) { - /* 3=len(".py") */ - match = case_ok(filename, -3, initstr); - if (match < 0) { - Py_DECREF(filename); - return -1; - } - if (match) { - Py_DECREF(filename); - return 1; - } - } - Py_DECREF(filename); - - filename = PyUnicode_FromFormat("%U%c__init__.py%c", - directory, SEP, Py_OptimizeFlag ? 'o' : 'c'); - if (filename == NULL) - return -1; - err = _Py_stat(filename, &statbuf); - if (err == -2) { - Py_DECREF(filename); - return -1; - } - if (err == 0) { - /* 4=len(".pyc") */ - match = case_ok(filename, -4, initstr); - if (match < 0) { - Py_DECREF(filename); - return -1; - } - if (match) { - Py_DECREF(filename); - return 1; - } - } - Py_DECREF(filename); - return 0; -} - -#endif /* HAVE_STAT */ - static int init_builtin(PyObject *); /* Forward */ @@ -2688,81 +2020,6 @@ imp_get_suffixes(PyObject *self, PyObject *noargs) return list; } -static PyObject * -call_find_module(PyObject *name, PyObject *path_list) -{ - extern int fclose(FILE *); - PyObject *fob, *ret; - PyObject *pathobj; - struct filedescr *fdp; - FILE *fp; - int fd = -1; - char *found_encoding = NULL; - char *encoding = NULL; - - if (path_list == Py_None) - path_list = NULL; - fdp = find_module(NULL, name, path_list, - &pathobj, &fp, NULL); - if (fdp == NULL) - return NULL; - if (fp != NULL) { - fd = fileno(fp); - if (fd != -1) - fd = dup(fd); - fclose(fp); - fp = NULL; - if (fd == -1) - return PyErr_SetFromErrno(PyExc_OSError); - } - if (fd != -1) { - if (strchr(fdp->mode, 'b') == NULL) { - /* PyTokenizer_FindEncodingFilename() returns PyMem_MALLOC'ed - memory. */ - found_encoding = PyTokenizer_FindEncodingFilename(fd, pathobj); - lseek(fd, 0, 0); /* Reset position */ - if (found_encoding == NULL && PyErr_Occurred()) { - Py_XDECREF(pathobj); - close(fd); - return NULL; - } - encoding = (found_encoding != NULL) ? found_encoding : - (char*)PyUnicode_GetDefaultEncoding(); - } - fob = PyFile_FromFd(fd, NULL, fdp->mode, -1, - (char*)encoding, NULL, NULL, 1); - if (fob == NULL) { - Py_XDECREF(pathobj); - close(fd); - PyMem_FREE(found_encoding); - return NULL; - } - } - else { - fob = Py_None; - Py_INCREF(fob); - } - if (pathobj == NULL) { - Py_INCREF(Py_None); - pathobj = Py_None; - } - ret = Py_BuildValue("NN(ssi)", - fob, pathobj, fdp->suffix, fdp->mode, fdp->type); - PyMem_FREE(found_encoding); - - return ret; -} - -static PyObject * -imp_find_module(PyObject *self, PyObject *args) -{ - PyObject *name, *path_list = NULL; - if (!PyArg_ParseTuple(args, "U|O:find_module", - &name, &path_list)) - return NULL; - return call_find_module(name, path_list); -} - static PyObject * imp_init_builtin(PyObject *self, PyObject *args) { @@ -2931,13 +2188,6 @@ Reload the module. The module must have been successfully imported before."); PyDoc_STRVAR(doc_imp, "(Extremely) low-level import machinery bits as used by importlib and imp."); -PyDoc_STRVAR(doc_find_module, -"find_module(name, [path]) -> (file, filename, (suffix, mode, type))\n\ -Search for a module. If path is omitted or None, search for a\n\ -built-in, frozen or special module and continue search in sys.path.\n\ -The module name cannot contain '.'; to search for a submodule of a\n\ -package, pass the submodule name and the package's __path__."); - PyDoc_STRVAR(doc_get_magic, "get_magic() -> string\n\ Return the magic number for .pyc or .pyo files."); @@ -2969,7 +2219,6 @@ Release the interpreter's import lock.\n\ On platforms without threads, this function does nothing."); static PyMethodDef imp_methods[] = { - {"find_module", imp_find_module, METH_VARARGS, doc_find_module}, {"get_magic", imp_get_magic, METH_NOARGS, doc_get_magic}, {"get_tag", imp_get_tag, METH_NOARGS, doc_get_tag}, {"get_suffixes", imp_get_suffixes, METH_NOARGS, doc_get_suffixes}, -- cgit v1.2.1