From b5162583f02ef19edb3b4804cac793f1183c6283 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Feb 2011 23:12:28 +0000 Subject: Issue #3080: Mark PyWin_FindRegisteredModule() as private This function was not declared in Python public API (in any .h file) and not documented. Mark it as private to prepare a change of its API. --- Python/import.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 23752eeb72..f4015b52fa 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1547,8 +1547,8 @@ PyImport_GetImporter(PyObject *path) { pathname and an open file. Return NULL if the module is not found. */ #ifdef MS_COREDLL -extern FILE *PyWin_FindRegisteredModule(const char *, struct filedescr **, - char *, Py_ssize_t); +extern FILE *_PyWin_FindRegisteredModule(const char *, struct filedescr **, + char *, Py_ssize_t); #endif static int case_ok(char *, Py_ssize_t, Py_ssize_t, char *); @@ -1631,7 +1631,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, return &fd_builtin; } #ifdef MS_COREDLL - fp = PyWin_FindRegisteredModule(name, &fdp, buf, buflen); + fp = _PyWin_FindRegisteredModule(name, &fdp, buf, buflen); if (fp != NULL) { *p_fp = fp; return fdp; -- cgit v1.2.1 From 90268ce7e125f2d8a1d9b8f41b51d31f96b1c8fd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Feb 2011 00:02:00 +0000 Subject: Issue #3080: Mark _PyImport_FindBuiltin() argument as constant And as a consequence, mark also name argument of _PyImport_FindExtensionUnicode() constant too. But I plan to change this argument type to PyObject* later. --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index f4015b52fa..39cd93f7dd 100644 --- a/Python/import.c +++ b/Python/import.c @@ -636,7 +636,7 @@ _PyImport_FixupBuiltin(PyObject *mod, char *name) } PyObject * -_PyImport_FindExtensionUnicode(char *name, PyObject *filename) +_PyImport_FindExtensionUnicode(const char *name, PyObject *filename) { PyObject *mod, *mdict; PyModuleDef* def; @@ -680,7 +680,7 @@ _PyImport_FindExtensionUnicode(char *name, PyObject *filename) } PyObject * -_PyImport_FindBuiltin(char *name) +_PyImport_FindBuiltin(const char *name) { PyObject *res, *filename; filename = PyUnicode_FromString(name); -- cgit v1.2.1 From b05cc9bc8518f72fcb3e33b764f24dae32d5524f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 4 Mar 2011 12:57:09 +0000 Subject: Issue #3080: Add PyImport_AddModuleObject() and PyImport_ExecCodeModuleObject() --- Python/import.c | 87 +++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 66 insertions(+), 21 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 39cd93f7dd..22a7c8744d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -698,18 +698,18 @@ _PyImport_FindBuiltin(const char *name) 'NEW' REFERENCE! */ PyObject * -PyImport_AddModule(const char *name) +PyImport_AddModuleObject(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); PyObject *m; - if ((m = PyDict_GetItemString(modules, name)) != NULL && + if ((m = PyDict_GetItem(modules, name)) != NULL && PyModule_Check(m)) return m; - m = PyModule_New(name); + m = PyModule_NewObject(name); if (m == NULL) return NULL; - if (PyDict_SetItemString(modules, name, m) != 0) { + if (PyDict_SetItem(modules, name, m) != 0) { Py_DECREF(m); return NULL; } @@ -718,14 +718,27 @@ PyImport_AddModule(const char *name) return m; } +PyObject * +PyImport_AddModule(const char *name) +{ + PyObject *nameobj, *module; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + module = PyImport_AddModuleObject(nameobj); + Py_DECREF(nameobj); + return module; +} + + /* Remove name from sys.modules, if it's there. */ static void -remove_module(const char *name) +remove_module(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); - if (PyDict_GetItemString(modules, name) == NULL) + if (PyDict_GetItem(modules, name) == NULL) return; - if (PyDict_DelItemString(modules, name) < 0) + if (PyDict_DelItem(modules, name) < 0) Py_FatalError("import: deleting existing key in" "sys.modules failed"); } @@ -762,11 +775,43 @@ PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname) PyObject * PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, char *cpathname) +{ + PyObject *m = NULL; + PyObject *nameobj, *pathobj = NULL, *cpathobj = NULL; + + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + + if (pathname != NULL) { + pathobj = PyUnicode_DecodeFSDefault(pathname); + if (pathobj == NULL) + goto error; + } else + pathobj = NULL; + if (cpathname != NULL) { + cpathobj = PyUnicode_DecodeFSDefault(cpathname); + if (cpathobj == NULL) + goto error; + } else + cpathobj = NULL; + m = PyImport_ExecCodeModuleObject(nameobj, co, pathobj, cpathobj); +error: + Py_DECREF(nameobj); + Py_XDECREF(pathobj); + Py_XDECREF(cpathobj); + return m; +} + +PyObject* +PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, + PyObject *cpathname) { PyObject *modules = PyImport_GetModuleDict(); PyObject *m, *d, *v; + PyObject *pathbytes; - m = PyImport_AddModule(name); + m = PyImport_AddModuleObject(name); if (m == NULL) return NULL; /* If the module is being reloaded, we get the old module back @@ -778,12 +823,18 @@ PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, goto error; } /* Remember the filename as the __file__ attribute */ - v = NULL; if (pathname != NULL) { - v = get_sourcefile(pathname); + pathbytes = PyUnicode_EncodeFSDefault(pathname); + if (pathbytes != NULL) { + v = get_sourcefile(PyBytes_AS_STRING(pathbytes)); + Py_DECREF(pathbytes); + } else + v = NULL; if (v == NULL) PyErr_Clear(); } + else + v = NULL; if (v == NULL) { v = ((PyCodeObject *)co)->co_filename; Py_INCREF(v); @@ -793,27 +844,21 @@ PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, Py_DECREF(v); /* Remember the pyc path name as the __cached__ attribute. */ - if (cpathname == NULL) { - v = Py_None; - Py_INCREF(v); - } - else if ((v = PyUnicode_FromString(cpathname)) == NULL) { - PyErr_Clear(); /* Not important enough to report */ + if (cpathname != NULL) + v = cpathname; + else v = Py_None; - Py_INCREF(v); - } if (PyDict_SetItemString(d, "__cached__", v) != 0) PyErr_Clear(); /* Not important enough to report */ - Py_DECREF(v); v = PyEval_EvalCode(co, d, d); if (v == NULL) goto error; Py_DECREF(v); - if ((m = PyDict_GetItemString(modules, name)) == NULL) { + if ((m = PyDict_GetItem(modules, name)) == NULL) { PyErr_Format(PyExc_ImportError, - "Loaded module %.200s not found in sys.modules", + "Loaded module %R not found in sys.modules", name); return NULL; } -- cgit v1.2.1 From 426e754b8e8cf35a60153914ae341ca57e9d5031 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 7 Mar 2011 16:57:48 +0100 Subject: import.c: replace tab by spaces --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 22a7c8744d..b2757dffdc 100644 --- a/Python/import.c +++ b/Python/import.c @@ -329,7 +329,7 @@ _PyImport_ReInitLock(void) /* Forked as a side effect of import */ long me = PyThread_get_thread_ident(); PyThread_acquire_lock(import_lock, 0); - /* XXX: can the previous line fail? */ + /* XXX: can the previous line fail? */ import_lock_thread = me; import_lock_level--; } else { -- cgit v1.2.1 From 13abff792323d15e50b40c4ef978441790bef7b3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 7 Mar 2011 17:08:21 +0100 Subject: Issue #3080: PyImport_Cleanup() uses Unicode Replace strcmp() by PyUnicode_CompareWithASCIIString() --- Python/import.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index b2757dffdc..9366be0589 100644 --- a/Python/import.c +++ b/Python/import.c @@ -418,7 +418,6 @@ void PyImport_Cleanup(void) { Py_ssize_t pos, ndone; - char *name; PyObject *key, *value, *dict; PyInterpreterState *interp = PyThreadState_GET()->interp; PyObject *modules = interp->modules; @@ -491,14 +490,13 @@ PyImport_Cleanup(void) if (value->ob_refcnt != 1) continue; if (PyUnicode_Check(key) && PyModule_Check(value)) { - name = _PyUnicode_AsString(key); - if (strcmp(name, "builtins") == 0) + if (PyUnicode_CompareWithASCIIString(key, "builtins") == 0) continue; - if (strcmp(name, "sys") == 0) + if (PyUnicode_CompareWithASCIIString(key, "sys") == 0) continue; if (Py_VerboseFlag) - PySys_WriteStderr( - "# cleanup[1] %s\n", name); + PySys_FormatStderr( + "# cleanup[1] %U\n", key); _PyModule_Clear(value); PyDict_SetItem(modules, key, Py_None); ndone++; @@ -510,13 +508,12 @@ PyImport_Cleanup(void) pos = 0; while (PyDict_Next(modules, &pos, &key, &value)) { if (PyUnicode_Check(key) && PyModule_Check(value)) { - name = _PyUnicode_AsString(key); - if (strcmp(name, "builtins") == 0) + if (PyUnicode_CompareWithASCIIString(key, "builtins") == 0) continue; - if (strcmp(name, "sys") == 0) + if (PyUnicode_CompareWithASCIIString(key, "sys") == 0) continue; if (Py_VerboseFlag) - PySys_WriteStderr("# cleanup[2] %s\n", name); + PySys_FormatStderr("# cleanup[2] %U\n", key); _PyModule_Clear(value); PyDict_SetItem(modules, key, Py_None); } -- cgit v1.2.1 From 46cbf15e71a6cd5841e2547357932aa1f8c93936 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 20 Mar 2011 01:50:21 +0100 Subject: Issue #3080: Add PyImport_ImportFrozenModuleObject() find_frozen(), get_frozen_object(), is_frozen_package() and other functions related to frozen modules use Unicode strings instead of byte strings. --- Python/import.c | 98 +++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 61 insertions(+), 37 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 9366be0589..09fc52cbda 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1429,7 +1429,7 @@ get_sourcefile(char *file) static PyObject *load_module(char *, FILE *, char *, int, PyObject *); static struct filedescr *find_module(char *, char *, PyObject *, char *, size_t, FILE **, PyObject **); -static struct _frozen * find_frozen(char *); +static struct _frozen * find_frozen(PyObject *); /* Load a package and return its module object WITH INCREMENTED REFERENCE COUNT */ @@ -1617,6 +1617,8 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, size_t saved_namelen; char *saved_buf = NULL; #endif + PyObject *fullname_obj; + if (p_loader != NULL) *p_loader = NULL; @@ -1662,9 +1664,16 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, Py_DECREF(meta_path); } - if (find_frozen(fullname) != NULL) { - strcpy(buf, fullname); - return &fd_frozen; + if (fullname != NULL) { + fullname_obj = PyUnicode_FromString(fullname); + if (fullname == NULL) + return NULL; + if (find_frozen(fullname_obj) != NULL) { + Py_DECREF(fullname_obj); + strcpy(buf, fullname); + return &fd_frozen; + } + Py_DECREF(fullname_obj); } if (path == NULL) { @@ -2226,37 +2235,37 @@ init_builtin(char *name) /* Frozen modules */ static struct _frozen * -find_frozen(char *name) +find_frozen(PyObject *name) { struct _frozen *p; - if (!name) + if (name == NULL) return NULL; for (p = PyImport_FrozenModules; ; p++) { if (p->name == NULL) return NULL; - if (strcmp(p->name, name) == 0) + if (PyUnicode_CompareWithASCIIString(name, p->name) == 0) break; } return p; } static PyObject * -get_frozen_object(char *name) +get_frozen_object(PyObject *name) { struct _frozen *p = find_frozen(name); int size; if (p == NULL) { PyErr_Format(PyExc_ImportError, - "No such frozen object named %.200s", + "No such frozen object named %R", name); return NULL; } if (p->code == NULL) { PyErr_Format(PyExc_ImportError, - "Excluded frozen object named %.200s", + "Excluded frozen object named %R", name); return NULL; } @@ -2267,14 +2276,14 @@ get_frozen_object(char *name) } static PyObject * -is_frozen_package(char *name) +is_frozen_package(PyObject *name) { struct _frozen *p = find_frozen(name); int size; if (p == NULL) { PyErr_Format(PyExc_ImportError, - "No such frozen object named %.200s", + "No such frozen object named %R", name); return NULL; } @@ -2294,19 +2303,20 @@ is_frozen_package(char *name) This function is also used from frozenmain.c */ int -PyImport_ImportFrozenModule(char *name) +PyImport_ImportFrozenModuleObject(PyObject *name) { - struct _frozen *p = find_frozen(name); - PyObject *co; - PyObject *m; + struct _frozen *p; + PyObject *co, *m, *path; int ispackage; int size; + p = find_frozen(name); + if (p == NULL) return 0; if (p->code == NULL) { PyErr_Format(PyExc_ImportError, - "Excluded frozen object named %.200s", + "Excluded frozen object named %R", name); return -1; } @@ -2315,40 +2325,41 @@ PyImport_ImportFrozenModule(char *name) if (ispackage) size = -size; if (Py_VerboseFlag) - PySys_WriteStderr("import %s # frozen%s\n", + PySys_FormatStderr("import %U # frozen%s\n", name, ispackage ? " package" : ""); co = PyMarshal_ReadObjectFromString((char *)p->code, size); if (co == NULL) return -1; if (!PyCode_Check(co)) { PyErr_Format(PyExc_TypeError, - "frozen object %.200s is not a code object", + "frozen object %R is not a code object", name); goto err_return; } if (ispackage) { /* Set __path__ to the package name */ - PyObject *d, *s, *l; + PyObject *d, *l; int err; - m = PyImport_AddModule(name); + m = PyImport_AddModuleObject(name); if (m == NULL) goto err_return; d = PyModule_GetDict(m); - s = PyUnicode_InternFromString(name); - if (s == NULL) - goto err_return; l = PyList_New(1); if (l == NULL) { - Py_DECREF(s); goto err_return; } - PyList_SET_ITEM(l, 0, s); + Py_INCREF(name); + PyList_SET_ITEM(l, 0, name); err = PyDict_SetItemString(d, "__path__", l); Py_DECREF(l); if (err != 0) goto err_return; } - m = PyImport_ExecCodeModuleEx(name, co, ""); + path = PyUnicode_FromString(""); + if (path == NULL) + goto err_return; + m = PyImport_ExecCodeModuleObject(name, co, path, NULL); + Py_DECREF(path); if (m == NULL) goto err_return; Py_DECREF(co); @@ -2359,6 +2370,19 @@ err_return: return -1; } +int +PyImport_ImportFrozenModule(char *name) +{ + PyObject *nameobj; + int ret; + nameobj = PyUnicode_InternFromString(name); + if (nameobj == NULL) + return -1; + ret = PyImport_ImportFrozenModuleObject(nameobj); + Py_DECREF(nameobj); + return ret; +} + /* Import a module, either built-in, frozen, or external, and return its module object WITH INCREMENTED REFERENCE COUNT */ @@ -3282,19 +3306,19 @@ imp_init_builtin(PyObject *self, PyObject *args) static PyObject * imp_init_frozen(PyObject *self, PyObject *args) { - char *name; + PyObject *name; int ret; PyObject *m; - if (!PyArg_ParseTuple(args, "s:init_frozen", &name)) + if (!PyArg_ParseTuple(args, "U:init_frozen", &name)) return NULL; - ret = PyImport_ImportFrozenModule(name); + ret = PyImport_ImportFrozenModuleObject(name); if (ret < 0) return NULL; if (ret == 0) { Py_INCREF(Py_None); return Py_None; } - m = PyImport_AddModule(name); + m = PyImport_AddModuleObject(name); Py_XINCREF(m); return m; } @@ -3302,9 +3326,9 @@ imp_init_frozen(PyObject *self, PyObject *args) static PyObject * imp_get_frozen_object(PyObject *self, PyObject *args) { - char *name; + PyObject *name; - if (!PyArg_ParseTuple(args, "s:get_frozen_object", &name)) + if (!PyArg_ParseTuple(args, "U:get_frozen_object", &name)) return NULL; return get_frozen_object(name); } @@ -3312,9 +3336,9 @@ imp_get_frozen_object(PyObject *self, PyObject *args) static PyObject * imp_is_frozen_package(PyObject *self, PyObject *args) { - char *name; + PyObject *name; - if (!PyArg_ParseTuple(args, "s:is_frozen_package", &name)) + if (!PyArg_ParseTuple(args, "U:is_frozen_package", &name)) return NULL; return is_frozen_package(name); } @@ -3331,9 +3355,9 @@ imp_is_builtin(PyObject *self, PyObject *args) static PyObject * imp_is_frozen(PyObject *self, PyObject *args) { - char *name; + PyObject *name; struct _frozen *p; - if (!PyArg_ParseTuple(args, "s:is_frozen", &name)) + if (!PyArg_ParseTuple(args, "U:is_frozen", &name)) return NULL; p = find_frozen(name); return PyBool_FromLong((long) (p == NULL ? 0 : p->size)); -- cgit v1.2.1 From 17a3c1b5c7bb9b20e89c29ee9fcf6ee78a6cce82 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 7 Mar 2011 18:20:56 +0100 Subject: Issue #3080: Import builtins using Unicode strings - is_builtin(), init_builtin(), load_builtin() and other builtin related functions use Unicode strings, instead of byte strings - Rename _PyImport_FixupExtensionUnicode() to _PyImport_FixupExtensionObject() - Rename _PyImport_FindExtensionUnicode() to _PyImport_FindExtensionObject() --- Python/import.c | 112 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 60 insertions(+), 52 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 09fc52cbda..597ef439c3 100644 --- a/Python/import.c +++ b/Python/import.c @@ -117,7 +117,7 @@ typedef unsigned short mode_t; static long pyc_magic = MAGIC; static const char *pyc_tag = TAG; -/* See _PyImport_FixupExtensionUnicode() below */ +/* See _PyImport_FixupExtensionObject() below */ static PyObject *extensions = NULL; /* This table is defined in config.c: */ @@ -563,10 +563,10 @@ PyImport_GetMagicTag(void) once, we keep a static dictionary 'extensions' keyed by module name (for built-in modules) or by filename (for dynamically loaded modules), containing these modules. A copy of the module's - dictionary is stored by calling _PyImport_FixupExtensionUnicode() + dictionary is stored by calling _PyImport_FixupExtensionObject() immediately after the module initialization function succeeds. A copy can be retrieved from there by calling - _PyImport_FindExtensionUnicode(). + _PyImport_FindExtensionObject(). Modules which do support multiple initialization set their m_size field to a non-negative number (indicating the size of the @@ -575,7 +575,8 @@ PyImport_GetMagicTag(void) */ int -_PyImport_FixupExtensionUnicode(PyObject *mod, char *name, PyObject *filename) +_PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, + PyObject *filename) { PyObject *modules, *dict; struct PyModuleDef *def; @@ -594,10 +595,10 @@ _PyImport_FixupExtensionUnicode(PyObject *mod, char *name, PyObject *filename) return -1; } modules = PyImport_GetModuleDict(); - if (PyDict_SetItemString(modules, name, mod) < 0) + if (PyDict_SetItem(modules, name, mod) < 0) return -1; if (_PyState_AddModule(mod, def) < 0) { - PyDict_DelItemString(modules, name); + PyDict_DelItem(modules, name); return -1; } if (def->m_size == -1) { @@ -623,17 +624,17 @@ int _PyImport_FixupBuiltin(PyObject *mod, char *name) { int res; - PyObject *filename; - filename = PyUnicode_FromString(name); - if (filename == NULL) + PyObject *nameobj; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) return -1; - res = _PyImport_FixupExtensionUnicode(mod, name, filename); - Py_DECREF(filename); + res = _PyImport_FixupExtensionObject(mod, nameobj, nameobj); + Py_DECREF(nameobj); return res; } PyObject * -_PyImport_FindExtensionUnicode(const char *name, PyObject *filename) +_PyImport_FindExtensionObject(PyObject *name, PyObject *filename) { PyObject *mod, *mdict; PyModuleDef* def; @@ -646,7 +647,7 @@ _PyImport_FindExtensionUnicode(const char *name, PyObject *filename) /* Module does not support repeated initialization */ if (def->m_base.m_copy == NULL) return NULL; - mod = PyImport_AddModule(name); + mod = PyImport_AddModuleObject(name); if (mod == NULL) return NULL; mdict = PyModule_GetDict(mod); @@ -661,16 +662,16 @@ _PyImport_FindExtensionUnicode(const char *name, PyObject *filename) mod = def->m_base.m_init(); if (mod == NULL) return NULL; - PyDict_SetItemString(PyImport_GetModuleDict(), name, mod); + PyDict_SetItem(PyImport_GetModuleDict(), name, mod); Py_DECREF(mod); } if (_PyState_AddModule(mod, def) < 0) { - PyDict_DelItemString(PyImport_GetModuleDict(), name); + PyDict_DelItem(PyImport_GetModuleDict(), name); Py_DECREF(mod); return NULL; } if (Py_VerboseFlag) - PySys_FormatStderr("import %s # previously loaded (%U)\n", + PySys_FormatStderr("import %U # previously loaded (%R)\n", name, filename); return mod; @@ -679,12 +680,12 @@ _PyImport_FindExtensionUnicode(const char *name, PyObject *filename) PyObject * _PyImport_FindBuiltin(const char *name) { - PyObject *res, *filename; - filename = PyUnicode_FromString(name); - if (filename == NULL) + PyObject *res, *nameobj; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) return NULL; - res = _PyImport_FindExtensionUnicode(name, filename); - Py_DECREF(filename); + res = _PyImport_FindExtensionObject(nameobj, nameobj); + Py_DECREF(nameobj); return res; } @@ -1491,11 +1492,12 @@ load_package(char *name, char *pathname) /* Helper to test for built-in module */ static int -is_builtin(char *name) +is_builtin(PyObject *name) { - int i; + int i, cmp; for (i = 0; PyImport_Inittab[i].name != NULL; i++) { - if (strcmp(name, PyImport_Inittab[i].name) == 0) { + cmp = PyUnicode_CompareWithASCIIString(name, PyImport_Inittab[i].name); + if (cmp == 0) { if (PyImport_Inittab[i].initfunc == NULL) return -1; else @@ -1617,7 +1619,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, size_t saved_namelen; char *saved_buf = NULL; #endif - PyObject *fullname_obj; + PyObject *fullname_obj, *nameobj; if (p_loader != NULL) *p_loader = NULL; @@ -1677,10 +1679,15 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, } if (path == NULL) { - if (is_builtin(name)) { + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + if (is_builtin(nameobj)) { + Py_DECREF(nameobj); strcpy(buf, name); return &fd_builtin; } + Py_DECREF(nameobj); #ifdef MS_COREDLL fp = _PyWin_FindRegisteredModule(name, &fdp, buf, buflen); if (fp != NULL) { @@ -2086,40 +2093,35 @@ find_init_module(char *buf) #endif /* HAVE_STAT */ -static int init_builtin(char *); /* Forward */ +static int init_builtin(PyObject *); /* Forward */ static PyObject* -load_builtin(char *name, char *pathname, int type) +load_builtin(PyObject *name, int type) { PyObject *m, *modules; int err; - if (pathname != NULL && pathname[0] != '\0') - name = pathname; - if (type == C_BUILTIN) err = init_builtin(name); else - err = PyImport_ImportFrozenModule(name); + err = PyImport_ImportFrozenModuleObject(name); if (err < 0) return NULL; if (err == 0) { PyErr_Format(PyExc_ImportError, - "Purported %s module %.200s not found", - type == C_BUILTIN ? - "builtin" : "frozen", + "Purported %s module %R not found", + type == C_BUILTIN ? "builtin" : "frozen", name); return NULL; } modules = PyImport_GetModuleDict(); - m = PyDict_GetItemString(modules, name); + m = PyDict_GetItem(modules, name); if (m == NULL) { PyErr_Format( PyExc_ImportError, - "%s module %.200s not properly initialized", - type == C_BUILTIN ? - "builtin" : "frozen", + "%s module %R not properly initialized", + type == C_BUILTIN ? "builtin" : "frozen", name); return NULL; } @@ -2168,9 +2170,15 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) break; case C_BUILTIN: - case PY_FROZEN: - m = load_builtin(name, pathname, type); + case PY_FROZEN: { + PyObject *nameobj = PyUnicode_FromString(name); + if (nameobj != NULL) { + m = load_builtin(nameobj, type); + Py_DECREF(nameobj); + } else + m = NULL; break; + } case IMP_HOOK: { if (loader == NULL) { @@ -2199,28 +2207,28 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) an exception set if the initialization failed. */ static int -init_builtin(char *name) +init_builtin(PyObject *name) { struct _inittab *p; - if (_PyImport_FindBuiltin(name) != NULL) + if (_PyImport_FindExtensionObject(name, name) != NULL) return 1; for (p = PyImport_Inittab; p->name != NULL; p++) { PyObject *mod; - if (strcmp(name, p->name) == 0) { + if (PyUnicode_CompareWithASCIIString(name, p->name) == 0) { if (p->initfunc == NULL) { PyErr_Format(PyExc_ImportError, - "Cannot re-init internal module %.200s", + "Cannot re-init internal module %R", name); return -1; } if (Py_VerboseFlag) - PySys_WriteStderr("import %s # builtin\n", name); + PySys_FormatStderr("import %U # builtin\n", name); mod = (*p->initfunc)(); if (mod == 0) return -1; - if (_PyImport_FixupBuiltin(mod, name) < 0) + if (_PyImport_FixupExtensionObject(mod, name, name) < 0) return -1; /* FixupExtension has put the module into sys.modules, so we can release our own reference. */ @@ -3286,10 +3294,10 @@ imp_find_module(PyObject *self, PyObject *args) static PyObject * imp_init_builtin(PyObject *self, PyObject *args) { - char *name; + PyObject *name; int ret; PyObject *m; - if (!PyArg_ParseTuple(args, "s:init_builtin", &name)) + if (!PyArg_ParseTuple(args, "U:init_builtin", &name)) return NULL; ret = init_builtin(name); if (ret < 0) @@ -3298,7 +3306,7 @@ imp_init_builtin(PyObject *self, PyObject *args) Py_INCREF(Py_None); return Py_None; } - m = PyImport_AddModule(name); + m = PyImport_AddModuleObject(name); Py_XINCREF(m); return m; } @@ -3346,8 +3354,8 @@ imp_is_frozen_package(PyObject *self, PyObject *args) static PyObject * imp_is_builtin(PyObject *self, PyObject *args) { - char *name; - if (!PyArg_ParseTuple(args, "s:is_builtin", &name)) + PyObject *name; + if (!PyArg_ParseTuple(args, "U:is_builtin", &name)) return NULL; return PyLong_FromLong(is_builtin(name)); } -- cgit v1.2.1 From d08ceb04095b21f699c9221e7e5d7bb2b86a20ea Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 7 Mar 2011 18:28:15 +0100 Subject: Issue #3080: Use PyUnicode_InternFromString() for builtins _PyImport_FixupBuiltin() and _PyImport_FindBuiltin() use PyUnicode_InternFromString() instead of PyUnicode_FromString(). --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 597ef439c3..d43b881c2f 100644 --- a/Python/import.c +++ b/Python/import.c @@ -625,7 +625,7 @@ _PyImport_FixupBuiltin(PyObject *mod, char *name) { int res; PyObject *nameobj; - nameobj = PyUnicode_FromString(name); + nameobj = PyUnicode_InternFromString(name); if (nameobj == NULL) return -1; res = _PyImport_FixupExtensionObject(mod, nameobj, nameobj); @@ -681,7 +681,7 @@ PyObject * _PyImport_FindBuiltin(const char *name) { PyObject *res, *nameobj; - nameobj = PyUnicode_FromString(name); + nameobj = PyUnicode_InternFromString(name); if (nameobj == NULL) return NULL; res = _PyImport_FindExtensionObject(nameobj, nameobj); -- cgit v1.2.1 From 4691aca1214aaa1b4433a667d875415f9a27c38e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 8 Mar 2011 23:49:04 +0100 Subject: Issue #3080: _PyWin_FindRegisteredModule() returns the path as Unicode * Document the function * Use RegQueryValueW() instead of RegQueryValueA() * Use _Py_fopen() instead of fopen() * Allocate registry key on the heap, not on the stack, and handle memory allocation failure * Handle Python exception in find_module() --- Python/import.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index d43b881c2f..8fab810eec 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1591,8 +1591,8 @@ PyImport_GetImporter(PyObject *path) { pathname and an open file. Return NULL if the module is not found. */ #ifdef MS_COREDLL -extern FILE *_PyWin_FindRegisteredModule(const char *, struct filedescr **, - char *, Py_ssize_t); +extern FILE *_PyWin_FindRegisteredModule(PyObject *, struct filedescr **, + PyObject **p_path); #endif static int case_ok(char *, Py_ssize_t, Py_ssize_t, char *); @@ -1679,6 +1679,9 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, } if (path == NULL) { +#ifdef MS_COREDLL + PyObject *filename, *filename_bytes; +#endif nameobj = PyUnicode_FromString(name); if (nameobj == NULL) return NULL; @@ -1687,14 +1690,24 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, strcpy(buf, name); return &fd_builtin; } - Py_DECREF(nameobj); #ifdef MS_COREDLL - fp = _PyWin_FindRegisteredModule(name, &fdp, buf, buflen); + fp = _PyWin_FindRegisteredModule(nameobj, &fdp, &filename); if (fp != NULL) { + Py_DECREF(nameobj); + filename_bytes = PyUnicode_EncodeFSDefault(filename); + Py_DECREF(filename); + if (filename_bytes == NULL) + return NULL; + strncpy(buf, PyBytes_AS_STRING(filename_bytes), buflen); + buf[buflen-1] = '\0'; + Py_DECREF(filename_bytes); *p_fp = fp; return fdp; } + else if (PyErr_Occurred()) + return NULL; #endif + Py_DECREF(nameobj); path = PySys_GetObject("path"); } -- cgit v1.2.1 From 7c1cd6d18d5ac37ec86183e7fabcf35ff11e3d21 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 15:54:07 -0400 Subject: Issue #3080: _PyImport_LoadDynamicModule() uses Unicode for name and path Document also that dynamic module names are ASCII only --- Python/import.c | 54 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 24 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 8fab810eec..4e5cb2ddbf 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2173,9 +2173,21 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) break; #ifdef HAVE_DYNAMIC_LOADING - case C_EXTENSION: - m = _PyImport_LoadDynamicModule(name, pathname, fp); + case C_EXTENSION: { + PyObject *nameobj, *pathobj; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + pathobj = PyUnicode_DecodeFSDefault(pathname); + if (pathobj == NULL) { + Py_DECREF(nameobj); + return NULL; + } + m = _PyImport_LoadDynamicModule(nameobj, pathobj, fp); + Py_DECREF(nameobj); + Py_DECREF(pathobj); break; + } #endif case PKG_DIRECTORY: @@ -2185,11 +2197,10 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) case C_BUILTIN: case PY_FROZEN: { PyObject *nameobj = PyUnicode_FromString(name); - if (nameobj != NULL) { - m = load_builtin(nameobj, type); - Py_DECREF(nameobj); - } else - m = NULL; + if (nameobj == NULL) + return NULL; + m = load_builtin(nameobj, type); + Py_DECREF(nameobj); break; } @@ -3443,28 +3454,23 @@ imp_load_compiled(PyObject *self, PyObject *args) static PyObject * imp_load_dynamic(PyObject *self, PyObject *args) { - char *name; - PyObject *pathbytes; - char *pathname; - PyObject *fob = NULL; - PyObject *m; - FILE *fp = NULL; - if (!PyArg_ParseTuple(args, "sO&|O:load_dynamic", - &name, PyUnicode_FSConverter, &pathbytes, &fob)) + PyObject *name, *pathname, *fob = NULL, *mod; + FILE *fp; + + if (!PyArg_ParseTuple(args, "UO&|O:load_dynamic", + &name, PyUnicode_FSDecoder, &pathname, &fob)) return NULL; - pathname = PyBytes_AS_STRING(pathbytes); - if (fob) { - fp = get_file(pathname, fob, "r"); - if (fp == NULL) { - Py_DECREF(pathbytes); + if (fob != NULL) { + fp = get_file(NULL, fob, "r"); + if (fp == NULL) return NULL; - } } - m = _PyImport_LoadDynamicModule(name, pathname, fp); - Py_DECREF(pathbytes); + else + fp = NULL; + mod = _PyImport_LoadDynamicModule(name, pathname, fp); if (fp) fclose(fp); - return m; + return mod; } #endif /* HAVE_DYNAMIC_LOADING */ -- cgit v1.2.1 From ab8eed250aee0dfe2e0e8cc1472113d5d2b4b9c7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 12 Mar 2011 08:45:02 -0500 Subject: Issue #3080: find_module() initialize buf and *p_fp Document also the find_module() function --- Python/import.c | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 4e5cb2ddbf..e58d6a04be 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1443,7 +1443,7 @@ load_package(char *name, char *pathname) PyObject *path = NULL; int err; char buf[MAXPATHLEN+1]; - FILE *fp = NULL; + FILE *fp; struct filedescr *fdp; m = PyImport_AddModule(name); @@ -1464,7 +1464,6 @@ load_package(char *name, char *pathname) err = PyDict_SetItemString(d, "__path__", path); if (err != 0) goto error; - buf[0] = '\0'; fdp = find_module(name, "__init__", path, buf, sizeof(buf), &fp, NULL); if (fdp == NULL) { if (PyErr_ExceptionMatches(PyExc_ImportError)) { @@ -1599,6 +1598,31 @@ static int case_ok(char *, Py_ssize_t, Py_ssize_t, char *); static int find_init_module(char *); /* Forward */ static struct filedescr importhookdescr = {"", "", IMP_HOOK}; +/* Find a module: + + - try find_module() of each sys.meta_path hook + - try find_frozen() + - try is_builtin() + - try _PyWin_FindRegisteredModule() (Windows only) + - otherwise, call find_module_path_list() with search_path_list (if not + NULL) or sys.path + + Return: + + - &fd_builtin (C_BUILTIN) if it is a builtin + - &fd_frozen (PY_FROZEN) if it is frozen + - &fd_package (PKG_DIRECTORY) and write the filename into *buf + if it is a package + - &importhookdescr (IMP_HOOK) and write the loader into *p_loader if a + importer loader was found + - a file descriptor (PY_SOURCE, PY_COMPILED, C_EXTENSION, PY_RESOURCE or + PY_CODERESOURCE: see _PyImport_Filetab), write the filename into + *buf and the pointer to the open file into *p_fp + - NULL on error + + By default, write an empty string into *buf, and *p_fp and *p_loader (if + set) are set to NULL. Eg. *buf is an empty string for a builtin package. */ + static struct filedescr * find_module(char *fullname, char *subname, PyObject *path, char *buf, size_t buflen, FILE **p_fp, PyObject **p_loader) @@ -1621,6 +1645,8 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, #endif PyObject *fullname_obj, *nameobj; + *buf = '\0'; + *p_fp = NULL; if (p_loader != NULL) *p_loader = NULL; @@ -2962,10 +2988,10 @@ import_submodule(PyObject *mod, char *subname, char *fullname) Py_INCREF(m); } else { - PyObject *path, *loader = NULL; + PyObject *path, *loader; char buf[MAXPATHLEN+1]; struct filedescr *fdp; - FILE *fp = NULL; + FILE *fp; if (mod == Py_None) path = NULL; @@ -2978,7 +3004,6 @@ import_submodule(PyObject *mod, char *subname, char *fullname) } } - buf[0] = '\0'; fdp = find_module(fullname, subname, path, buf, MAXPATHLEN+1, &fp, &loader); Py_XDECREF(path); @@ -3012,11 +3037,11 @@ PyImport_ReloadModule(PyObject *m) PyInterpreterState *interp = PyThreadState_Get()->interp; PyObject *modules_reloading = interp->modules_reloading; PyObject *modules = PyImport_GetModuleDict(); - PyObject *path = NULL, *loader = NULL, *existing_m = NULL; + PyObject *path = NULL, *loader, *existing_m = NULL; char *name, *subname; char buf[MAXPATHLEN+1]; struct filedescr *fdp; - FILE *fp = NULL; + FILE *fp; PyObject *newm; if (modules_reloading == NULL) { @@ -3074,7 +3099,6 @@ PyImport_ReloadModule(PyObject *m) if (path == NULL) PyErr_Clear(); } - buf[0] = '\0'; fdp = find_module(name, subname, path, buf, MAXPATHLEN+1, &fp, &loader); Py_XDECREF(path); @@ -3252,7 +3276,7 @@ call_find_module(char *name, PyObject *path) PyObject *pathobj; struct filedescr *fdp; char pathname[MAXPATHLEN+1]; - FILE *fp = NULL; + FILE *fp; int fd = -1; char *found_encoding = NULL; char *encoding = NULL; -- cgit v1.2.1 From 8d647fdca120414a1901c3b1414b405ad7f415f4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 12 Mar 2011 09:26:54 -0500 Subject: Issue #3080: Remove useless name buffer from find_module() Rename subname argument to name, and mark it as constant. --- Python/import.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index e58d6a04be..37cd4b81f2 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1428,7 +1428,7 @@ get_sourcefile(char *file) /* Forward */ static PyObject *load_module(char *, FILE *, char *, int, PyObject *); -static struct filedescr *find_module(char *, char *, PyObject *, +static struct filedescr *find_module(char *, const char *, PyObject *, char *, size_t, FILE **, PyObject **); static struct _frozen * find_frozen(PyObject *); @@ -1594,7 +1594,7 @@ extern FILE *_PyWin_FindRegisteredModule(PyObject *, struct filedescr **, PyObject **p_path); #endif -static int case_ok(char *, Py_ssize_t, Py_ssize_t, char *); +static int case_ok(char *, Py_ssize_t, Py_ssize_t, const char *); static int find_init_module(char *); /* Forward */ static struct filedescr importhookdescr = {"", "", IMP_HOOK}; @@ -1624,7 +1624,7 @@ static struct filedescr importhookdescr = {"", "", IMP_HOOK}; set) are set to NULL. Eg. *buf is an empty string for a builtin package. */ static struct filedescr * -find_module(char *fullname, char *subname, PyObject *path, char *buf, +find_module(char *fullname, const char *name, PyObject *path, char *buf, size_t buflen, FILE **p_fp, PyObject **p_loader) { Py_ssize_t i, npath; @@ -1637,7 +1637,6 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, static struct filedescr fd_frozen = {"", "", PY_FROZEN}; static struct filedescr fd_builtin = {"", "", C_BUILTIN}; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; - char name[MAXPATHLEN+1]; #if defined(PYOS_OS2) size_t saved_len; size_t saved_namelen; @@ -1650,12 +1649,11 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, if (p_loader != NULL) *p_loader = NULL; - if (strlen(subname) > MAXPATHLEN) { + if (strlen(name) > MAXPATHLEN) { PyErr_SetString(PyExc_OverflowError, "module name is too long"); return NULL; } - strcpy(name, subname); /* sys.meta_path import hook */ if (p_loader != NULL) { @@ -1867,7 +1865,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, * dynamically loaded module we're going to try, * truncate the name before trying */ - if (strlen(subname) > 8) { + if (strlen(name) > 8) { /* is this an attempt to load a C extension? */ const struct filedescr *scan; scan = _PyImport_DynLoadFiletab; @@ -1880,7 +1878,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, if (scan->suffix != NULL) { /* yes, so truncate the name */ namelen = 8; - len -= strlen(subname) - namelen; + len -= strlen(name) - namelen; buf[len] = '\0'; } } @@ -1972,7 +1970,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, #endif static int -case_ok(char *buf, Py_ssize_t len, Py_ssize_t namelen, char *name) +case_ok(char *buf, Py_ssize_t len, Py_ssize_t namelen, const char *name) { /* Pick a platform-specific implementation; the sequence of #if's here should * match the sequence just above. -- cgit v1.2.1 From f44ffd60a466519364cc45d367f3505f1ecbe090 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 20 Mar 2011 01:34:43 +0100 Subject: Issue #3080: Create find_module_path_list() subfunction --- Python/import.c | 310 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 164 insertions(+), 146 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 37cd4b81f2..565f1336cb 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1598,168 +1598,30 @@ static int case_ok(char *, Py_ssize_t, Py_ssize_t, const char *); static int find_init_module(char *); /* Forward */ static struct filedescr importhookdescr = {"", "", IMP_HOOK}; -/* Find a module: - - - try find_module() of each sys.meta_path hook - - try find_frozen() - - try is_builtin() - - try _PyWin_FindRegisteredModule() (Windows only) - - otherwise, call find_module_path_list() with search_path_list (if not - NULL) or sys.path - - Return: - - - &fd_builtin (C_BUILTIN) if it is a builtin - - &fd_frozen (PY_FROZEN) if it is frozen - - &fd_package (PKG_DIRECTORY) and write the filename into *buf - if it is a package - - &importhookdescr (IMP_HOOK) and write the loader into *p_loader if a - importer loader was found - - a file descriptor (PY_SOURCE, PY_COMPILED, C_EXTENSION, PY_RESOURCE or - PY_CODERESOURCE: see _PyImport_Filetab), write the filename into - *buf and the pointer to the open file into *p_fp - - NULL on error - - By default, write an empty string into *buf, and *p_fp and *p_loader (if - set) are set to NULL. Eg. *buf is an empty string for a builtin package. */ - -static struct filedescr * -find_module(char *fullname, const char *name, PyObject *path, char *buf, - size_t buflen, FILE **p_fp, PyObject **p_loader) +static struct filedescr* +find_module_path_list(char *fullname, const char *name, + PyObject *search_path_list, PyObject *path_hooks, + PyObject *path_importer_cache, + char *buf, size_t buflen, + FILE **p_fp, PyObject **p_loader) { Py_ssize_t i, npath; size_t len, namelen; struct filedescr *fdp = NULL; char *filemode; FILE *fp = NULL; - PyObject *path_hooks, *path_importer_cache; struct stat statbuf; - static struct filedescr fd_frozen = {"", "", PY_FROZEN}; - static struct filedescr fd_builtin = {"", "", C_BUILTIN}; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; #if defined(PYOS_OS2) size_t saved_len; size_t saved_namelen; char *saved_buf = NULL; #endif - PyObject *fullname_obj, *nameobj; - *buf = '\0'; - *p_fp = NULL; - if (p_loader != NULL) - *p_loader = NULL; - - if (strlen(name) > MAXPATHLEN) { - PyErr_SetString(PyExc_OverflowError, - "module name is too long"); - return NULL; - } - - /* sys.meta_path import hook */ - if (p_loader != NULL) { - PyObject *meta_path; - - meta_path = PySys_GetObject("meta_path"); - if (meta_path == NULL || !PyList_Check(meta_path)) { - PyErr_SetString(PyExc_ImportError, - "sys.meta_path must be a list of " - "import hooks"); - return NULL; - } - Py_INCREF(meta_path); /* zap guard */ - npath = PyList_Size(meta_path); - for (i = 0; i < npath; i++) { - PyObject *loader; - PyObject *hook = PyList_GetItem(meta_path, i); - loader = PyObject_CallMethod(hook, "find_module", - "sO", fullname, - path != NULL ? - path : Py_None); - if (loader == NULL) { - Py_DECREF(meta_path); - return NULL; /* true error */ - } - if (loader != Py_None) { - /* a loader was found */ - *p_loader = loader; - Py_DECREF(meta_path); - return &importhookdescr; - } - Py_DECREF(loader); - } - Py_DECREF(meta_path); - } - - if (fullname != NULL) { - fullname_obj = PyUnicode_FromString(fullname); - if (fullname == NULL) - return NULL; - if (find_frozen(fullname_obj) != NULL) { - Py_DECREF(fullname_obj); - strcpy(buf, fullname); - return &fd_frozen; - } - Py_DECREF(fullname_obj); - } - - if (path == NULL) { -#ifdef MS_COREDLL - PyObject *filename, *filename_bytes; -#endif - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - if (is_builtin(nameobj)) { - Py_DECREF(nameobj); - strcpy(buf, name); - return &fd_builtin; - } -#ifdef MS_COREDLL - fp = _PyWin_FindRegisteredModule(nameobj, &fdp, &filename); - if (fp != NULL) { - Py_DECREF(nameobj); - filename_bytes = PyUnicode_EncodeFSDefault(filename); - Py_DECREF(filename); - if (filename_bytes == NULL) - return NULL; - strncpy(buf, PyBytes_AS_STRING(filename_bytes), buflen); - buf[buflen-1] = '\0'; - Py_DECREF(filename_bytes); - *p_fp = fp; - return fdp; - } - else if (PyErr_Occurred()) - return NULL; -#endif - Py_DECREF(nameobj); - path = PySys_GetObject("path"); - } - - if (path == NULL || !PyList_Check(path)) { - PyErr_SetString(PyExc_ImportError, - "sys.path must be a list of directory names"); - return NULL; - } - - path_hooks = PySys_GetObject("path_hooks"); - if (path_hooks == NULL || !PyList_Check(path_hooks)) { - PyErr_SetString(PyExc_ImportError, - "sys.path_hooks must be a list of " - "import hooks"); - return NULL; - } - path_importer_cache = PySys_GetObject("path_importer_cache"); - if (path_importer_cache == NULL || - !PyDict_Check(path_importer_cache)) { - PyErr_SetString(PyExc_ImportError, - "sys.path_importer_cache must be a dict"); - return NULL; - } - - npath = PyList_Size(path); + npath = PyList_Size(search_path_list); namelen = strlen(name); for (i = 0; i < npath; i++) { - PyObject *v = PyList_GetItem(path, i); + PyObject *v = PyList_GetItem(search_path_list, i); PyObject *origv = v; const char *base; Py_ssize_t size; @@ -1925,6 +1787,162 @@ find_module(char *fullname, const char *name, PyObject *path, char *buf, return fdp; } +/* Find a module: + + - try find_module() of each sys.meta_path hook + - try find_frozen() + - try is_builtin() + - try _PyWin_FindRegisteredModule() (Windows only) + - otherwise, call find_module_path_list() with search_path_list (if not + NULL) or sys.path + + Return: + + - &fd_builtin (C_BUILTIN) if it is a builtin + - &fd_frozen (PY_FROZEN) if it is frozen + - &fd_package (PKG_DIRECTORY) and write the filename into *buf + if it is a package + - &importhookdescr (IMP_HOOK) and write the loader into *p_loader if a + importer loader was found + - a file descriptor (PY_SOURCE, PY_COMPILED, C_EXTENSION, PY_RESOURCE or + PY_CODERESOURCE: see _PyImport_Filetab), write the filename into + *buf and the pointer to the open file into *p_fp + - NULL on error + + By default, write an empty string into *buf, and *p_fp and *p_loader (if + set) are set to NULL. Eg. *buf is an empty string for a builtin package. */ + +static struct filedescr * +find_module(char *fullname, const char *name, PyObject *search_path_list, + char *buf, size_t buflen, FILE **p_fp, PyObject **p_loader) +{ + Py_ssize_t i, npath; + static struct filedescr fd_frozen = {"", "", PY_FROZEN}; + static struct filedescr fd_builtin = {"", "", C_BUILTIN}; + PyObject *path_hooks, *path_importer_cache; + PyObject *fullname_obj, *nameobj; + + *buf = '\0'; + *p_fp = NULL; + if (p_loader != NULL) + *p_loader = NULL; + + if (strlen(name) > MAXPATHLEN) { + PyErr_SetString(PyExc_OverflowError, + "module name is too long"); + return NULL; + } + + /* sys.meta_path import hook */ + if (p_loader != NULL) { + PyObject *meta_path; + + meta_path = PySys_GetObject("meta_path"); + if (meta_path == NULL || !PyList_Check(meta_path)) { + PyErr_SetString(PyExc_ImportError, + "sys.meta_path must be a list of " + "import hooks"); + return NULL; + } + Py_INCREF(meta_path); /* zap guard */ + npath = PyList_Size(meta_path); + for (i = 0; i < npath; i++) { + PyObject *loader; + PyObject *hook = PyList_GetItem(meta_path, i); + loader = PyObject_CallMethod(hook, "find_module", + "sO", fullname, + search_path_list != NULL ? + search_path_list : Py_None); + if (loader == NULL) { + Py_DECREF(meta_path); + return NULL; /* true error */ + } + if (loader != Py_None) { + /* a loader was found */ + *p_loader = loader; + Py_DECREF(meta_path); + return &importhookdescr; + } + Py_DECREF(loader); + } + Py_DECREF(meta_path); + } + + if (fullname != NULL) { + fullname_obj = PyUnicode_FromString(fullname); + if (fullname == NULL) + return NULL; + if (find_frozen(fullname_obj) != NULL) { + Py_DECREF(fullname_obj); + strcpy(buf, fullname); + return &fd_frozen; + } + Py_DECREF(fullname_obj); + } + + if (search_path_list == NULL) { +#ifdef MS_COREDLL + FILE *fp; + struct filedescr *fdp; + PyObject *filename, *filename_bytes; +#endif + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + if (is_builtin(nameobj)) { + Py_DECREF(nameobj); + strcpy(buf, name); + return &fd_builtin; + } +#ifdef MS_COREDLL + fp = _PyWin_FindRegisteredModule(nameobj, &fdp, &filename); + if (fp != NULL) { + Py_DECREF(nameobj); + filename_bytes = PyUnicode_EncodeFSDefault(filename); + Py_DECREF(filename); + if (filename_bytes == NULL) + return NULL; + strncpy(buf, PyBytes_AS_STRING(filename_bytes), buflen); + buf[buflen-1] = '\0'; + Py_DECREF(filename_bytes); + *p_fp = fp; + return fdp; + } + else if (PyErr_Occurred()) + return NULL; +#endif + Py_DECREF(nameobj); + search_path_list = PySys_GetObject("path"); + } + + if (search_path_list == NULL || !PyList_Check(search_path_list)) { + PyErr_SetString(PyExc_ImportError, + "sys.path must be a list of directory names"); + return NULL; + } + + path_hooks = PySys_GetObject("path_hooks"); + if (path_hooks == NULL || !PyList_Check(path_hooks)) { + PyErr_SetString(PyExc_ImportError, + "sys.path_hooks must be a list of " + "import hooks"); + return NULL; + } + path_importer_cache = PySys_GetObject("path_importer_cache"); + if (path_importer_cache == NULL || + !PyDict_Check(path_importer_cache)) { + PyErr_SetString(PyExc_ImportError, + "sys.path_importer_cache must be a dict"); + return NULL; + } + + return find_module_path_list(fullname, name, + search_path_list, path_hooks, + path_importer_cache, + buf, buflen, + p_fp, p_loader); +} + /* case_ok(char* buf, Py_ssize_t len, Py_ssize_t namelen, char* name) * The arguments here are tricky, best shown by example: * /a/b/c/d/e/f/g/h/i/j/k/some_long_module_name.py\0 -- cgit v1.2.1 From 2d77d7eabcf50177446677639435d797b85722e7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 12 Mar 2011 16:02:28 -0500 Subject: Issue #3080: Create find_module_path() subfunction --- Python/import.c | 228 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 139 insertions(+), 89 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 565f1336cb..59b284654e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1598,6 +1598,129 @@ static int case_ok(char *, Py_ssize_t, Py_ssize_t, const char *); static int find_init_module(char *); /* Forward */ static struct filedescr importhookdescr = {"", "", IMP_HOOK}; +/* Get the path of a module: get its importer and call importer.find_module() + hook, or check if the module if a package (if path/__init__.py exists). + + -1: error: a Python error occurred + 0: ignore: an error occurred because of invalid data, but the error is not + important enough to be reported. + 1: get path: module not found, but *buf contains its path + 2: found: *p_fd is the file descriptor (IMP_HOOK or PKG_DIRECTORY) + and *buf is the path */ + +static int +find_module_path(char *fullname, const char *name, PyObject *path, + PyObject *path_hooks, PyObject *path_importer_cache, + char *buf, size_t buflen, + PyObject **p_loader, struct filedescr **p_fd) +{ + PyObject *path_bytes; + const char *base; + Py_ssize_t len; + size_t namelen; + struct stat statbuf; + static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; + + if (PyUnicode_Check(path)) { + path_bytes = PyUnicode_EncodeFSDefault(path); + if (path_bytes == NULL) + return -1; + } + else if (PyBytes_Check(path)) { + Py_INCREF(path); + path_bytes = path; + } + else + return 0; + + namelen = strlen(name); + base = PyBytes_AS_STRING(path_bytes); + len = PyBytes_GET_SIZE(path_bytes); + if (len + 2 + namelen + MAXSUFFIXSIZE >= buflen) { + Py_DECREF(path_bytes); + return 0; /* Too long */ + } + strcpy(buf, base); + Py_DECREF(path_bytes); + + if (strlen(buf) != len) { + return 0; /* path_bytes contains '\0' */ + } + + /* sys.path_hooks import hook */ + if (p_loader != NULL) { + PyObject *importer; + + importer = get_path_importer(path_importer_cache, + path_hooks, path); + if (importer == NULL) { + return -1; + } + /* Note: importer is a borrowed reference */ + if (importer != Py_None) { + PyObject *loader; + loader = PyObject_CallMethod(importer, + "find_module", + "s", fullname); + if (loader == NULL) + return -1; /* error */ + if (loader != Py_None) { + /* a loader was found */ + *p_loader = loader; + *p_fd = &importhookdescr; + return 2; + } + Py_DECREF(loader); + return 0; + } + } + /* no hook was found, use builtin import */ + + if (len > 0 && buf[len-1] != SEP +#ifdef ALTSEP + && buf[len-1] != ALTSEP +#endif + ) + buf[len++] = SEP; + strcpy(buf+len, name); + len += namelen; + + /* Check for package import (buf holds a directory name, + and there's an __init__ module in that directory */ +#ifdef HAVE_STAT + if (stat(buf, &statbuf) == 0 && /* it exists */ + S_ISDIR(statbuf.st_mode) && /* it's a directory */ + case_ok(buf, len, namelen, name)) { /* case matches */ + if (find_init_module(buf)) { /* and has __init__.py */ + *p_fd = &fd_package; + return 2; + } + else { + int err; + PyObject *unicode = PyUnicode_DecodeFSDefault(buf); + if (unicode == NULL) + return -1; + err = PyErr_WarnFormat(PyExc_ImportWarning, 1, + "Not importing directory '%U': missing __init__.py", + unicode); + Py_DECREF(unicode); + if (err) + return -1; + } + } +#endif + return 1; +} + +/* Find a module in search_path_list. For each path, try + find_module_filename() or try each _PyImport_Filetab suffix. + + If the module is found, return a file descriptor, write the path in + *p_filename, write the pointer to the file object into *p_fp, and (if + p_loader is not NULL) the loader into *p_loader. + + Otherwise, raise an exception and return NULL. */ + static struct filedescr* find_module_path_list(char *fullname, const char *name, PyObject *search_path_list, PyObject *path_hooks, @@ -1610,8 +1733,6 @@ find_module_path_list(char *fullname, const char *name, struct filedescr *fdp = NULL; char *filemode; FILE *fp = NULL; - struct stat statbuf; - static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; #if defined(PYOS_OS2) size_t saved_len; size_t saved_namelen; @@ -1621,96 +1742,25 @@ find_module_path_list(char *fullname, const char *name, npath = PyList_Size(search_path_list); namelen = strlen(name); for (i = 0; i < npath; i++) { - PyObject *v = PyList_GetItem(search_path_list, i); - PyObject *origv = v; - const char *base; - Py_ssize_t size; - if (!v) - return NULL; - if (PyUnicode_Check(v)) { - v = PyUnicode_EncodeFSDefault(v); - if (v == NULL) - return NULL; - } - else if (!PyBytes_Check(v)) - continue; - else - Py_INCREF(v); - - base = PyBytes_AS_STRING(v); - size = PyBytes_GET_SIZE(v); - len = size; - if (len + 2 + namelen + MAXSUFFIXSIZE >= buflen) { - Py_DECREF(v); - continue; /* Too long */ - } - strcpy(buf, base); - Py_DECREF(v); + PyObject *path; + int ok; - if (strlen(buf) != len) { - continue; /* v contains '\0' */ - } - - /* sys.path_hooks import hook */ - if (p_loader != NULL) { - PyObject *importer; - - importer = get_path_importer(path_importer_cache, - path_hooks, origv); - if (importer == NULL) { - return NULL; - } - /* Note: importer is a borrowed reference */ - if (importer != Py_None) { - PyObject *loader; - loader = PyObject_CallMethod(importer, - "find_module", - "s", fullname); - if (loader == NULL) - return NULL; /* error */ - if (loader != Py_None) { - /* a loader was found */ - *p_loader = loader; - return &importhookdescr; - } - Py_DECREF(loader); - continue; - } - } - /* no hook was found, use builtin import */ + path = PyList_GetItem(search_path_list, i); + if (path == NULL) + return NULL; - if (len > 0 && buf[len-1] != SEP -#ifdef ALTSEP - && buf[len-1] != ALTSEP -#endif - ) - buf[len++] = SEP; - strcpy(buf+len, name); - len += namelen; + ok = find_module_path(fullname, name, path, + path_hooks, path_importer_cache, + buf, buflen, + p_loader, &fdp); + if (ok < 0) + return NULL; + if (ok == 0) + continue; + if (ok == 2) + return fdp; - /* Check for package import (buf holds a directory name, - and there's an __init__ module in that directory */ -#ifdef HAVE_STAT - if (stat(buf, &statbuf) == 0 && /* it exists */ - S_ISDIR(statbuf.st_mode) && /* it's a directory */ - case_ok(buf, len, namelen, name)) { /* case matches */ - if (find_init_module(buf)) { /* and has __init__.py */ - return &fd_package; - } - else { - int err; - PyObject *unicode = PyUnicode_DecodeFSDefault(buf); - if (unicode == NULL) - return NULL; - err = PyErr_WarnFormat(PyExc_ImportWarning, 1, - "Not importing directory '%U': missing __init__.py", - unicode); - Py_DECREF(unicode); - if (err) - return NULL; - } - } -#endif + len = strlen(buf); #if defined(PYOS_OS2) /* take a snapshot of the module spec for restoration * after the 8 character DLL hackery -- cgit v1.2.1 From 8509b2931d82b09524d8922b5ffd1871f5ff6311 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 13:33:46 -0400 Subject: Issue #3080: get_sourcefile(), make_source_pathname(), load_package() Use Unicode for module name and path in get_sourcefile(), make_source_pathname() and load_package() functions. --- Python/import.c | 219 +++++++++++++++++++++++++++++++++----------------------- 1 file changed, 131 insertions(+), 88 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 59b284654e..f6cde957a8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -113,6 +113,8 @@ typedef unsigned short mode_t; #define MAGIC (3180 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-32" #define CACHEDIR "__pycache__" +static const Py_UNICODE CACHEDIR_UNICODE[] = { + '_', '_', 'p', 'y', 'c', 'a', 'c', 'h', 'e', '_', '_', '\0'}; /* Current magic word and string tag as globals. */ static long pyc_magic = MAGIC; static const char *pyc_tag = TAG; @@ -741,8 +743,8 @@ remove_module(PyObject *name) "sys.modules failed"); } -static PyObject * get_sourcefile(char *file); -static char *make_source_pathname(char *pathname, char *buf); +static PyObject * get_sourcefile(PyObject *filename); +static PyObject *make_source_pathname(PyObject *pathname); static char *make_compiled_pathname(char *pathname, char *buf, size_t buflen, int debug); @@ -807,7 +809,6 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, { PyObject *modules = PyImport_GetModuleDict(); PyObject *m, *d, *v; - PyObject *pathbytes; m = PyImport_AddModuleObject(name); if (m == NULL) @@ -822,12 +823,7 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, } /* Remember the filename as the __file__ attribute */ if (pathname != NULL) { - pathbytes = PyUnicode_EncodeFSDefault(pathname); - if (pathbytes != NULL) { - v = get_sourcefile(PyBytes_AS_STRING(pathbytes)); - Py_DECREF(pathbytes); - } else - v = NULL; + v = get_sourcefile(pathname); if (v == NULL) PyErr_Clear(); } @@ -892,6 +888,27 @@ rightmost_sep(char *s) } +/* Like strrchr(string, '/') but searches for the rightmost of either SEP + or ALTSEP, if the latter is defined. +*/ +static Py_UNICODE* +rightmost_sep_unicode(Py_UNICODE *s) +{ + Py_UNICODE *found, c; + for (found = NULL; (c = *s); s++) { + if (c == SEP +#ifdef ALTSEP + || c == ALTSEP +#endif + ) + { + found = s; + } + } + return found; +} + + /* Given a pathname for a Python source file, fill a buffer with the pathname for the corresponding compiled file. Return the pathname for the compiled file, or NULL if there's no space in the buffer. @@ -1005,42 +1022,50 @@ make_compiled_pathname(char *pathname, char *buf, size_t buflen, int debug) source file, if the path matches the PEP 3147 format. This does not check for any file existence, however, if the pyc file name does not match PEP 3147 style, NULL is returned. buf must be at least as big as pathname; - the resulting path will always be shorter. */ + the resulting path will always be shorter. -static char * -make_source_pathname(char *pathname, char *buf) + (...)/__pycache__/foo..pyc -> (...)/foo.py */ + +static PyObject* +make_source_pathname(PyObject *pathobj) { - /* __pycache__/foo..pyc -> foo.py */ + Py_UNICODE buf[MAXPATHLEN]; + Py_UNICODE *pathname; + Py_UNICODE *left, *right, *dot0, *dot1, sep; size_t i, j; - char *left, *right, *dot0, *dot1, sep; + + if (PyUnicode_GET_SIZE(pathobj) > MAXPATHLEN) + return NULL; + pathname = PyUnicode_AS_UNICODE(pathobj); /* Look back two slashes from the end. In between these two slashes must be the string __pycache__ or this is not a PEP 3147 style path. It's possible for there to be only one slash. */ - if ((right = rightmost_sep(pathname)) == NULL) + right = rightmost_sep_unicode(pathname); + if (right == NULL) return NULL; sep = *right; *right = '\0'; - left = rightmost_sep(pathname); + left = rightmost_sep_unicode(pathname); *right = sep; if (left == NULL) left = pathname; else left++; - if (right-left != strlen(CACHEDIR) || - strncmp(left, CACHEDIR, right-left) != 0) + if (right-left != Py_UNICODE_strlen(CACHEDIR_UNICODE) || + Py_UNICODE_strncmp(left, CACHEDIR_UNICODE, right-left) != 0) return NULL; /* Now verify that the path component to the right of the last slash has two dots in it. */ - if ((dot0 = strchr(right + 1, '.')) == NULL) + if ((dot0 = Py_UNICODE_strchr(right + 1, '.')) == NULL) return NULL; - if ((dot1 = strchr(dot0 + 1, '.')) == NULL) + if ((dot1 = Py_UNICODE_strchr(dot0 + 1, '.')) == NULL) return NULL; /* Too many dots? */ - if (strchr(dot1 + 1, '.') != NULL) + if (Py_UNICODE_strchr(dot1 + 1, '.') != NULL) return NULL; /* This is a PEP 3147 path. Start by copying everything from the @@ -1048,10 +1073,11 @@ make_source_pathname(char *pathname, char *buf) copy the file's basename, removing the magic tag and adding a .py suffix. */ - strncpy(buf, pathname, (i=left-pathname)); - strncpy(buf+i, right+1, (j=dot0-right)); - strcpy(buf+i+j, "py"); - return buf; + Py_UNICODE_strncpy(buf, pathname, (i=left-pathname)); + Py_UNICODE_strncpy(buf+i, right+1, (j=dot0-right)); + buf[i+j] = 'p'; + buf[i+j+1] = 'y'; + return PyUnicode_FromUnicode(buf, i+j+2); } /* Given a pathname for a Python source file, its time of last @@ -1390,40 +1416,47 @@ load_source_module(char *name, char *pathname, FILE *fp) * Returns the path to the py file if available, else the given path */ static PyObject * -get_sourcefile(char *file) +get_sourcefile(PyObject *filename) { - char py[MAXPATHLEN + 1]; Py_ssize_t len; - PyObject *u; + Py_UNICODE *fileuni; + PyObject *py; struct stat statbuf; - if (!file || !*file) { + len = PyUnicode_GET_SIZE(filename); + if (len == 0) Py_RETURN_NONE; - } - len = strlen(file); - /* match '*.py?' */ - if (len > MAXPATHLEN || PyOS_strnicmp(&file[len-4], ".py", 3) != 0) { - return PyUnicode_DecodeFSDefault(file); - } + /* don't match *.pyc or *.pyo? */ + fileuni = PyUnicode_AS_UNICODE(filename); + if (len < 5 + || fileuni[len-4] != '.' + || (fileuni[len-3] != 'p' && fileuni[len-3] != 'P') + || (fileuni[len-2] != 'y' && fileuni[len-2] != 'Y')) + goto unchanged; /* Start by trying to turn PEP 3147 path into source path. If that * fails, just chop off the trailing character, i.e. legacy pyc path * to py. */ - if (make_source_pathname(file, py) == NULL) { - strncpy(py, file, len-1); - py[len-1] = '\0'; + py = make_source_pathname(filename); + if (py == NULL) { + PyErr_Clear(); + py = PyUnicode_FromUnicode(fileuni, len - 1); } + if (py == NULL) + goto error; - if (stat(py, &statbuf) == 0 && - S_ISREG(statbuf.st_mode)) { - u = PyUnicode_DecodeFSDefault(py); - } - else { - u = PyUnicode_DecodeFSDefault(file); - } - return u; + if (_Py_stat(py, &statbuf) == 0 && S_ISREG(statbuf.st_mode)) + return py; + Py_DECREF(py); + goto unchanged; + +error: + PyErr_Clear(); +unchanged: + Py_INCREF(filename); + return filename; } /* Forward */ @@ -1436,54 +1469,56 @@ static struct _frozen * find_frozen(PyObject *); REFERENCE COUNT */ static PyObject * -load_package(char *name, char *pathname) +load_package(PyObject *name, PyObject *pathname) { PyObject *m, *d; - PyObject *file = NULL; - PyObject *path = NULL; + PyObject *file = NULL, *path_list = NULL; int err; char buf[MAXPATHLEN+1]; - FILE *fp; + FILE *fp = NULL; struct filedescr *fdp; + char *namestr; - m = PyImport_AddModule(name); + m = PyImport_AddModuleObject(name); if (m == NULL) return NULL; if (Py_VerboseFlag) - PySys_WriteStderr("import %s # directory %s\n", + PySys_FormatStderr("import %U # directory %U\n", name, pathname); - d = PyModule_GetDict(m); file = get_sourcefile(pathname); if (file == NULL) - goto error; - path = Py_BuildValue("[O]", file); - if (path == NULL) - goto error; + return NULL; + path_list = Py_BuildValue("[O]", file); + if (path_list == NULL) { + Py_DECREF(file); + return NULL; + } + d = PyModule_GetDict(m); err = PyDict_SetItemString(d, "__file__", file); + Py_DECREF(file); if (err == 0) - err = PyDict_SetItemString(d, "__path__", path); - if (err != 0) + err = PyDict_SetItemString(d, "__path__", path_list); + if (err != 0) { + Py_DECREF(path_list); + return NULL; + } + namestr = _PyUnicode_AsString(name); + if (namestr == NULL) goto error; - fdp = find_module(name, "__init__", path, buf, sizeof(buf), &fp, NULL); + fdp = find_module(namestr, "__init__", path_list, buf, sizeof(buf), &fp, NULL); + Py_DECREF(path_list); if (fdp == NULL) { if (PyErr_ExceptionMatches(PyExc_ImportError)) { PyErr_Clear(); Py_INCREF(m); + return m; } else - m = NULL; - goto cleanup; + return NULL; } - m = load_module(name, fp, buf, fdp->type, NULL); + m = load_module(namestr, fp, buf, fdp->type, NULL); if (fp != NULL) fclose(fp); - goto cleanup; - - error: - m = NULL; - cleanup: - Py_XDECREF(path); - Py_XDECREF(file); return m; } @@ -2282,9 +2317,21 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) } #endif - case PKG_DIRECTORY: - m = load_package(name, pathname); + case PKG_DIRECTORY: { + PyObject *nameobj, *pathobj; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + pathobj = PyUnicode_DecodeFSDefault(pathname); + if (pathobj == NULL) { + Py_DECREF(nameobj); + return NULL; + } + m = load_package(nameobj, pathobj); + Py_DECREF(nameobj); + Py_DECREF(pathobj); break; + } case C_BUILTIN: case PY_FROZEN: { @@ -3637,13 +3684,12 @@ imp_load_module(PyObject *self, PyObject *args) static PyObject * imp_load_package(PyObject *self, PyObject *args) { - char *name; - PyObject *pathname; + PyObject *name, *pathname; PyObject * ret; - if (!PyArg_ParseTuple(args, "sO&:load_package", - &name, PyUnicode_FSConverter, &pathname)) + if (!PyArg_ParseTuple(args, "UO&:load_package", + &name, PyUnicode_FSDecoder, &pathname)) return NULL; - ret = load_package(name, PyBytes_AS_STRING(pathname)); + ret = load_package(name, pathname); Py_DECREF(pathname); return ret; } @@ -3716,25 +3762,22 @@ static PyObject * imp_source_from_cache(PyObject *self, PyObject *args, PyObject *kws) { static char *kwlist[] = {"path", NULL}; - - PyObject *pathname_obj; - char *pathname; - char buf[MAXPATHLEN+1]; + PyObject *pathname, *source; if (!PyArg_ParseTupleAndKeywords( args, kws, "O&", kwlist, - PyUnicode_FSConverter, &pathname_obj)) + PyUnicode_FSDecoder, &pathname)) return NULL; - pathname = PyBytes_AS_STRING(pathname_obj); - if (make_source_pathname(pathname, buf) == NULL) { - PyErr_Format(PyExc_ValueError, "Not a PEP 3147 pyc path: %s", + source = make_source_pathname(pathname); + if (source == NULL) { + PyErr_Format(PyExc_ValueError, "Not a PEP 3147 pyc path: %R", pathname); - Py_DECREF(pathname_obj); + Py_DECREF(pathname); return NULL; } - Py_DECREF(pathname_obj); - return PyUnicode_FromString(buf); + Py_DECREF(pathname); + return source; } PyDoc_STRVAR(doc_source_from_cache, -- cgit v1.2.1 From 26e6d2c66ccad73bad2edeb441f54141164d0bc1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 20 Mar 2011 00:41:24 +0100 Subject: Issue #3080: Use Unicode to import source and compiled modules * Use Unicode for module name and path in the following functions: * get_file() * load_source_module(), parse_source_module() * make_compiled_pathname(), check_compiled_module(), read_compiled_module(), load_compiled_module(), write_compiled_module(), update_compiled_module() * On Windows, use CreateDirectoryW() instead of mkdir() * update_compiled_module() cannot fail anymore --- Python/import.c | 352 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 201 insertions(+), 151 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index f6cde957a8..0efffe9149 100644 --- a/Python/import.c +++ b/Python/import.c @@ -107,8 +107,8 @@ typedef unsigned short mode_t; /* MAGIC must change whenever the bytecode emitted by the compiler may no longer be understood by older implementations of the eval loop (usually due to the addition of new opcodes) - TAG must change for each major Python release. The magic number will take - care of any bytecode changes that occur during development. + TAG and PYC_TAG_UNICODE must change for each major Python release. The magic + number will take care of any bytecode changes that occur during development. */ #define MAGIC (3180 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-32" @@ -118,6 +118,8 @@ static const Py_UNICODE CACHEDIR_UNICODE[] = { /* Current magic word and string tag as globals. */ static long pyc_magic = MAGIC; static const char *pyc_tag = TAG; +static const Py_UNICODE PYC_TAG_UNICODE[] = { + 'c', 'p', 'y', 't', 'h', 'o', 'n', '-', '3', '2', '\0'}; /* See _PyImport_FixupExtensionObject() below */ static PyObject *extensions = NULL; @@ -745,8 +747,7 @@ remove_module(PyObject *name) static PyObject * get_sourcefile(PyObject *filename); static PyObject *make_source_pathname(PyObject *pathname); -static char *make_compiled_pathname(char *pathname, char *buf, size_t buflen, - int debug); +static PyObject* make_compiled_pathname(Py_UNICODE *pathname, int debug); /* Execute a code object in a module and return the module object * WITH INCREMENTED REFERENCE COUNT. If an error occurs, name is @@ -867,32 +868,11 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, } -/* Like strrchr(string, '/') but searches for the rightmost of either SEP - or ALTSEP, if the latter is defined. -*/ -static char * -rightmost_sep(char *s) -{ - char *found, c; - for (found = NULL; (c = *s); s++) { - if (c == SEP -#ifdef ALTSEP - || c == ALTSEP -#endif - ) - { - found = s; - } - } - return found; -} - - /* Like strrchr(string, '/') but searches for the rightmost of either SEP or ALTSEP, if the latter is defined. */ static Py_UNICODE* -rightmost_sep_unicode(Py_UNICODE *s) +rightmost_sep(Py_UNICODE *s) { Py_UNICODE *found, c; for (found = NULL; (c = *s); s++) { @@ -912,15 +892,18 @@ rightmost_sep_unicode(Py_UNICODE *s) /* Given a pathname for a Python source file, fill a buffer with the pathname for the corresponding compiled file. Return the pathname for the compiled file, or NULL if there's no space in the buffer. - Doesn't set an exception. */ + Doesn't set an exception. + + foo.py -> __pycache__/foo..pyc */ -static char * -make_compiled_pathname(char *pathname, char *buf, size_t buflen, int debug) +static PyObject* +make_compiled_pathname(Py_UNICODE *pathname, int debug) { - /* foo.py -> __pycache__/foo..pyc */ - size_t len = strlen(pathname); + Py_UNICODE buf[MAXPATHLEN]; + size_t buflen = (size_t)MAXPATHLEN; + size_t len = Py_UNICODE_strlen(pathname); size_t i, save; - char *pos; + Py_UNICODE *pos; int sep = SEP; /* Sanity check that the buffer has roughly enough space to hold what @@ -932,35 +915,37 @@ make_compiled_pathname(char *pathname, char *buf, size_t buflen, int debug) sanity check before writing the extension to ensure we do not overflow the buffer. */ - if (len + strlen(CACHEDIR) + strlen(pyc_tag) + 5 > buflen) + if (len + Py_UNICODE_strlen(CACHEDIR_UNICODE) + Py_UNICODE_strlen(PYC_TAG_UNICODE) + 5 > buflen) return NULL; /* Find the last path separator and copy everything from the start of the source string up to and including the separator. */ - if ((pos = rightmost_sep(pathname)) == NULL) { + pos = rightmost_sep(pathname); + if (pos == NULL) { i = 0; } else { sep = *pos; i = pos - pathname + 1; - strncpy(buf, pathname, i); + Py_UNICODE_strncpy(buf, pathname, i); } save = i; buf[i++] = '\0'; /* Add __pycache__/ */ - strcat(buf, CACHEDIR); - i += strlen(CACHEDIR) - 1; + Py_UNICODE_strcat(buf, CACHEDIR_UNICODE); + i += Py_UNICODE_strlen(CACHEDIR_UNICODE) - 1; buf[i++] = sep; buf[i++] = '\0'; /* Add the base filename, but remove the .py or .pyw extension, since the tag name must go before the extension. */ - strcat(buf, pathname + save); - if ((pos = strrchr(buf, '.')) != NULL) + Py_UNICODE_strcat(buf, pathname + save); + pos = Py_UNICODE_strrchr(buf, '.'); + if (pos != NULL) *++pos = '\0'; - strcat(buf, pyc_tag); + Py_UNICODE_strcat(buf, PYC_TAG_UNICODE); /* The length test above assumes that we're only adding one character to the end of what would normally be the extension. What if there is no extension, or the string ends in '.' or '.p', and otherwise @@ -1010,11 +995,15 @@ make_compiled_pathname(char *pathname, char *buf, size_t buflen, int debug) #if 0 printf("strlen(buf): %d; buflen: %d\n", (int)strlen(buf), (int)buflen); #endif - if (strlen(buf) + 5 > buflen) + len = Py_UNICODE_strlen(buf); + if (len + 5 > buflen) return NULL; - strcat(buf, debug ? ".pyc" : ".pyo"); - assert(strlen(buf) < buflen); - return buf; + buf[len] = '.'; len++; + buf[len] = 'p'; len++; + buf[len] = 'y'; len++; + buf[len] = debug ? 'c' : 'o'; len++; + assert(len <= buflen); + return PyUnicode_FromUnicode(buf, len); } @@ -1042,12 +1031,12 @@ make_source_pathname(PyObject *pathobj) must be the string __pycache__ or this is not a PEP 3147 style path. It's possible for there to be only one slash. */ - right = rightmost_sep_unicode(pathname); + right = rightmost_sep(pathname); if (right == NULL) return NULL; sep = *right; *right = '\0'; - left = rightmost_sep_unicode(pathname); + left = rightmost_sep(pathname); *right = sep; if (left == NULL) left = pathname; @@ -1088,31 +1077,31 @@ make_source_pathname(PyObject *pathobj) Doesn't set an exception. */ static FILE * -check_compiled_module(char *pathname, time_t mtime, char *cpathname) +check_compiled_module(PyObject *pathname, time_t mtime, PyObject *cpathname) { FILE *fp; long magic; long pyc_mtime; - fp = fopen(cpathname, "rb"); + fp = _Py_fopen(cpathname, "rb"); if (fp == NULL) return NULL; magic = PyMarshal_ReadLongFromFile(fp); if (magic != pyc_magic) { if (Py_VerboseFlag) - PySys_WriteStderr("# %s has bad magic\n", cpathname); + PySys_FormatStderr("# %R has bad magic\n", cpathname); fclose(fp); return NULL; } pyc_mtime = PyMarshal_ReadLongFromFile(fp); if (pyc_mtime != mtime) { if (Py_VerboseFlag) - PySys_WriteStderr("# %s has bad mtime\n", cpathname); + PySys_FormatStderr("# %R has bad mtime\n", cpathname); fclose(fp); return NULL; } if (Py_VerboseFlag) - PySys_WriteStderr("# %s matches %s\n", cpathname, pathname); + PySys_FormatStderr("# %R matches %R\n", cpathname, pathname); return fp; } @@ -1120,7 +1109,7 @@ check_compiled_module(char *pathname, time_t mtime, char *cpathname) /* Read a code object from a file and check it for validity */ static PyCodeObject * -read_compiled_module(char *cpathname, FILE *fp) +read_compiled_module(PyObject *cpathname, FILE *fp) { PyObject *co; @@ -1129,7 +1118,7 @@ read_compiled_module(char *cpathname, FILE *fp) return NULL; if (!PyCode_Check(co)) { PyErr_Format(PyExc_ImportError, - "Non-code object in %.200s", cpathname); + "Non-code object in %R", cpathname); Py_DECREF(co); return NULL; } @@ -1141,7 +1130,7 @@ read_compiled_module(char *cpathname, FILE *fp) module object WITH INCREMENTED REFERENCE COUNT */ static PyObject * -load_compiled_module(char *name, char *cpathname, FILE *fp) +load_compiled_module(PyObject *name, PyObject *cpathname, FILE *fp) { long magic; PyCodeObject *co; @@ -1150,7 +1139,7 @@ load_compiled_module(char *name, char *cpathname, FILE *fp) magic = PyMarshal_ReadLongFromFile(fp); if (magic != pyc_magic) { PyErr_Format(PyExc_ImportError, - "Bad magic number in %.200s", cpathname); + "Bad magic number in %R", cpathname); return NULL; } (void) PyMarshal_ReadLongFromFile(fp); @@ -1158,10 +1147,10 @@ load_compiled_module(char *name, char *cpathname, FILE *fp) if (co == NULL) return NULL; if (Py_VerboseFlag) - PySys_WriteStderr("import %s # precompiled from %s\n", - name, cpathname); - m = PyImport_ExecCodeModuleWithPathnames( - name, (PyObject *)co, cpathname, cpathname); + PySys_FormatStderr("import %U # precompiled from %R\n", + name, cpathname); + m = PyImport_ExecCodeModuleObject(name, (PyObject *)co, + cpathname, cpathname); Py_DECREF(co); return m; @@ -1170,27 +1159,37 @@ load_compiled_module(char *name, char *cpathname, FILE *fp) /* Parse a source file and return the corresponding code object */ static PyCodeObject * -parse_source_module(const char *pathname, FILE *fp) +parse_source_module(PyObject *pathname, FILE *fp) { - PyCodeObject *co = NULL; + PyCodeObject *co; + PyObject *pathbytes; mod_ty mod; PyCompilerFlags flags; - PyArena *arena = PyArena_New(); - if (arena == NULL) + PyArena *arena; + + pathbytes = PyUnicode_EncodeFSDefault(pathname); + if (pathbytes == NULL) + return NULL; + + arena = PyArena_New(); + if (arena == NULL) { + Py_DECREF(pathbytes); return NULL; + } flags.cf_flags = 0; - mod = PyParser_ASTFromFile(fp, pathname, NULL, + mod = PyParser_ASTFromFile(fp, PyBytes_AS_STRING(pathbytes), NULL, Py_file_input, 0, 0, &flags, NULL, arena); - if (mod) { - co = PyAST_Compile(mod, pathname, NULL, arena); - } + if (mod != NULL) + co = PyAST_Compile(mod, PyBytes_AS_STRING(pathbytes), NULL, arena); + else + co = NULL; + Py_DECREF(pathbytes); PyArena_Free(arena); return co; } - /* Helper to open a bytecode file for writing in exclusive mode */ static FILE * @@ -1231,10 +1230,10 @@ open_exclusive(char *filename, mode_t mode) remove the file. */ static void -write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat) +write_compiled_module(PyCodeObject *co, PyObject *cpathname, + struct stat *srcstat) { FILE *fp; - char *dirpath; time_t mtime = srcstat->st_mtime; #ifdef MS_WINDOWS /* since Windows uses different permissions */ mode_t mode = srcstat->st_mode & ~S_IEXEC; @@ -1243,39 +1242,64 @@ write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat) mode_t dirmode = (srcstat->st_mode | S_IXUSR | S_IXGRP | S_IXOTH | S_IWUSR | S_IWGRP | S_IWOTH); + PyObject *dirbytes; #endif - int saved; + PyObject *cpathbytes, *dirname; + Py_UNICODE *dirsep; + int res, ok; /* Ensure that the __pycache__ directory exists. */ - dirpath = rightmost_sep(cpathname); - if (dirpath == NULL) { + dirsep = rightmost_sep(PyUnicode_AS_UNICODE(cpathname)); + if (dirsep == NULL) { if (Py_VerboseFlag) - PySys_WriteStderr( - "# no %s path found %s\n", - CACHEDIR, cpathname); + PySys_FormatStderr("# no %s path found %R\n", CACHEDIR, cpathname); + return; + } + dirname = PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(cpathname), + dirsep - PyUnicode_AS_UNICODE(cpathname)); + if (dirname == NULL) { + PyErr_Clear(); return; } - saved = *dirpath; - *dirpath = '\0'; #ifdef MS_WINDOWS - if (_mkdir(cpathname) < 0 && errno != EEXIST) { + res = CreateDirectoryW(PyUnicode_AS_UNICODE(dirname), NULL); + ok = (res != 0); + if (!ok && GetLastError() == ERROR_ALREADY_EXISTS) + ok = 1; #else - if (mkdir(cpathname, dirmode) < 0 && errno != EEXIST) { + dirbytes = PyUnicode_EncodeFSDefault(dirname); + if (dirbytes == NULL) { + PyErr_Clear(); + return; + } + res = mkdir(PyBytes_AS_STRING(dirbytes), dirmode); + Py_DECREF(dirbytes); + if (0 <= res) + ok = 1; + else + ok = (errno == EEXIST); #endif - *dirpath = saved; + if (!ok) { if (Py_VerboseFlag) - PySys_WriteStderr( - "# cannot create cache dir %s\n", cpathname); + PySys_FormatStderr("# cannot create cache dir %R\n", dirname); + Py_DECREF(dirname); + return; + } + Py_DECREF(dirname); + + cpathbytes = PyUnicode_EncodeFSDefault(cpathname); + if (cpathbytes == NULL) { + PyErr_Clear(); return; } - *dirpath = saved; - fp = open_exclusive(cpathname, mode); + fp = open_exclusive(PyBytes_AS_STRING(cpathbytes), mode); if (fp == NULL) { if (Py_VerboseFlag) - PySys_WriteStderr( - "# can't create %s\n", cpathname); + PySys_FormatStderr( + "# can't create %R\n", cpathname); + Py_DECREF(cpathbytes); return; } PyMarshal_WriteLongToFile(pyc_magic, fp, Py_MARSHAL_VERSION); @@ -1284,12 +1308,18 @@ write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat) PyMarshal_WriteObjectToFile((PyObject *)co, fp, Py_MARSHAL_VERSION); if (fflush(fp) != 0 || ferror(fp)) { if (Py_VerboseFlag) - PySys_WriteStderr("# can't write %s\n", cpathname); + PySys_FormatStderr("# can't write %R\n", cpathname); /* Don't keep partial file */ fclose(fp); - (void) unlink(cpathname); +#ifdef MS_WINDOWS + (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname)); +#else + (void) unlink(PyBytes_AS_STRING(cpathbytes)); +#endif + Py_DECREF(cpathbytes); return; } + Py_DECREF(cpathbytes); /* Now write the true mtime */ fseek(fp, 4L, 0); assert(mtime < LONG_MAX); @@ -1297,7 +1327,7 @@ write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat) fflush(fp); fclose(fp); if (Py_VerboseFlag) - PySys_WriteStderr("# wrote %s\n", cpathname); + PySys_FormatStderr("# wrote %R\n", cpathname); } static void @@ -1324,26 +1354,18 @@ update_code_filenames(PyCodeObject *co, PyObject *oldname, PyObject *newname) } } -static int -update_compiled_module(PyCodeObject *co, char *pathname) +static void +update_compiled_module(PyCodeObject *co, PyObject *newname) { - PyObject *oldname, *newname; + PyObject *oldname; - newname = PyUnicode_DecodeFSDefault(pathname); - if (newname == NULL) - return -1; - - if (!PyUnicode_Compare(co->co_filename, newname)) { - Py_DECREF(newname); - return 0; - } + if (PyUnicode_Compare(co->co_filename, newname) == 0) + return; oldname = co->co_filename; Py_INCREF(oldname); update_code_filenames(co, oldname, newname); Py_DECREF(oldname); - Py_DECREF(newname); - return 1; } /* Load a source module from a given file and return its module @@ -1351,20 +1373,19 @@ update_compiled_module(PyCodeObject *co, char *pathname) byte-compiled file, use that instead. */ static PyObject * -load_source_module(char *name, char *pathname, FILE *fp) +load_source_module(PyObject *name, PyObject *pathname, FILE *fp) { struct stat st; FILE *fpc; - char buf[MAXPATHLEN+1]; - char *cpathname; + PyObject *cpathname = NULL, *cpathbytes = NULL; PyCodeObject *co; - PyObject *m; + PyObject *m = NULL; if (fstat(fileno(fp), &st) != 0) { PyErr_Format(PyExc_RuntimeError, - "unable to get file status from '%s'", + "unable to get file status from %R", pathname); - return NULL; + goto error; } #if SIZEOF_TIME_T > 4 /* Python's .pyc timestamp handling presumes that the timestamp fits @@ -1374,41 +1395,50 @@ load_source_module(char *name, char *pathname, FILE *fp) if (st.st_mtime >> 32) { PyErr_SetString(PyExc_OverflowError, "modification time overflows a 4 byte field"); - return NULL; + goto error; } #endif cpathname = make_compiled_pathname( - pathname, buf, (size_t)MAXPATHLEN + 1, !Py_OptimizeFlag); - if (cpathname != NULL && - (fpc = check_compiled_module(pathname, st.st_mtime, cpathname))) { + PyUnicode_AS_UNICODE(pathname), + !Py_OptimizeFlag); + + if (cpathname != NULL) + fpc = check_compiled_module(pathname, st.st_mtime, cpathname); + else + fpc = NULL; + + if (fpc) { co = read_compiled_module(cpathname, fpc); fclose(fpc); if (co == NULL) - return NULL; - if (update_compiled_module(co, pathname) < 0) - return NULL; + goto error; + update_compiled_module(co, pathname); if (Py_VerboseFlag) - PySys_WriteStderr("import %s # precompiled from %s\n", - name, cpathname); - pathname = cpathname; + PySys_FormatStderr("import %U # precompiled from %R\n", + name, cpathname); + m = PyImport_ExecCodeModuleObject(name, (PyObject *)co, + cpathname, cpathname); } else { co = parse_source_module(pathname, fp); if (co == NULL) - return NULL; + goto error; if (Py_VerboseFlag) - PySys_WriteStderr("import %s # from %s\n", + PySys_FormatStderr("import %U # from %R\n", name, pathname); - if (cpathname) { + if (cpathname != NULL) { PyObject *ro = PySys_GetObject("dont_write_bytecode"); if (ro == NULL || !PyObject_IsTrue(ro)) write_compiled_module(co, cpathname, &st); } + m = PyImport_ExecCodeModuleObject(name, (PyObject *)co, + pathname, cpathname); } - m = PyImport_ExecCodeModuleWithPathnames( - name, (PyObject *)co, pathname, cpathname); Py_DECREF(co); +error: + Py_XDECREF(cpathbytes); + Py_XDECREF(cpathname); return m; } @@ -2291,13 +2321,37 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) switch (type) { - case PY_SOURCE: - m = load_source_module(name, pathname, fp); + case PY_SOURCE: { + PyObject *nameobj, *pathobj; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + pathobj = PyUnicode_DecodeFSDefault(pathname); + if (pathobj == NULL) { + Py_DECREF(nameobj); + return NULL; + } + m = load_source_module(nameobj, pathobj, fp); + Py_DECREF(nameobj); + Py_DECREF(pathobj); break; + } - case PY_COMPILED: - m = load_compiled_module(name, pathname, fp); + case PY_COMPILED: { + PyObject *nameobj, *pathobj; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + pathobj = PyUnicode_DecodeFSDefault(pathname); + if (pathobj == NULL) { + Py_DECREF(nameobj); + return NULL; + } + m = load_compiled_module(nameobj, pathobj, fp); + Py_DECREF(nameobj); + Py_DECREF(pathobj); break; + } #ifdef HAVE_DYNAMIC_LOADING case C_EXTENSION: { @@ -3533,13 +3587,13 @@ imp_is_frozen(PyObject *self, PyObject *args) } static FILE * -get_file(char *pathname, PyObject *fob, char *mode) +get_file(PyObject *pathname, PyObject *fob, char *mode) { FILE *fp; if (mode[0] == 'U') mode = "r" PY_STDIOTEXTMODE; if (fob == NULL) { - fp = fopen(pathname, mode); + fp = _Py_fopen(pathname, mode); } else { int fd = PyObject_AsFileDescriptor(fob); @@ -3565,22 +3619,21 @@ error: static PyObject * imp_load_compiled(PyObject *self, PyObject *args) { - char *name; - PyObject *pathname; + PyObject *name, *pathname; PyObject *fob = NULL; PyObject *m; FILE *fp; - if (!PyArg_ParseTuple(args, "sO&|O:load_compiled", + if (!PyArg_ParseTuple(args, "UO&|O:load_compiled", &name, - PyUnicode_FSConverter, &pathname, + PyUnicode_FSDecoder, &pathname, &fob)) return NULL; - fp = get_file(PyBytes_AS_STRING(pathname), fob, "rb"); + fp = get_file(pathname, fob, "rb"); if (fp == NULL) { Py_DECREF(pathname); return NULL; } - m = load_compiled_module(name, PyBytes_AS_STRING(pathname), fp); + m = load_compiled_module(name, pathname, fp); fclose(fp); Py_DECREF(pathname); return m; @@ -3615,22 +3668,21 @@ imp_load_dynamic(PyObject *self, PyObject *args) static PyObject * imp_load_source(PyObject *self, PyObject *args) { - char *name; - PyObject *pathname; + PyObject *name, *pathname; PyObject *fob = NULL; PyObject *m; FILE *fp; - if (!PyArg_ParseTuple(args, "sO&|O:load_source", + if (!PyArg_ParseTuple(args, "UO&|O:load_source", &name, - PyUnicode_FSConverter, &pathname, + PyUnicode_FSDecoder, &pathname, &fob)) return NULL; - fp = get_file(PyBytes_AS_STRING(pathname), fob, "r"); + fp = get_file(pathname, fob, "r"); if (fp == NULL) { Py_DECREF(pathname); return NULL; } - m = load_source_module(name, PyBytes_AS_STRING(pathname), fp); + m = load_source_module(name, pathname, fp); Py_DECREF(pathname); fclose(fp); return m; @@ -3719,33 +3771,31 @@ imp_cache_from_source(PyObject *self, PyObject *args, PyObject *kws) { static char *kwlist[] = {"path", "debug_override", NULL}; - char buf[MAXPATHLEN+1]; - PyObject *pathbytes; - char *cpathname; + PyObject *pathname, *cpathname; PyObject *debug_override = NULL; int debug = !Py_OptimizeFlag; if (!PyArg_ParseTupleAndKeywords( args, kws, "O&|O", kwlist, - PyUnicode_FSConverter, &pathbytes, &debug_override)) + PyUnicode_FSDecoder, &pathname, &debug_override)) return NULL; if (debug_override != NULL && (debug = PyObject_IsTrue(debug_override)) < 0) { - Py_DECREF(pathbytes); + Py_DECREF(pathname); return NULL; } cpathname = make_compiled_pathname( - PyBytes_AS_STRING(pathbytes), - buf, MAXPATHLEN+1, debug); - Py_DECREF(pathbytes); + PyUnicode_AS_UNICODE(pathname), + debug); + Py_DECREF(pathname); if (cpathname == NULL) { PyErr_Format(PyExc_SystemError, "path buffer too short"); return NULL; } - return PyUnicode_DecodeFSDefault(buf); + return cpathname; } PyDoc_STRVAR(doc_cache_from_source, -- cgit v1.2.1 From 1bd9d3e0561cad30f87017b5ee00be6f7c92ced8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 13 Mar 2011 21:46:30 -0400 Subject: Issue #3080: load_module() expects name and path as Unicode --- Python/import.c | 136 +++++++++++++++++++++++--------------------------------- 1 file changed, 55 insertions(+), 81 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 0efffe9149..00a9eccaf8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1490,7 +1490,7 @@ unchanged: } /* Forward */ -static PyObject *load_module(char *, FILE *, char *, int, PyObject *); +static PyObject *load_module(PyObject *, FILE *, PyObject *, int, PyObject *); static struct filedescr *find_module(char *, const char *, PyObject *, char *, size_t, FILE **, PyObject **); static struct _frozen * find_frozen(PyObject *); @@ -1501,7 +1501,7 @@ static struct _frozen * find_frozen(PyObject *); static PyObject * load_package(PyObject *name, PyObject *pathname) { - PyObject *m, *d; + PyObject *m, *d, *bufobj; PyObject *file = NULL, *path_list = NULL; int err; char buf[MAXPATHLEN+1]; @@ -1546,7 +1546,13 @@ load_package(PyObject *name, PyObject *pathname) else return NULL; } - m = load_module(namestr, fp, buf, fdp->type, NULL); + bufobj = PyUnicode_DecodeFSDefault(buf); + if (bufobj != NULL) { + m = load_module(name, fp, bufobj, fdp->type, NULL); + Py_DECREF(bufobj); + } + else + m = NULL; if (fp != NULL) fclose(fp); return m; @@ -2303,7 +2309,7 @@ load_builtin(PyObject *name, int type) its module object WITH INCREMENTED REFERENCE COUNT */ static PyObject * -load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) +load_module(PyObject *name, FILE *fp, PyObject *pathname, int type, PyObject *loader) { PyObject *m; @@ -2313,7 +2319,7 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) case PY_COMPILED: if (fp == NULL) { PyErr_Format(PyExc_ValueError, - "file object required for import (type code %d)", + "file object required for import (type code %d)", type); return NULL; } @@ -2321,81 +2327,28 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) switch (type) { - case PY_SOURCE: { - PyObject *nameobj, *pathobj; - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - pathobj = PyUnicode_DecodeFSDefault(pathname); - if (pathobj == NULL) { - Py_DECREF(nameobj); - return NULL; - } - m = load_source_module(nameobj, pathobj, fp); - Py_DECREF(nameobj); - Py_DECREF(pathobj); + case PY_SOURCE: + m = load_source_module(name, pathname, fp); break; - } - case PY_COMPILED: { - PyObject *nameobj, *pathobj; - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - pathobj = PyUnicode_DecodeFSDefault(pathname); - if (pathobj == NULL) { - Py_DECREF(nameobj); - return NULL; - } - m = load_compiled_module(nameobj, pathobj, fp); - Py_DECREF(nameobj); - Py_DECREF(pathobj); + case PY_COMPILED: + m = load_compiled_module(name, pathname, fp); break; - } #ifdef HAVE_DYNAMIC_LOADING - case C_EXTENSION: { - PyObject *nameobj, *pathobj; - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - pathobj = PyUnicode_DecodeFSDefault(pathname); - if (pathobj == NULL) { - Py_DECREF(nameobj); - return NULL; - } - m = _PyImport_LoadDynamicModule(nameobj, pathobj, fp); - Py_DECREF(nameobj); - Py_DECREF(pathobj); + case C_EXTENSION: + m = _PyImport_LoadDynamicModule(name, pathname, fp); break; - } #endif - case PKG_DIRECTORY: { - PyObject *nameobj, *pathobj; - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - pathobj = PyUnicode_DecodeFSDefault(pathname); - if (pathobj == NULL) { - Py_DECREF(nameobj); - return NULL; - } - m = load_package(nameobj, pathobj); - Py_DECREF(nameobj); - Py_DECREF(pathobj); + case PKG_DIRECTORY: + m = load_package(name, pathname); break; - } case C_BUILTIN: - case PY_FROZEN: { - PyObject *nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - m = load_builtin(nameobj, type); - Py_DECREF(nameobj); + case PY_FROZEN: + m = load_builtin(name, type); break; - } case IMP_HOOK: { if (loader == NULL) { @@ -2403,13 +2356,13 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) "import hook without loader"); return NULL; } - m = PyObject_CallMethod(loader, "load_module", "s", name); + m = PyObject_CallMethod(loader, "load_module", "O", name); break; } default: PyErr_Format(PyExc_ImportError, - "Don't know how to import %.200s (type code %d)", + "Don't know how to import %U (type code %d)", name, type); m = NULL; @@ -3144,7 +3097,7 @@ static PyObject * import_submodule(PyObject *mod, char *subname, char *fullname) { PyObject *modules = PyImport_GetModuleDict(); - PyObject *m = NULL; + PyObject *m = NULL, *fullnameobj, *bufobj; /* Require: if mod == None: subname == fullname @@ -3181,7 +3134,19 @@ import_submodule(PyObject *mod, char *subname, char *fullname) Py_INCREF(Py_None); return Py_None; } - m = load_module(fullname, fp, buf, fdp->type, loader); + fullnameobj = PyUnicode_FromString(fullname); + if (fullnameobj != NULL) { + bufobj = PyUnicode_DecodeFSDefault(buf); + if (bufobj != NULL) { + m = load_module(fullnameobj, fp, bufobj, fdp->type, loader); + Py_DECREF(bufobj); + } + else + m = NULL; + Py_DECREF(fullnameobj); + } + else + m = NULL; Py_XDECREF(loader); if (fp) fclose(fp); @@ -3209,7 +3174,7 @@ PyImport_ReloadModule(PyObject *m) char buf[MAXPATHLEN+1]; struct filedescr *fdp; FILE *fp; - PyObject *newm; + PyObject *newm, *nameobj, *bufobj; if (modules_reloading == NULL) { Py_FatalError("PyImport_ReloadModule: " @@ -3275,7 +3240,19 @@ PyImport_ReloadModule(PyObject *m) return NULL; } - newm = load_module(name, fp, buf, fdp->type, loader); + nameobj = PyUnicode_FromString(name); + if (nameobj != NULL) { + bufobj = PyUnicode_DecodeFSDefault(buf); + if (bufobj != NULL) { + newm = load_module(nameobj, fp, bufobj, fdp->type, loader); + Py_DECREF(bufobj); + } + else + newm = NULL; + Py_DECREF(nameobj); + } + else + newm = NULL; Py_XDECREF(loader); if (fp) @@ -3691,18 +3668,15 @@ imp_load_source(PyObject *self, PyObject *args) static PyObject * imp_load_module(PyObject *self, PyObject *args) { - char *name; - PyObject *fob; - PyObject *pathname; - PyObject * ret; + PyObject *name, *fob, *pathname, *ret; char *suffix; /* Unused */ char *mode; int type; FILE *fp; - if (!PyArg_ParseTuple(args, "sOO&(ssi):load_module", + if (!PyArg_ParseTuple(args, "UOO&(ssi):load_module", &name, &fob, - PyUnicode_FSConverter, &pathname, + PyUnicode_FSDecoder, &pathname, &suffix, &mode, &type)) return NULL; if (*mode) { @@ -3726,7 +3700,7 @@ imp_load_module(PyObject *self, PyObject *args) return NULL; } } - ret = load_module(name, fp, PyBytes_AS_STRING(pathname), type, NULL); + ret = load_module(name, fp, pathname, type, NULL); Py_DECREF(pathname); if (fp) fclose(fp); -- cgit v1.2.1 From d5668295d2303564770b7dbdfbb02e7d007292bc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 13 Mar 2011 21:57:27 -0400 Subject: Issue #3080: PyImport_ImportModuleNoBlock() uses Unicode --- Python/import.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 00a9eccaf8..08237aff95 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2591,8 +2591,7 @@ PyImport_ImportModule(const char *name) PyObject * PyImport_ImportModuleNoBlock(const char *name) { - PyObject *result; - PyObject *modules; + PyObject *nameobj, *modules, *result; long me; /* Try to get the module from sys.modules[name] */ @@ -2600,14 +2599,16 @@ PyImport_ImportModuleNoBlock(const char *name) if (modules == NULL) return NULL; - result = PyDict_GetItemString(modules, name); + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + result = PyDict_GetItem(modules, nameobj); if (result != NULL) { + Py_DECREF(nameobj); Py_INCREF(result); return result; } - else { - PyErr_Clear(); - } + PyErr_Clear(); #ifdef WITH_THREAD /* check the import lock * me might be -1 but I ignore the error here, the lock function @@ -2615,18 +2616,20 @@ PyImport_ImportModuleNoBlock(const char *name) me = PyThread_get_thread_ident(); if (import_lock_thread == -1 || import_lock_thread == me) { /* no thread or me is holding the lock */ - return PyImport_ImportModule(name); + result = PyImport_Import(nameobj); } else { PyErr_Format(PyExc_ImportError, - "Failed to import %.200s because the import lock" + "Failed to import %U because the import lock" "is held by another thread.", - name); - return NULL; + nameobj); + result = NULL; } #else - return PyImport_ImportModule(name); + result = PyImport_Import(nameobj); #endif + Py_DECREF(nameobj); + return result; } /* Forward declarations for helper routines */ -- cgit v1.2.1 From 93d6b25bf514369f9325ae1dd062a7ba9f1c1097 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Mar 2011 09:33:57 +0100 Subject: Issue #3080: Use Unicode for the "The Magnum Opus of dotted-name import" Use Unicode for module name and paths in the following functions: * PyImport_ImportModuleLevel() * add_submodule() * ensure_from_list() * get_parent() * import_module_level() * import_submodule() * load_next() * mark_miss() --- Python/import.c | 381 +++++++++++++++++++++++++++++++++----------------------- 1 file changed, 222 insertions(+), 159 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 08237aff95..95faeee5d8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2633,28 +2633,37 @@ PyImport_ImportModuleNoBlock(const char *name) } /* Forward declarations for helper routines */ -static PyObject *get_parent(PyObject *globals, char *buf, - Py_ssize_t *p_buflen, int level); +static PyObject *get_parent(PyObject *globals, + PyObject **p_name, + int level); static PyObject *load_next(PyObject *mod, PyObject *altmod, - char **p_name, char *buf, Py_ssize_t *p_buflen); -static int mark_miss(char *name); + PyObject *inputname, PyObject **p_outputname, + Py_UNICODE *buf, Py_ssize_t *p_buflen, + Py_ssize_t bufsize); +static int mark_miss(PyObject *name); static int ensure_fromlist(PyObject *mod, PyObject *fromlist, - char *buf, Py_ssize_t buflen, int recursive); -static PyObject * import_submodule(PyObject *mod, char *name, char *fullname); + PyObject *buf, int recursive); +static PyObject * import_submodule(PyObject *mod, PyObject *name, + PyObject *fullname); /* The Magnum Opus of dotted-name import :-) */ static PyObject * -import_module_level(char *name, PyObject *globals, PyObject *locals, +import_module_level(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) { - char buf[MAXPATHLEN+1]; - Py_ssize_t buflen = 0; - PyObject *parent, *head, *next, *tail; + Py_UNICODE buf[MAXPATHLEN+1]; + Py_ssize_t buflen; + Py_ssize_t bufsize = MAXPATHLEN+1; + PyObject *parent, *head, *next, *tail, *inputname, *outputname; + PyObject *parent_name, *ensure_name; + const Py_UNICODE *nameunicode; - if (strchr(name, '/') != NULL -#ifdef MS_WINDOWS - || strchr(name, '\\') != NULL + nameunicode = PyUnicode_AS_UNICODE(name); + + if (Py_UNICODE_strchr(nameunicode, SEP) != NULL +#ifdef ALTSEP + || Py_UNICODE_strchr(nameunicode, ALTSEP) != NULL #endif ) { PyErr_SetString(PyExc_ImportError, @@ -2662,25 +2671,45 @@ import_module_level(char *name, PyObject *globals, PyObject *locals, return NULL; } - parent = get_parent(globals, buf, &buflen, level); + parent = get_parent(globals, &parent_name, level); if (parent == NULL) return NULL; - head = load_next(parent, level < 0 ? Py_None : parent, &name, buf, - &buflen); + buflen = PyUnicode_GET_SIZE(parent_name); + if (buflen+1 > bufsize) { + Py_DECREF(parent_name); + PyErr_SetString(PyExc_ValueError, + "Module name too long"); + return NULL; + } + Py_UNICODE_strcpy(buf, PyUnicode_AS_UNICODE(parent_name)); + Py_DECREF(parent_name); + + head = load_next(parent, level < 0 ? Py_None : parent, name, &outputname, + buf, &buflen, bufsize); if (head == NULL) return NULL; tail = head; Py_INCREF(tail); - while (name) { - next = load_next(tail, tail, &name, buf, &buflen); - Py_DECREF(tail); - if (next == NULL) { - Py_DECREF(head); - return NULL; + + if (outputname != NULL) { + while (1) { + inputname = outputname; + next = load_next(tail, tail, inputname, &outputname, + buf, &buflen, bufsize); + Py_DECREF(tail); + Py_DECREF(inputname); + if (next == NULL) { + Py_DECREF(head); + return NULL; + } + tail = next; + + if (outputname == NULL) { + break; + } } - tail = next; } if (tail == Py_None) { /* If tail is Py_None, both get_parent and load_next found @@ -2688,8 +2717,7 @@ import_module_level(char *name, PyObject *globals, PyObject *locals, doctored faulty bytecode */ Py_DECREF(tail); Py_DECREF(head); - PyErr_SetString(PyExc_ValueError, - "Empty module name"); + PyErr_SetString(PyExc_ValueError, "Empty module name"); return NULL; } @@ -2704,21 +2732,33 @@ import_module_level(char *name, PyObject *globals, PyObject *locals, } Py_DECREF(head); - if (!ensure_fromlist(tail, fromlist, buf, buflen, 0)) { + + ensure_name = PyUnicode_FromUnicode(buf, Py_UNICODE_strlen(buf)); + if (ensure_name == NULL) { + Py_DECREF(tail); + return NULL; + } + if (!ensure_fromlist(tail, fromlist, ensure_name, 0)) { Py_DECREF(tail); + Py_DECREF(ensure_name); return NULL; } + Py_DECREF(ensure_name); return tail; } PyObject * PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, - PyObject *fromlist, int level) + PyObject *fromlist, int level) { - PyObject *result; + PyObject *nameobj, *result; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; _PyImport_AcquireLock(); - result = import_module_level(name, globals, locals, fromlist, level); + result = import_module_level(nameobj, globals, locals, fromlist, level); + Py_DECREF(nameobj); if (_PyImport_ReleaseLock() < 0) { Py_XDECREF(result); PyErr_SetString(PyExc_RuntimeError, @@ -2733,15 +2773,20 @@ PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, sys.modules entry for foo.bar. If globals is from a package's __init__.py, the package's entry in sys.modules is returned, as a borrowed reference. - The *name* of the returned package is returned in buf, with the length of - the name in *p_buflen. + The name of the returned package is returned in *p_name. If globals doesn't come from a package or a module in a package, or a corresponding entry is not found in sys.modules, Py_None is returned. */ static PyObject * -get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) +get_parent(PyObject *globals, + PyObject **p_name, + int level) { + Py_UNICODE name[MAXPATHLEN+1]; + const Py_ssize_t bufsize = MAXPATHLEN+1; + PyObject *nameobj; + static PyObject *namestr = NULL; static PyObject *pathstr = NULL; static PyObject *pkgstr = NULL; @@ -2749,7 +2794,7 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) int orig_level = level; if (globals == NULL || !PyDict_Check(globals) || !level) - return Py_None; + goto return_none; if (namestr == NULL) { namestr = PyUnicode_InternFromString("__name__"); @@ -2767,55 +2812,46 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) return NULL; } - *buf = '\0'; - *p_buflen = 0; pkgname = PyDict_GetItem(globals, pkgstr); if ((pkgname != NULL) && (pkgname != Py_None)) { /* __package__ is set, so use it */ - char *pkgname_str; - Py_ssize_t len; - if (!PyUnicode_Check(pkgname)) { PyErr_SetString(PyExc_ValueError, "__package__ set to non-string"); return NULL; } - pkgname_str = _PyUnicode_AsStringAndSize(pkgname, &len); - if (len == 0) { + if (PyUnicode_GET_SIZE(pkgname) == 0) { if (level > 0) { PyErr_SetString(PyExc_ValueError, "Attempted relative import in non-package"); return NULL; } - return Py_None; + goto return_none; } - if (len > MAXPATHLEN) { + if (PyUnicode_GET_SIZE(pkgname)+1 > bufsize) { PyErr_SetString(PyExc_ValueError, "Package name too long"); return NULL; } - strcpy(buf, pkgname_str); + Py_UNICODE_strcpy(name, PyUnicode_AS_UNICODE(pkgname)); } else { /* __package__ not set, so figure it out and set it */ modname = PyDict_GetItem(globals, namestr); if (modname == NULL || !PyUnicode_Check(modname)) - return Py_None; + goto return_none; modpath = PyDict_GetItem(globals, pathstr); if (modpath != NULL) { /* __path__ is set, so modname is already the package name */ - char *modname_str; - Py_ssize_t len; int error; - modname_str = _PyUnicode_AsStringAndSize(modname, &len); - if (len > MAXPATHLEN) { + if (PyUnicode_GET_SIZE(modname)+1 > bufsize) { PyErr_SetString(PyExc_ValueError, "Module name too long"); return NULL; } - strcpy(buf, modname_str); + Py_UNICODE_strcpy(name, PyUnicode_AS_UNICODE(modname)); error = PyDict_SetItem(globals, pkgstr, modname); if (error) { PyErr_SetString(PyExc_ValueError, @@ -2824,9 +2860,9 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) } } else { /* Normal module, so work out the package name if any */ - char *start = _PyUnicode_AsString(modname); - char *lastdot = strrchr(start, '.'); - size_t len; + Py_UNICODE *start = PyUnicode_AS_UNICODE(modname); + Py_UNICODE *lastdot = Py_UNICODE_strrchr(start, '.'); + Py_ssize_t len; int error; if (lastdot == NULL && level > 0) { PyErr_SetString(PyExc_ValueError, @@ -2840,17 +2876,17 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) "Could not set __package__"); return NULL; } - return Py_None; + goto return_none; } len = lastdot - start; - if (len >= MAXPATHLEN) { + if (len+1 > bufsize) { PyErr_SetString(PyExc_ValueError, "Module name too long"); return NULL; } - strncpy(buf, start, len); - buf[len] = '\0'; - pkgname = PyUnicode_FromString(buf); + Py_UNICODE_strncpy(name, start, len); + name[len] = '\0'; + pkgname = PyUnicode_FromUnicode(name, len); if (pkgname == NULL) { return NULL; } @@ -2864,7 +2900,7 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) } } while (--level > 0) { - char *dot = strrchr(buf, '.'); + Py_UNICODE *dot = Py_UNICODE_strrchr(name, '.'); if (dot == NULL) { PyErr_SetString(PyExc_ValueError, "Attempted relative import beyond " @@ -2873,137 +2909,181 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) } *dot = '\0'; } - *p_buflen = strlen(buf); + + nameobj = PyUnicode_FromUnicode(name, Py_UNICODE_strlen(name)); + if (nameobj == NULL) + return NULL; modules = PyImport_GetModuleDict(); - parent = PyDict_GetItemString(modules, buf); + parent = PyDict_GetItem(modules, nameobj); if (parent == NULL) { - if (orig_level < 1) { - PyObject *err_msg = PyBytes_FromFormat( - "Parent module '%.200s' not found " - "while handling absolute import", buf); - if (err_msg == NULL) { - return NULL; - } - if (!PyErr_WarnEx(PyExc_RuntimeWarning, - PyBytes_AsString(err_msg), 1)) { - *buf = '\0'; - *p_buflen = 0; - parent = Py_None; - } - Py_DECREF(err_msg); - } else { + int err; + + if (orig_level >= 1) { PyErr_Format(PyExc_SystemError, - "Parent module '%.200s' not loaded, " - "cannot perform relative import", buf); + "Parent module %R not loaded, " + "cannot perform relative import", nameobj); + Py_DECREF(nameobj); + return NULL; } + + err = PyErr_WarnFormat( + PyExc_RuntimeWarning, 1, + "Parent module %R not found while handling absolute import", + nameobj); + Py_DECREF(nameobj); + if (err) + return NULL; + + goto return_none; } + *p_name = nameobj; return parent; /* We expect, but can't guarantee, if parent != None, that: - - parent.__name__ == buf + - parent.__name__ == name - parent.__dict__ is globals If this is violated... Who cares? */ + +return_none: + nameobj = PyUnicode_FromUnicode(NULL, 0); + if (nameobj == NULL) + return NULL; + *p_name = nameobj; + return Py_None; } /* altmod is either None or same as mod */ static PyObject * -load_next(PyObject *mod, PyObject *altmod, char **p_name, char *buf, - Py_ssize_t *p_buflen) +load_next(PyObject *mod, PyObject *altmod, + PyObject *inputname, PyObject **p_outputname, + Py_UNICODE *buf, Py_ssize_t *p_buflen, Py_ssize_t bufsize) { - char *name = *p_name; - char *dot = strchr(name, '.'); - size_t len; - char *p; - PyObject *result; + const Py_UNICODE *dot; + Py_ssize_t len; + Py_UNICODE *p; + PyObject *fullname, *name, *result, *mark_name; + const Py_UNICODE *nameuni; + + *p_outputname = NULL; - if (strlen(name) == 0) { + if (PyUnicode_GET_SIZE(inputname) == 0) { /* completely empty module name should only happen in 'from . import' (or '__import__("")')*/ Py_INCREF(mod); - *p_name = NULL; return mod; } - if (dot == NULL) { - *p_name = NULL; - len = strlen(name); - } - else { - *p_name = dot+1; - len = dot-name; + nameuni = PyUnicode_AS_UNICODE(inputname); + if (nameuni == NULL) + return NULL; + + dot = Py_UNICODE_strchr(nameuni, '.'); + if (dot != NULL) { + len = dot - nameuni; + if (len == 0) { + PyErr_SetString(PyExc_ValueError, + "Empty module name"); + return NULL; + } } - if (len == 0) { + else + len = PyUnicode_GET_SIZE(inputname); + + if (*p_buflen+len+1 >= bufsize) { PyErr_SetString(PyExc_ValueError, - "Empty module name"); + "Module name too long"); return NULL; } p = buf + *p_buflen; - if (p != buf) + if (p != buf) { *p++ = '.'; - if (p+len-buf >= MAXPATHLEN) { - PyErr_SetString(PyExc_ValueError, - "Module name too long"); - return NULL; + *p_buflen += 1; } - strncpy(p, name, len); + Py_UNICODE_strncpy(p, nameuni, len); p[len] = '\0'; - *p_buflen = p+len-buf; + *p_buflen += len; - result = import_submodule(mod, p, buf); + fullname = PyUnicode_FromUnicode(buf, *p_buflen); + if (fullname == NULL) + return NULL; + name = PyUnicode_FromUnicode(p, len); + if (name == NULL) { + Py_DECREF(fullname); + return NULL; + } + result = import_submodule(mod, name, fullname); + Py_DECREF(fullname); if (result == Py_None && altmod != mod) { Py_DECREF(result); /* Here, altmod must be None and mod must not be None */ - result = import_submodule(altmod, p, p); + result = import_submodule(altmod, name, name); + Py_DECREF(name); if (result != NULL && result != Py_None) { - if (mark_miss(buf) != 0) { + mark_name = PyUnicode_FromUnicode(buf, *p_buflen); + if (mark_name == NULL) { + Py_DECREF(result); + return NULL; + } + if (mark_miss(mark_name) != 0) { Py_DECREF(result); + Py_DECREF(mark_name); return NULL; } - strncpy(buf, name, len); + Py_DECREF(mark_name); + Py_UNICODE_strncpy(buf, nameuni, len); buf[len] = '\0'; *p_buflen = len; } } + else + Py_DECREF(name); if (result == NULL) return NULL; if (result == Py_None) { Py_DECREF(result); PyErr_Format(PyExc_ImportError, - "No module named %.200s", name); + "No module named %R", inputname); return NULL; } + if (dot != NULL) { + *p_outputname = PyUnicode_FromUnicode(dot+1, Py_UNICODE_strlen(dot+1)); + if (*p_outputname == NULL) { + Py_DECREF(result); + return NULL; + } + } + return result; } static int -mark_miss(char *name) +mark_miss(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); - return PyDict_SetItemString(modules, name, Py_None); + return PyDict_SetItem(modules, name, Py_None); } static int -ensure_fromlist(PyObject *mod, PyObject *fromlist, char *buf, Py_ssize_t buflen, +ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *name, int recursive) { int i; + PyObject *fullname; + Py_ssize_t fromlist_len; if (!PyObject_HasAttrString(mod, "__path__")) return 1; - for (i = 0; ; i++) { + fromlist_len = PySequence_Size(fromlist); + + for (i = 0; i < fromlist_len; i++) { PyObject *item = PySequence_GetItem(fromlist, i); int hasit; - if (item == NULL) { - if (PyErr_ExceptionMatches(PyExc_IndexError)) { - PyErr_Clear(); - return 1; - } + if (item == NULL) return 0; - } if (!PyUnicode_Check(item)) { PyErr_SetString(PyExc_TypeError, "Item in ``from list'' not a string"); @@ -3020,7 +3100,7 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, char *buf, Py_ssize_t buflen, if (all == NULL) PyErr_Clear(); else { - int ret = ensure_fromlist(mod, all, buf, buflen, 1); + int ret = ensure_fromlist(mod, all, name, 1); Py_DECREF(all); if (!ret) return 0; @@ -3029,27 +3109,14 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, char *buf, Py_ssize_t buflen, } hasit = PyObject_HasAttr(mod, item); if (!hasit) { - PyObject *item8; - char *subname; PyObject *submod; - char *p; - item8 = PyUnicode_EncodeFSDefault(item); - if (!item8) { - PyErr_SetString(PyExc_ValueError, "Cannot encode path item"); - return 0; + fullname = PyUnicode_FromFormat("%U.%U", name, item); + if (fullname != NULL) { + submod = import_submodule(mod, item, fullname); + Py_DECREF(fullname); } - subname = PyBytes_AS_STRING(item8); - if (buflen + strlen(subname) >= MAXPATHLEN) { - PyErr_SetString(PyExc_ValueError, - "Module name too long"); - Py_DECREF(item); - return 0; - } - p = buf + buflen; - *p++ = '.'; - strcpy(p, subname); - submod = import_submodule(mod, subname, buf); - Py_DECREF(item8); + else + submod = NULL; Py_XDECREF(submod); if (submod == NULL) { Py_DECREF(item); @@ -3059,12 +3126,12 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, char *buf, Py_ssize_t buflen, Py_DECREF(item); } - /* NOTREACHED */ + return 1; } static int -add_submodule(PyObject *mod, PyObject *submod, char *fullname, char *subname, - PyObject *modules) +add_submodule(PyObject *mod, PyObject *submod, PyObject *fullname, + PyObject *subname, PyObject *modules) { if (mod == Py_None) return 1; @@ -3075,7 +3142,7 @@ add_submodule(PyObject *mod, PyObject *submod, char *fullname, char *subname, load failed with a SyntaxError -- then there's no trace in sys.modules. In that case, of course, do nothing extra.) */ if (submod == NULL) { - submod = PyDict_GetItemString(modules, fullname); + submod = PyDict_GetItem(modules, fullname); if (submod == NULL) return 1; } @@ -3086,28 +3153,28 @@ add_submodule(PyObject *mod, PyObject *submod, char *fullname, char *subname, PyObject *dict = PyModule_GetDict(mod); if (!dict) return 0; - if (PyDict_SetItemString(dict, subname, submod) < 0) + if (PyDict_SetItem(dict, subname, submod) < 0) return 0; } else { - if (PyObject_SetAttrString(mod, subname, submod) < 0) + if (PyObject_SetAttr(mod, subname, submod) < 0) return 0; } return 1; } static PyObject * -import_submodule(PyObject *mod, char *subname, char *fullname) +import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) { PyObject *modules = PyImport_GetModuleDict(); - PyObject *m = NULL, *fullnameobj, *bufobj; + PyObject *m = NULL, *bufobj; /* Require: if mod == None: subname == fullname else: mod.__name__ + "." + subname == fullname */ - if ((m = PyDict_GetItemString(modules, fullname)) != NULL) { + if ((m = PyDict_GetItem(modules, fullname)) != NULL) { Py_INCREF(m); } else { @@ -3127,7 +3194,9 @@ import_submodule(PyObject *mod, char *subname, char *fullname) } } - fdp = find_module(fullname, subname, path, buf, MAXPATHLEN+1, + fdp = find_module(_PyUnicode_AsString(fullname), + _PyUnicode_AsString(subname), + path, buf, MAXPATHLEN+1, &fp, &loader); Py_XDECREF(path); if (fdp == NULL) { @@ -3137,23 +3206,17 @@ import_submodule(PyObject *mod, char *subname, char *fullname) Py_INCREF(Py_None); return Py_None; } - fullnameobj = PyUnicode_FromString(fullname); - if (fullnameobj != NULL) { - bufobj = PyUnicode_DecodeFSDefault(buf); - if (bufobj != NULL) { - m = load_module(fullnameobj, fp, bufobj, fdp->type, loader); - Py_DECREF(bufobj); - } - else - m = NULL; - Py_DECREF(fullnameobj); + bufobj = PyUnicode_DecodeFSDefault(buf); + if (bufobj != NULL) { + m = load_module(fullname, fp, bufobj, fdp->type, loader); + Py_DECREF(bufobj); } else m = NULL; Py_XDECREF(loader); if (fp) fclose(fp); - if (!add_submodule(mod, m, fullname, subname, modules)) { + if (m != NULL && !add_submodule(mod, m, fullname, subname, modules)) { Py_XDECREF(m); m = NULL; } -- cgit v1.2.1 From 74c6f0ba2d358a8d6717772150740b5f49142711 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 13 Mar 2011 22:38:06 -0400 Subject: Issue #3080: Use %R to format module name in error messages %R format instead of %U --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 95faeee5d8..90ba4b5e22 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2620,7 +2620,7 @@ PyImport_ImportModuleNoBlock(const char *name) } else { PyErr_Format(PyExc_ImportError, - "Failed to import %U because the import lock" + "Failed to import %R because the import lock" "is held by another thread.", nameobj); result = NULL; @@ -3285,7 +3285,7 @@ PyImport_ReloadModule(PyObject *m) parent = PyDict_GetItem(modules, parentname); if (parent == NULL) { PyErr_Format(PyExc_ImportError, - "reload(): parent %U not in sys.modules", + "reload(): parent %R not in sys.modules", parentname); Py_DECREF(parentname); imp_modules_reloading_clear(); -- cgit v1.2.1 From ec2c770704d651436f9217bf78ef289f63162209 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 13 Mar 2011 22:38:38 -0400 Subject: Issue #3080: Reindent and simplify import_submodule() --- Python/import.c | 81 +++++++++++++++++++++++++++------------------------------ 1 file changed, 39 insertions(+), 42 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 90ba4b5e22..cce2de1b44 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2779,9 +2779,7 @@ PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, corresponding entry is not found in sys.modules, Py_None is returned. */ static PyObject * -get_parent(PyObject *globals, - PyObject **p_name, - int level) +get_parent(PyObject *globals, PyObject **p_name, int level) { Py_UNICODE name[MAXPATHLEN+1]; const Py_ssize_t bufsize = MAXPATHLEN+1; @@ -3167,7 +3165,10 @@ static PyObject * import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) { PyObject *modules = PyImport_GetModuleDict(); - PyObject *m = NULL, *bufobj; + PyObject *m = NULL, *bufobj, *path, *loader; + char buf[MAXPATHLEN+1]; + struct filedescr *fdp; + FILE *fp; /* Require: if mod == None: subname == fullname @@ -3176,52 +3177,48 @@ import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) if ((m = PyDict_GetItem(modules, fullname)) != NULL) { Py_INCREF(m); + return m; } - else { - PyObject *path, *loader; - char buf[MAXPATHLEN+1]; - struct filedescr *fdp; - FILE *fp; - - if (mod == Py_None) - path = NULL; - else { - path = PyObject_GetAttrString(mod, "__path__"); - if (path == NULL) { - PyErr_Clear(); - Py_INCREF(Py_None); - return Py_None; - } - } - fdp = find_module(_PyUnicode_AsString(fullname), - _PyUnicode_AsString(subname), - path, buf, MAXPATHLEN+1, - &fp, &loader); - Py_XDECREF(path); - if (fdp == NULL) { - if (!PyErr_ExceptionMatches(PyExc_ImportError)) - return NULL; + if (mod == Py_None) + path = NULL; + else { + path = PyObject_GetAttrString(mod, "__path__"); + if (path == NULL) { PyErr_Clear(); Py_INCREF(Py_None); return Py_None; } - bufobj = PyUnicode_DecodeFSDefault(buf); - if (bufobj != NULL) { - m = load_module(fullname, fp, bufobj, fdp->type, loader); - Py_DECREF(bufobj); - } - else - m = NULL; - Py_XDECREF(loader); - if (fp) - fclose(fp); - if (m != NULL && !add_submodule(mod, m, fullname, subname, modules)) { - Py_XDECREF(m); - m = NULL; - } } + fdp = find_module(_PyUnicode_AsString(fullname), + _PyUnicode_AsString(subname), + path, buf, MAXPATHLEN+1, + &fp, &loader); + Py_XDECREF(path); + if (fdp == NULL) { + if (!PyErr_ExceptionMatches(PyExc_ImportError)) + return NULL; + PyErr_Clear(); + Py_INCREF(Py_None); + return Py_None; + } + bufobj = PyUnicode_DecodeFSDefault(buf); + if (bufobj != NULL) { + m = load_module(fullname, fp, bufobj, fdp->type, loader); + Py_DECREF(bufobj); + } + else + m = NULL; + Py_XDECREF(loader); + if (fp) + fclose(fp); + if (m == NULL) + return NULL; + if (!add_submodule(mod, m, fullname, subname, modules)) { + Py_XDECREF(m); + return NULL; + } return m; } -- cgit v1.2.1 From 7b12e183cf51fb59b0d62f26e37f91ec881a0dd9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 13 Mar 2011 23:11:02 -0400 Subject: Issue #3080: Drop OS/2 support for the import machinery Sorry Andrew I MacIntyre! --- Python/import.c | 52 ---------------------------------------------------- 1 file changed, 52 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index cce2de1b44..e6d7c060fe 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1804,11 +1804,6 @@ find_module_path_list(char *fullname, const char *name, struct filedescr *fdp = NULL; char *filemode; FILE *fp = NULL; -#if defined(PYOS_OS2) - size_t saved_len; - size_t saved_namelen; - char *saved_buf = NULL; -#endif npath = PyList_Size(search_path_list); namelen = strlen(name); @@ -1832,40 +1827,7 @@ find_module_path_list(char *fullname, const char *name, return fdp; len = strlen(buf); -#if defined(PYOS_OS2) - /* take a snapshot of the module spec for restoration - * after the 8 character DLL hackery - */ - saved_buf = strdup(buf); - saved_len = len; - saved_namelen = namelen; -#endif /* PYOS_OS2 */ for (fdp = _PyImport_Filetab; fdp->suffix != NULL; fdp++) { -#if defined(PYOS_OS2) && defined(HAVE_DYNAMIC_LOADING) - /* OS/2 limits DLLs to 8 character names (w/o - extension) - * so if the name is longer than that and its a - * dynamically loaded module we're going to try, - * truncate the name before trying - */ - if (strlen(name) > 8) { - /* is this an attempt to load a C extension? */ - const struct filedescr *scan; - scan = _PyImport_DynLoadFiletab; - while (scan->suffix != NULL) { - if (!strcmp(scan->suffix, fdp->suffix)) - break; - else - scan++; - } - if (scan->suffix != NULL) { - /* yes, so truncate the name */ - namelen = 8; - len -= strlen(name) - namelen; - buf[len] = '\0'; - } - } -#endif /* PYOS_OS2 */ strcpy(buf+len, fdp->suffix); if (Py_VerboseFlag > 1) PySys_WriteStderr("# trying %s\n", buf); @@ -1881,21 +1843,7 @@ find_module_path_list(char *fullname, const char *name, fp = NULL; } } -#if defined(PYOS_OS2) - /* restore the saved snapshot */ - strcpy(buf, saved_buf); - len = saved_len; - namelen = saved_namelen; -#endif - } -#if defined(PYOS_OS2) - /* don't need/want the module name snapshot anymore */ - if (saved_buf) - { - free(saved_buf); - saved_buf = NULL; } -#endif if (fp != NULL) break; } -- cgit v1.2.1 From e7fd2eaecbfdc259fc9ae199608d33d721d88f57 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 09:21:33 -0400 Subject: Issue #3080: find_module() expects module fullname and subname as Unicode And PyImport_ReloadModule() uses Unicode for the module name. --- Python/import.c | 170 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 84 insertions(+), 86 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index e6d7c060fe..a3f263146c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1491,7 +1491,7 @@ unchanged: /* Forward */ static PyObject *load_module(PyObject *, FILE *, PyObject *, int, PyObject *); -static struct filedescr *find_module(char *, const char *, PyObject *, +static struct filedescr *find_module(PyObject *, PyObject *, PyObject *, char *, size_t, FILE **, PyObject **); static struct _frozen * find_frozen(PyObject *); @@ -1507,7 +1507,13 @@ load_package(PyObject *name, PyObject *pathname) char buf[MAXPATHLEN+1]; FILE *fp = NULL; struct filedescr *fdp; - char *namestr; + static PyObject *initstr = NULL; + + if (initstr == NULL) { + initstr = PyUnicode_InternFromString("__init__"); + if (initstr == NULL) + return NULL; + } m = PyImport_AddModuleObject(name); if (m == NULL) @@ -1532,10 +1538,7 @@ load_package(PyObject *name, PyObject *pathname) Py_DECREF(path_list); return NULL; } - namestr = _PyUnicode_AsString(name); - if (namestr == NULL) - goto error; - fdp = find_module(namestr, "__init__", path_list, buf, sizeof(buf), &fp, NULL); + fdp = find_module(name, initstr, path_list, buf, sizeof(buf), &fp, NULL); Py_DECREF(path_list); if (fdp == NULL) { if (PyErr_ExceptionMatches(PyExc_ImportError)) { @@ -1680,7 +1683,7 @@ static struct filedescr importhookdescr = {"", "", IMP_HOOK}; and *buf is the path */ static int -find_module_path(char *fullname, const char *name, PyObject *path, +find_module_path(PyObject *fullname, PyObject *name, PyObject *path, PyObject *path_hooks, PyObject *path_importer_cache, char *buf, size_t buflen, PyObject **p_loader, struct filedescr **p_fd) @@ -1691,6 +1694,7 @@ find_module_path(char *fullname, const char *name, PyObject *path, size_t namelen; struct stat statbuf; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; + char *namestr; if (PyUnicode_Check(path)) { path_bytes = PyUnicode_EncodeFSDefault(path); @@ -1704,7 +1708,8 @@ find_module_path(char *fullname, const char *name, PyObject *path, else return 0; - namelen = strlen(name); + namestr = _PyUnicode_AsString(name); + namelen = strlen(namestr); base = PyBytes_AS_STRING(path_bytes); len = PyBytes_GET_SIZE(path_bytes); if (len + 2 + namelen + MAXSUFFIXSIZE >= buflen) { @@ -1731,8 +1736,7 @@ find_module_path(char *fullname, const char *name, PyObject *path, if (importer != Py_None) { PyObject *loader; loader = PyObject_CallMethod(importer, - "find_module", - "s", fullname); + "find_module", "O", fullname); if (loader == NULL) return -1; /* error */ if (loader != Py_None) { @@ -1753,7 +1757,7 @@ find_module_path(char *fullname, const char *name, PyObject *path, #endif ) buf[len++] = SEP; - strcpy(buf+len, name); + strcpy(buf+len, namestr); len += namelen; /* Check for package import (buf holds a directory name, @@ -1761,7 +1765,7 @@ find_module_path(char *fullname, const char *name, PyObject *path, #ifdef HAVE_STAT if (stat(buf, &statbuf) == 0 && /* it exists */ S_ISDIR(statbuf.st_mode) && /* it's a directory */ - case_ok(buf, len, namelen, name)) { /* case matches */ + case_ok(buf, len, namelen, namestr)) { /* case matches */ if (find_init_module(buf)) { /* and has __init__.py */ *p_fd = &fd_package; return 2; @@ -1784,7 +1788,7 @@ find_module_path(char *fullname, const char *name, PyObject *path, } /* Find a module in search_path_list. For each path, try - find_module_filename() or try each _PyImport_Filetab suffix. + find_module_path() or try each _PyImport_Filetab suffix. If the module is found, return a file descriptor, write the path in *p_filename, write the pointer to the file object into *p_fp, and (if @@ -1793,7 +1797,7 @@ find_module_path(char *fullname, const char *name, PyObject *path, Otherwise, raise an exception and return NULL. */ static struct filedescr* -find_module_path_list(char *fullname, const char *name, +find_module_path_list(PyObject *fullname, PyObject *name, PyObject *search_path_list, PyObject *path_hooks, PyObject *path_importer_cache, char *buf, size_t buflen, @@ -1804,9 +1808,11 @@ find_module_path_list(char *fullname, const char *name, struct filedescr *fdp = NULL; char *filemode; FILE *fp = NULL; + char *namestr; npath = PyList_Size(search_path_list); - namelen = strlen(name); + namestr = _PyUnicode_AsString(name); + namelen = strlen(namestr); for (i = 0; i < npath; i++) { PyObject *path; int ok; @@ -1836,7 +1842,7 @@ find_module_path_list(char *fullname, const char *name, filemode = "r" PY_STDIOTEXTMODE; fp = fopen(buf, filemode); if (fp != NULL) { - if (case_ok(buf, len, namelen, name)) + if (case_ok(buf, len, namelen, namestr)) break; else { /* continue search */ fclose(fp); @@ -1849,7 +1855,7 @@ find_module_path_list(char *fullname, const char *name, } if (fp == NULL) { PyErr_Format(PyExc_ImportError, - "No module named %.200s", name); + "No module named %U", name); return NULL; } *p_fp = fp; @@ -1882,21 +1888,20 @@ find_module_path_list(char *fullname, const char *name, set) are set to NULL. Eg. *buf is an empty string for a builtin package. */ static struct filedescr * -find_module(char *fullname, const char *name, PyObject *search_path_list, +find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, char *buf, size_t buflen, FILE **p_fp, PyObject **p_loader) { Py_ssize_t i, npath; static struct filedescr fd_frozen = {"", "", PY_FROZEN}; static struct filedescr fd_builtin = {"", "", C_BUILTIN}; PyObject *path_hooks, *path_importer_cache; - PyObject *fullname_obj, *nameobj; *buf = '\0'; *p_fp = NULL; if (p_loader != NULL) *p_loader = NULL; - if (strlen(name) > MAXPATHLEN) { + if (PyUnicode_GET_SIZE(name) > MAXPATHLEN) { PyErr_SetString(PyExc_OverflowError, "module name is too long"); return NULL; @@ -1919,7 +1924,7 @@ find_module(char *fullname, const char *name, PyObject *search_path_list, PyObject *loader; PyObject *hook = PyList_GetItem(meta_path, i); loader = PyObject_CallMethod(hook, "find_module", - "sO", fullname, + "OO", fullname, search_path_list != NULL ? search_path_list : Py_None); if (loader == NULL) { @@ -1937,16 +1942,9 @@ find_module(char *fullname, const char *name, PyObject *search_path_list, Py_DECREF(meta_path); } - if (fullname != NULL) { - fullname_obj = PyUnicode_FromString(fullname); - if (fullname == NULL) - return NULL; - if (find_frozen(fullname_obj) != NULL) { - Py_DECREF(fullname_obj); - strcpy(buf, fullname); - return &fd_frozen; - } - Py_DECREF(fullname_obj); + if (find_frozen(fullname) != NULL) { + strcpy(buf, _PyUnicode_AsString(fullname)); + return &fd_frozen; } if (search_path_list == NULL) { @@ -1955,18 +1953,13 @@ find_module(char *fullname, const char *name, PyObject *search_path_list, struct filedescr *fdp; PyObject *filename, *filename_bytes; #endif - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - if (is_builtin(nameobj)) { - Py_DECREF(nameobj); - strcpy(buf, name); + if (is_builtin(name)) { + strcpy(buf, _PyUnicode_AsString(name)); return &fd_builtin; } #ifdef MS_COREDLL - fp = _PyWin_FindRegisteredModule(nameobj, &fdp, &filename); + fp = _PyWin_FindRegisteredModule(name, &fdp, &filename); if (fp != NULL) { - Py_DECREF(nameobj); filename_bytes = PyUnicode_EncodeFSDefault(filename); Py_DECREF(filename); if (filename_bytes == NULL) @@ -1980,7 +1973,6 @@ find_module(char *fullname, const char *name, PyObject *search_path_list, else if (PyErr_Occurred()) return NULL; #endif - Py_DECREF(nameobj); search_path_list = PySys_GetObject("path"); } @@ -3139,8 +3131,8 @@ import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) } } - fdp = find_module(_PyUnicode_AsString(fullname), - _PyUnicode_AsString(subname), + fdp = find_module(fullname, + subname, path, buf, MAXPATHLEN+1, &fp, &loader); Py_XDECREF(path); @@ -3180,12 +3172,13 @@ PyImport_ReloadModule(PyObject *m) PyInterpreterState *interp = PyThreadState_Get()->interp; PyObject *modules_reloading = interp->modules_reloading; PyObject *modules = PyImport_GetModuleDict(); - PyObject *path = NULL, *loader, *existing_m = NULL; - char *name, *subname; char buf[MAXPATHLEN+1]; + PyObject *path = NULL, *loader = NULL, *existing_m = NULL; + PyObject *nameobj, *bufobj, *subnameobj; + Py_UNICODE *name, *subname; struct filedescr *fdp; - FILE *fp; - PyObject *newm, *nameobj, *bufobj; + FILE *fp = NULL; + PyObject *newm = NULL; if (modules_reloading == NULL) { Py_FatalError("PyImport_ReloadModule: " @@ -3198,69 +3191,75 @@ PyImport_ReloadModule(PyObject *m) "reload() argument must be module"); return NULL; } - name = (char*)PyModule_GetName(m); - if (name == NULL) + nameobj = PyModule_GetNameObject(m); + if (nameobj == NULL) return NULL; - if (m != PyDict_GetItemString(modules, name)) { + if (m != PyDict_GetItem(modules, nameobj)) { PyErr_Format(PyExc_ImportError, - "reload(): module %.200s not in sys.modules", - name); + "reload(): module %U not in sys.modules", + nameobj); + Py_DECREF(nameobj); return NULL; } - existing_m = PyDict_GetItemString(modules_reloading, name); + existing_m = PyDict_GetItem(modules_reloading, nameobj); if (existing_m != NULL) { /* Due to a recursive reload, this module is already being reloaded. */ + Py_DECREF(nameobj); Py_INCREF(existing_m); return existing_m; } - if (PyDict_SetItemString(modules_reloading, name, m) < 0) + if (PyDict_SetItem(modules_reloading, nameobj, m) < 0) { + Py_DECREF(nameobj); return NULL; + } - subname = strrchr(name, '.'); - if (subname == NULL) - subname = name; + name = PyUnicode_AS_UNICODE(nameobj); + subname = Py_UNICODE_strrchr(name, '.'); + if (subname == NULL) { + Py_INCREF(nameobj); + subnameobj = nameobj; + } else { PyObject *parentname, *parent; - parentname = PyUnicode_FromStringAndSize(name, (subname-name)); + Py_ssize_t len; + len = subname - name; + parentname = PyUnicode_FromUnicode(name, len); if (parentname == NULL) { - imp_modules_reloading_clear(); - return NULL; + goto error; } parent = PyDict_GetItem(modules, parentname); if (parent == NULL) { PyErr_Format(PyExc_ImportError, - "reload(): parent %R not in sys.modules", + "reload(): parent %U not in sys.modules", parentname); Py_DECREF(parentname); - imp_modules_reloading_clear(); - return NULL; + goto error; } Py_DECREF(parentname); - subname++; path = PyObject_GetAttrString(parent, "__path__"); if (path == NULL) PyErr_Clear(); + subname++; + len = PyUnicode_GET_SIZE(nameobj) - (len + 1); + subnameobj = PyUnicode_FromUnicode(subname, len); } - fdp = find_module(name, subname, path, buf, MAXPATHLEN+1, &fp, &loader); + if (subnameobj == NULL) + goto error; + fdp = find_module(nameobj, subnameobj, + path, buf, MAXPATHLEN+1, &fp, &loader); + Py_DECREF(subnameobj); Py_XDECREF(path); if (fdp == NULL) { Py_XDECREF(loader); - imp_modules_reloading_clear(); - return NULL; + goto error; } - nameobj = PyUnicode_FromString(name); - if (nameobj != NULL) { - bufobj = PyUnicode_DecodeFSDefault(buf); - if (bufobj != NULL) { - newm = load_module(nameobj, fp, bufobj, fdp->type, loader); - Py_DECREF(bufobj); - } - else - newm = NULL; - Py_DECREF(nameobj); + bufobj = PyUnicode_DecodeFSDefault(buf); + if (bufobj != NULL) { + newm = load_module(nameobj, fp, bufobj, fdp->type, loader); + Py_DECREF(bufobj); } else newm = NULL; @@ -3274,9 +3273,12 @@ PyImport_ReloadModule(PyObject *m) * going to return NULL in this case regardless of whether * replacing name succeeds, so the return value is ignored. */ - PyDict_SetItemString(modules, name, m); + PyDict_SetItem(modules, nameobj, m); } + +error: imp_modules_reloading_clear(); + Py_DECREF(nameobj); return newm; } @@ -3424,7 +3426,7 @@ imp_get_suffixes(PyObject *self, PyObject *noargs) } static PyObject * -call_find_module(char *name, PyObject *path) +call_find_module(PyObject *name, PyObject *path) { extern int fclose(FILE *); PyObject *fob, *ret; @@ -3483,15 +3485,11 @@ call_find_module(char *name, PyObject *path) static PyObject * imp_find_module(PyObject *self, PyObject *args) { - PyObject *name; - PyObject *ret, *path = NULL; - if (!PyArg_ParseTuple(args, "O&|O:find_module", - PyUnicode_FSConverter, &name, - &path)) + PyObject *name, *path_list = NULL; + if (!PyArg_ParseTuple(args, "U|O:find_module", + &name, &path_list)) return NULL; - ret = call_find_module(PyBytes_AS_STRING(name), path); - Py_DECREF(name); - return ret; + return call_find_module(name, path_list); } static PyObject * -- cgit v1.2.1 From c23cd9690f4b004e965101735800e7c352fcfc78 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 13:22:54 -0400 Subject: Issue #3080: Rename some path variables to path_list --- Python/import.c | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index a3f263146c..73d38fe10f 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1538,7 +1538,8 @@ load_package(PyObject *name, PyObject *pathname) Py_DECREF(path_list); return NULL; } - fdp = find_module(name, initstr, path_list, buf, sizeof(buf), &fp, NULL); + fdp = find_module(name, initstr, path_list, + buf, sizeof(buf), &fp, NULL); Py_DECREF(path_list); if (fdp == NULL) { if (PyErr_ExceptionMatches(PyExc_ImportError)) { @@ -3105,7 +3106,7 @@ static PyObject * import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) { PyObject *modules = PyImport_GetModuleDict(); - PyObject *m = NULL, *bufobj, *path, *loader; + PyObject *m = NULL, *bufobj, *path_list, *loader; char buf[MAXPATHLEN+1]; struct filedescr *fdp; FILE *fp; @@ -3121,21 +3122,19 @@ import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) } if (mod == Py_None) - path = NULL; + path_list = NULL; else { - path = PyObject_GetAttrString(mod, "__path__"); - if (path == NULL) { + path_list = PyObject_GetAttrString(mod, "__path__"); + if (path_list == NULL) { PyErr_Clear(); Py_INCREF(Py_None); return Py_None; } } - fdp = find_module(fullname, - subname, - path, buf, MAXPATHLEN+1, - &fp, &loader); - Py_XDECREF(path); + fdp = find_module(fullname, subname, path_list, + buf, MAXPATHLEN+1, &fp, &loader); + Py_XDECREF(path_list); if (fdp == NULL) { if (!PyErr_ExceptionMatches(PyExc_ImportError)) return NULL; @@ -3173,7 +3172,7 @@ PyImport_ReloadModule(PyObject *m) PyObject *modules_reloading = interp->modules_reloading; PyObject *modules = PyImport_GetModuleDict(); char buf[MAXPATHLEN+1]; - PyObject *path = NULL, *loader = NULL, *existing_m = NULL; + PyObject *path_list = NULL, *loader = NULL, *existing_m = NULL; PyObject *nameobj, *bufobj, *subnameobj; Py_UNICODE *name, *subname; struct filedescr *fdp; @@ -3237,8 +3236,8 @@ PyImport_ReloadModule(PyObject *m) goto error; } Py_DECREF(parentname); - path = PyObject_GetAttrString(parent, "__path__"); - if (path == NULL) + path_list = PyObject_GetAttrString(parent, "__path__"); + if (path_list == NULL) PyErr_Clear(); subname++; len = PyUnicode_GET_SIZE(nameobj) - (len + 1); @@ -3246,10 +3245,10 @@ PyImport_ReloadModule(PyObject *m) } if (subnameobj == NULL) goto error; - fdp = find_module(nameobj, subnameobj, - path, buf, MAXPATHLEN+1, &fp, &loader); + fdp = find_module(nameobj, subnameobj, path_list, + buf, MAXPATHLEN+1, &fp, &loader); Py_DECREF(subnameobj); - Py_XDECREF(path); + Py_XDECREF(path_list); if (fdp == NULL) { Py_XDECREF(loader); @@ -3426,7 +3425,7 @@ imp_get_suffixes(PyObject *self, PyObject *noargs) } static PyObject * -call_find_module(PyObject *name, PyObject *path) +call_find_module(PyObject *name, PyObject *path_list) { extern int fclose(FILE *); PyObject *fob, *ret; @@ -3439,9 +3438,10 @@ call_find_module(PyObject *name, PyObject *path) char *encoding = NULL; pathname[0] = '\0'; - if (path == Py_None) - path = NULL; - fdp = find_module(NULL, name, path, pathname, MAXPATHLEN+1, &fp, NULL); + if (path_list == Py_None) + path_list = NULL; + fdp = find_module(NULL, name, path_list, + pathname, MAXPATHLEN+1, &fp, NULL); if (fdp == NULL) return NULL; if (fp != NULL) { -- cgit v1.2.1 From d5658be50acead958aef78e5c41c8d4d4bd0e857 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 13:40:04 -0400 Subject: Issue #3080: find_module() sets an empty path for builtin and frozen modules --- Python/import.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 73d38fe10f..2c21112c4b 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1943,10 +1943,8 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, Py_DECREF(meta_path); } - if (find_frozen(fullname) != NULL) { - strcpy(buf, _PyUnicode_AsString(fullname)); + if (find_frozen(fullname) != NULL) return &fd_frozen; - } if (search_path_list == NULL) { #ifdef MS_COREDLL @@ -1954,10 +1952,8 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, struct filedescr *fdp; PyObject *filename, *filename_bytes; #endif - if (is_builtin(name)) { - strcpy(buf, _PyUnicode_AsString(name)); + if (is_builtin(name)) return &fd_builtin; - } #ifdef MS_COREDLL fp = _PyWin_FindRegisteredModule(name, &fdp, &filename); if (fp != NULL) { -- cgit v1.2.1 From 2c6f2066619d885ebc3fd2c23dd6ca17f18ba71c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 14:04:10 -0400 Subject: Issue #3080: Refactor find_module_path(), use return instead of break Prepare also the API change of case_ok() --- Python/import.c | 77 +++++++++++++++++++++++++++------------------------------ 1 file changed, 37 insertions(+), 40 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 2c21112c4b..979058e583 100644 --- a/Python/import.c +++ b/Python/import.c @@ -144,6 +144,8 @@ static const struct filedescr _PyImport_StandardFiletab[] = { {0, 0} }; +static PyObject *initstr = NULL; + /* Initialize things */ @@ -155,6 +157,10 @@ _PyImport_Init(void) int countD = 0; int countS = 0; + initstr = PyUnicode_InternFromString("__init__"); + if (initstr == NULL) + Py_FatalError("Can't initialize import variables"); + /* prepare _PyImport_Filetab: copy entries from _PyImport_DynLoadFiletab and _PyImport_StandardFiletab. */ @@ -1507,13 +1513,6 @@ load_package(PyObject *name, PyObject *pathname) char buf[MAXPATHLEN+1]; FILE *fp = NULL; struct filedescr *fdp; - static PyObject *initstr = NULL; - - if (initstr == NULL) { - initstr = PyUnicode_InternFromString("__init__"); - if (initstr == NULL) - return NULL; - } m = PyImport_AddModuleObject(name); if (m == NULL) @@ -1765,23 +1764,25 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, and there's an __init__ module in that directory */ #ifdef HAVE_STAT if (stat(buf, &statbuf) == 0 && /* it exists */ - S_ISDIR(statbuf.st_mode) && /* it's a directory */ - case_ok(buf, len, namelen, namestr)) { /* case matches */ - if (find_init_module(buf)) { /* and has __init__.py */ - *p_fd = &fd_package; - return 2; - } - else { - int err; - PyObject *unicode = PyUnicode_DecodeFSDefault(buf); - if (unicode == NULL) - return -1; - err = PyErr_WarnFormat(PyExc_ImportWarning, 1, - "Not importing directory '%U': missing __init__.py", - unicode); - Py_DECREF(unicode); - if (err) - return -1; + S_ISDIR(statbuf.st_mode)) /* it's a directory */ + { + if (case_ok(buf, len, namelen, namestr)) { /* case matches */ + if (find_init_module(buf)) { /* and has __init__.py */ + *p_fd = &fd_package; + return 2; + } + else { + int err; + PyObject *unicode = PyUnicode_DecodeFSDefault(buf); + if (unicode == NULL) + return -1; + err = PyErr_WarnFormat(PyExc_ImportWarning, 1, + "Not importing directory '%U': missing __init__.py", + unicode); + Py_DECREF(unicode); + if (err) + return -1; + } } } #endif @@ -1842,25 +1843,21 @@ find_module_path_list(PyObject *fullname, PyObject *name, if (filemode[0] == 'U') filemode = "r" PY_STDIOTEXTMODE; fp = fopen(buf, filemode); - if (fp != NULL) { - if (case_ok(buf, len, namelen, namestr)) - break; - else { /* continue search */ - fclose(fp); - fp = NULL; - } + if (fp == NULL) + continue; + + if (case_ok(buf, len, namelen, namestr)) { + *p_fp = fp; + return fdp; } + + fclose(fp); + fp = NULL; } - if (fp != NULL) - break; - } - if (fp == NULL) { - PyErr_Format(PyExc_ImportError, - "No module named %U", name); - return NULL; } - *p_fp = fp; - return fdp; + PyErr_Format(PyExc_ImportError, + "No module named %U", name); + return NULL; } /* Find a module: -- cgit v1.2.1 From 87e72d7343dc9ff213470b182fbba2bd73e7a6d1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 20 Mar 2011 03:07:28 +0100 Subject: Issue #3080: find_init_module() expects Unicode --- Python/import.c | 88 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 45 insertions(+), 43 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 979058e583..8a242a57eb 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1668,8 +1668,9 @@ extern FILE *_PyWin_FindRegisteredModule(PyObject *, struct filedescr **, PyObject **p_path); #endif +/* Forward */ static int case_ok(char *, Py_ssize_t, Py_ssize_t, const char *); -static int find_init_module(char *); /* Forward */ +static int find_init_module(PyObject *); static struct filedescr importhookdescr = {"", "", IMP_HOOK}; /* Get the path of a module: get its importer and call importer.find_module() @@ -1766,24 +1767,27 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, if (stat(buf, &statbuf) == 0 && /* it exists */ S_ISDIR(statbuf.st_mode)) /* it's a directory */ { + PyObject *bufobj = PyUnicode_DecodeFSDefault(buf); + if (bufobj == NULL) + return -1; if (case_ok(buf, len, namelen, namestr)) { /* case matches */ - if (find_init_module(buf)) { /* and has __init__.py */ + if (find_init_module(bufobj)) { /* and has __init__.py */ + Py_DECREF(bufobj); *p_fd = &fd_package; return 2; } else { int err; - PyObject *unicode = PyUnicode_DecodeFSDefault(buf); - if (unicode == NULL) - return -1; err = PyErr_WarnFormat(PyExc_ImportWarning, 1, - "Not importing directory '%U': missing __init__.py", - unicode); - Py_DECREF(unicode); - if (err) + "Not importing directory %R: missing __init__.py", + bufobj); + if (err) { + Py_DECREF(bufobj); return -1; + } } } + Py_DECREF(bufobj); } #endif return 1; @@ -2154,49 +2158,47 @@ case_ok(char *buf, Py_ssize_t len, Py_ssize_t namelen, const char *name) #ifdef HAVE_STAT -/* Helper to look for __init__.py or __init__.py[co] in potential package */ +/* Helper to look for __init__.py or __init__.py[co] in potential package. + Return 1 if __init__ was found, 0 if not, or -1 on error. */ static int -find_init_module(char *buf) +find_init_module(PyObject *directory) { - const size_t save_len = strlen(buf); - size_t i = save_len; - char *pname; /* pointer to start of __init__ */ + size_t len; struct stat statbuf; - -/* For calling case_ok(buf, len, namelen, name): - * /a/b/c/d/e/f/g/h/i/j/k/some_long_module_name.py\0 - * ^ ^ ^ ^ - * |--------------------- buf ---------------------| - * |------------------- len ------------------| - * |------ name -------| - * |----- namelen -----| - */ - if (save_len + 13 >= MAXPATHLEN) - return 0; - buf[i++] = SEP; - pname = buf + i; - strcpy(pname, "__init__.py"); - if (stat(buf, &statbuf) == 0) { - if (case_ok(buf, - save_len + 9, /* len("/__init__") */ - 8, /* len("__init__") */ - pname)) { - buf[save_len] = '\0'; + PyObject *filename; + int match; + char *filestr; + size_t filelen; + + len = PyUnicode_GET_SIZE(directory); + filename = PyUnicode_FromFormat("%U%c__init__.py", directory, SEP); + if (filename == NULL) + return -1; + if (_Py_stat(filename, &statbuf) == 0) { + /* 9=len("/__init__") */ + filestr = _PyUnicode_AsString(filename); + filelen = strlen(filestr); + if (case_ok(filestr, filelen-9, 8, "__init__")) { + Py_DECREF(filename); return 1; } } - i += strlen(pname); - strcpy(buf+i, Py_OptimizeFlag ? "o" : "c"); - if (stat(buf, &statbuf) == 0) { - if (case_ok(buf, - save_len + 9, /* len("/__init__") */ - 8, /* len("__init__") */ - pname)) { - buf[save_len] = '\0'; + Py_DECREF(filename); + + filename = PyUnicode_FromFormat("%U%c__init__.py%c", + directory, SEP, Py_OptimizeFlag ? 'o' : 'c'); + if (filename == NULL) + return -1; + if (_Py_stat(filename, &statbuf) == 0) { + /* 9=len("/__init__") */ + filestr = _PyUnicode_AsString(filename); + filelen = strlen(filestr); + if (case_ok(filestr, filelen-9, 8, "__init__")) { + Py_DECREF(filename); return 1; } } - buf[save_len] = '\0'; + Py_DECREF(filename); return 0; } -- cgit v1.2.1 From 7bca4dba35971a2705590aee419a18ad29fefd06 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 14:34:13 -0400 Subject: Issue #3080: case_ok() expects Unicode strings --- Python/import.c | 181 +++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 132 insertions(+), 49 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 8a242a57eb..bba0120321 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1669,7 +1669,7 @@ extern FILE *_PyWin_FindRegisteredModule(PyObject *, struct filedescr **, #endif /* Forward */ -static int case_ok(char *, Py_ssize_t, Py_ssize_t, const char *); +static int case_ok(PyObject *, Py_ssize_t, PyObject *); static int find_init_module(PyObject *); static struct filedescr importhookdescr = {"", "", IMP_HOOK}; @@ -1767,12 +1767,20 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, if (stat(buf, &statbuf) == 0 && /* it exists */ S_ISDIR(statbuf.st_mode)) /* it's a directory */ { - PyObject *bufobj = PyUnicode_DecodeFSDefault(buf); - if (bufobj == NULL) + int match; + PyObject *filename; + + filename = PyUnicode_DecodeFSDefault(buf); + if (filename == NULL) + return -1; + match = case_ok(filename, 0, name); + if (match < 0) { + Py_DECREF(filename); return -1; - if (case_ok(buf, len, namelen, namestr)) { /* case matches */ - if (find_init_module(bufobj)) { /* and has __init__.py */ - Py_DECREF(bufobj); + } + if (match) { /* case matches */ + if (find_init_module(filename)) { /* and has __init__.py */ + Py_DECREF(filename); *p_fd = &fd_package; return 2; } @@ -1780,14 +1788,14 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, int err; err = PyErr_WarnFormat(PyExc_ImportWarning, 1, "Not importing directory %R: missing __init__.py", - bufobj); + filename); if (err) { - Py_DECREF(bufobj); + Py_DECREF(filename); return -1; } } } - Py_DECREF(bufobj); + Py_DECREF(filename); } #endif return 1; @@ -1815,6 +1823,8 @@ find_module_path_list(PyObject *fullname, PyObject *name, char *filemode; FILE *fp = NULL; char *namestr; + PyObject *filename; + int match; npath = PyList_Size(search_path_list); namestr = _PyUnicode_AsString(name); @@ -1840,20 +1850,34 @@ find_module_path_list(PyObject *fullname, PyObject *name, len = strlen(buf); for (fdp = _PyImport_Filetab; fdp->suffix != NULL; fdp++) { - strcpy(buf+len, fdp->suffix); - if (Py_VerboseFlag > 1) - PySys_WriteStderr("# trying %s\n", buf); filemode = fdp->mode; if (filemode[0] == 'U') filemode = "r" PY_STDIOTEXTMODE; - fp = fopen(buf, filemode); - if (fp == NULL) - continue; - if (case_ok(buf, len, namelen, namestr)) { + strcpy(buf+len, fdp->suffix); + filename = PyUnicode_DecodeFSDefault(buf); + if (filename == NULL) + return NULL; + + if (Py_VerboseFlag > 1) + PySys_FormatStderr("# trying %R\n", filename); + + fp = _Py_fopen(filename, filemode); + if (fp == NULL) { + Py_DECREF(filename); + continue; + } + match = case_ok(filename, -(Py_ssize_t)strlen(fdp->suffix), name); + if (match < 0) { + Py_DECREF(filename); + return NULL; + } + if (match) { + Py_DECREF(filename); *p_fp = fp; return fdp; } + Py_DECREF(filename); fclose(fp); fp = NULL; @@ -2002,7 +2026,7 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, p_fp, p_loader); } -/* case_ok(char* buf, Py_ssize_t len, Py_ssize_t namelen, char* name) +/* case_bytes(char* buf, Py_ssize_t len, Py_ssize_t namelen, char* name) * The arguments here are tricky, best shown by example: * /a/b/c/d/e/f/g/h/i/j/k/some_long_module_name.py\0 * ^ ^ ^ ^ @@ -2016,18 +2040,18 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, * We've already done a successful stat() or fopen() on buf, so know that * there's some match, possibly case-insensitive. * - * case_ok() is to return 1 if there's a case-sensitive match for - * name, else 0. case_ok() is also to return 1 if envar PYTHONCASEOK + * case_bytes() is to return 1 if there's a case-sensitive match for + * name, else 0. case_bytes() is also to return 1 if envar PYTHONCASEOK * exists. * - * case_ok() is used to implement case-sensitive import semantics even + * case_bytes() is used to implement case-sensitive import semantics even * on platforms with case-insensitive filesystems. It's trivial to implement * for case-sensitive filesystems. It's pretty much a cross-platform * nightmare for systems with case-insensitive filesystems. */ /* First we may need a pile of platform-specific header files; the sequence - * of #if's here should match the sequence in the body of case_ok(). + * of #if's here should match the sequence in the body of case_bytes(). */ #if defined(MS_WINDOWS) #include @@ -2046,33 +2070,24 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, #include #endif +#if defined(DJGPP) \ + || ((defined(__MACH__) && defined(__APPLE__) || defined(__CYGWIN__)) \ + && defined(HAVE_DIRENT_H)) \ + || defined(PYOS_OS2) +# define USE_CASE_OK_BYTES +#endif + + +#ifdef USE_CASE_OK_BYTES static int -case_ok(char *buf, Py_ssize_t len, Py_ssize_t namelen, const char *name) +case_bytes(char *buf, Py_ssize_t len, Py_ssize_t namelen, const char *name) { /* Pick a platform-specific implementation; the sequence of #if's here should * match the sequence just above. */ -/* MS_WINDOWS */ -#if defined(MS_WINDOWS) - WIN32_FIND_DATA data; - HANDLE h; - - if (Py_GETENV("PYTHONCASEOK") != NULL) - return 1; - - h = FindFirstFile(buf, &data); - if (h == INVALID_HANDLE_VALUE) { - PyErr_Format(PyExc_NameError, - "Can't find file for module %.100s\n(filename %.300s)", - name, buf); - return 0; - } - FindClose(h); - return strncmp(data.cFileName, name, namelen) == 0; - /* DJGPP */ -#elif defined(DJGPP) +#if defined(DJGPP) struct ffblk ffblk; int done; @@ -2151,6 +2166,70 @@ case_ok(char *buf, Py_ssize_t len, Py_ssize_t namelen, const char *name) /* assuming it's a case-sensitive filesystem, so there's nothing to do! */ #else +# error "USE_CASE_OK_BYTES is not correctly defined" +#endif +} +#endif + +/* + * Check if a filename case matchs the name case. We've already done a + * successful stat() or fopen() on buf, so know that there's some match, + * possibly case-insensitive. + * + * case_ok() is to return 1 if there's a case-sensitive match for name, 0 if it + * the filename doesn't match, or -1 on error. case_ok() is also to return 1 + * if envar PYTHONCASEOK exists. + * + * case_ok() is used to implement case-sensitive import semantics even + * on platforms with case-insensitive filesystems. It's trivial to implement + * for case-sensitive filesystems. It's pretty much a cross-platform + * nightmare for systems with case-insensitive filesystems. + */ + +static int +case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) +{ +#ifdef MS_WINDOWS + WIN32_FIND_DATAW data; + HANDLE h; + int cmp; + + if (Py_GETENV("PYTHONCASEOK") != NULL) + return 1; + + h = FindFirstFileW(PyUnicode_AS_UNICODE(filename), &data); + if (h == INVALID_HANDLE_VALUE) { + PyErr_Format(PyExc_NameError, + "Can't find file for module %R\n(filename %R)", + name, filename); + return 0; + } + FindClose(h); + cmp = wcsncmp(data.cFileName, + PyUnicode_AS_UNICODE(name), + PyUnicode_GET_SIZE(name)); + return cmp == 0; +#elif defined(USE_CASE_OK_BYTES) + int match; + PyObject *filebytes, *namebytes; + filebytes = PyUnicode_EncodeFSDefault(filename); + if (filebytes == NULL) + return -1; + namebytes = PyUnicode_EncodeFSDefault(name); + if (namebytes == NULL) { + Py_DECREF(filebytes); + return -1; + } + match = case_bytes( + PyBytes_AS_STRING(filebytes), + PyBytes_GET_SIZE(filebytes) + prefix_delta, + PyBytes_AS_STRING(namebytes), + PyBytes_GET_SIZE(namebytes)); + Py_DECREF(filebytes); + Py_DECREF(namebytes); + return match; +#else + /* assuming it's a case-sensitive filesystem, so there's nothing to do! */ return 1; #endif @@ -2167,8 +2246,6 @@ find_init_module(PyObject *directory) struct stat statbuf; PyObject *filename; int match; - char *filestr; - size_t filelen; len = PyUnicode_GET_SIZE(directory); filename = PyUnicode_FromFormat("%U%c__init__.py", directory, SEP); @@ -2176,9 +2253,12 @@ find_init_module(PyObject *directory) return -1; if (_Py_stat(filename, &statbuf) == 0) { /* 9=len("/__init__") */ - filestr = _PyUnicode_AsString(filename); - filelen = strlen(filestr); - if (case_ok(filestr, filelen-9, 8, "__init__")) { + match = case_ok(filename, 9, initstr); + if (match < 0) { + Py_DECREF(filename); + return -1; + } + if (match) { Py_DECREF(filename); return 1; } @@ -2191,9 +2271,12 @@ find_init_module(PyObject *directory) return -1; if (_Py_stat(filename, &statbuf) == 0) { /* 9=len("/__init__") */ - filestr = _PyUnicode_AsString(filename); - filelen = strlen(filestr); - if (case_ok(filestr, filelen-9, 8, "__init__")) { + match = case_ok(filename, 9, initstr); + if (match < 0) { + Py_DECREF(filename); + return -1; + } + if (match) { Py_DECREF(filename); return 1; } -- cgit v1.2.1 From 02352565abd1eed9eb32c5c273bc640869f2100b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 15:19:39 -0400 Subject: Issue #3080: find_module() returns the path as Unicode --- Python/import.c | 171 ++++++++++++++++++++++++-------------------------------- 1 file changed, 74 insertions(+), 97 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index bba0120321..5dcbf17710 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1498,7 +1498,7 @@ unchanged: /* Forward */ static PyObject *load_module(PyObject *, FILE *, PyObject *, int, PyObject *); static struct filedescr *find_module(PyObject *, PyObject *, PyObject *, - char *, size_t, FILE **, PyObject **); + PyObject **, FILE **, PyObject **); static struct _frozen * find_frozen(PyObject *); /* Load a package and return its module object WITH INCREMENTED @@ -1510,7 +1510,6 @@ load_package(PyObject *name, PyObject *pathname) PyObject *m, *d, *bufobj; PyObject *file = NULL, *path_list = NULL; int err; - char buf[MAXPATHLEN+1]; FILE *fp = NULL; struct filedescr *fdp; @@ -1538,7 +1537,7 @@ load_package(PyObject *name, PyObject *pathname) return NULL; } fdp = find_module(name, initstr, path_list, - buf, sizeof(buf), &fp, NULL); + &bufobj, &fp, NULL); Py_DECREF(path_list); if (fdp == NULL) { if (PyErr_ExceptionMatches(PyExc_ImportError)) { @@ -1549,13 +1548,8 @@ load_package(PyObject *name, PyObject *pathname) else return NULL; } - bufobj = PyUnicode_DecodeFSDefault(buf); - if (bufobj != NULL) { - m = load_module(name, fp, bufobj, fdp->type, NULL); - Py_DECREF(bufobj); - } - else - m = NULL; + m = load_module(name, fp, bufobj, fdp->type, NULL); + Py_XDECREF(bufobj); if (fp != NULL) fclose(fp); return m; @@ -1686,43 +1680,40 @@ static struct filedescr importhookdescr = {"", "", IMP_HOOK}; static int find_module_path(PyObject *fullname, PyObject *name, PyObject *path, PyObject *path_hooks, PyObject *path_importer_cache, - char *buf, size_t buflen, - PyObject **p_loader, struct filedescr **p_fd) + PyObject **p_path, PyObject **p_loader, struct filedescr **p_fd) { - PyObject *path_bytes; - const char *base; + Py_UNICODE buf[MAXPATHLEN+1]; + Py_ssize_t buflen = MAXPATHLEN+1; + PyObject *path_unicode, *filename; + const Py_UNICODE *base; Py_ssize_t len; - size_t namelen; struct stat statbuf; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; - char *namestr; if (PyUnicode_Check(path)) { - path_bytes = PyUnicode_EncodeFSDefault(path); - if (path_bytes == NULL) - return -1; + Py_INCREF(path); + path_unicode = path; } else if (PyBytes_Check(path)) { - Py_INCREF(path); - path_bytes = path; + path_unicode = PyUnicode_DecodeFSDefaultAndSize( + PyBytes_AS_STRING(path), PyBytes_GET_SIZE(path)); + if (path_unicode == NULL) + return -1; } else return 0; - namestr = _PyUnicode_AsString(name); - namelen = strlen(namestr); - base = PyBytes_AS_STRING(path_bytes); - len = PyBytes_GET_SIZE(path_bytes); - if (len + 2 + namelen + MAXSUFFIXSIZE >= buflen) { - Py_DECREF(path_bytes); + base = PyUnicode_AS_UNICODE(path_unicode); + len = PyUnicode_GET_SIZE(path_unicode); + if (len + 2 + PyUnicode_GET_SIZE(name) + MAXSUFFIXSIZE >= buflen) { + Py_DECREF(path_unicode); return 0; /* Too long */ } - strcpy(buf, base); - Py_DECREF(path_bytes); + Py_UNICODE_strcpy(buf, PyUnicode_AS_UNICODE(path_unicode)); + Py_DECREF(path_unicode); - if (strlen(buf) != len) { - return 0; /* path_bytes contains '\0' */ - } + if (Py_UNICODE_strlen(buf) != len) + return 0; /* path contains '\0' */ /* sys.path_hooks import hook */ if (p_loader != NULL) { @@ -1758,21 +1749,21 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, #endif ) buf[len++] = SEP; - strcpy(buf+len, namestr); - len += namelen; + Py_UNICODE_strcpy(buf+len, PyUnicode_AS_UNICODE(name)); + len += PyUnicode_GET_SIZE(name); + + filename = PyUnicode_FromUnicode(buf, len); + if (filename == NULL) + return -1; /* Check for package import (buf holds a directory name, and there's an __init__ module in that directory */ #ifdef HAVE_STAT - if (stat(buf, &statbuf) == 0 && /* it exists */ + if (_Py_stat(filename, &statbuf) == 0 && /* it exists */ S_ISDIR(statbuf.st_mode)) /* it's a directory */ { int match; - PyObject *filename; - filename = PyUnicode_DecodeFSDefault(buf); - if (filename == NULL) - return -1; match = case_ok(filename, 0, name); if (match < 0) { Py_DECREF(filename); @@ -1780,7 +1771,7 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, } if (match) { /* case matches */ if (find_init_module(filename)) { /* and has __init__.py */ - Py_DECREF(filename); + *p_path = filename; *p_fd = &fd_package; return 2; } @@ -1795,9 +1786,9 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, } } } - Py_DECREF(filename); } #endif + *p_path = filename; return 1; } @@ -1814,21 +1805,16 @@ static struct filedescr* find_module_path_list(PyObject *fullname, PyObject *name, PyObject *search_path_list, PyObject *path_hooks, PyObject *path_importer_cache, - char *buf, size_t buflen, - FILE **p_fp, PyObject **p_loader) + PyObject **p_path, FILE **p_fp, PyObject **p_loader) { Py_ssize_t i, npath; - size_t len, namelen; struct filedescr *fdp = NULL; char *filemode; FILE *fp = NULL; - char *namestr; - PyObject *filename; + PyObject *prefix, *filename; int match; npath = PyList_Size(search_path_list); - namestr = _PyUnicode_AsString(name); - namelen = strlen(namestr); for (i = 0; i < npath; i++) { PyObject *path; int ok; @@ -1837,27 +1823,29 @@ find_module_path_list(PyObject *fullname, PyObject *name, if (path == NULL) return NULL; + prefix = NULL; ok = find_module_path(fullname, name, path, path_hooks, path_importer_cache, - buf, buflen, - p_loader, &fdp); + &prefix, p_loader, &fdp); if (ok < 0) return NULL; if (ok == 0) continue; - if (ok == 2) + if (ok == 2) { + *p_path = prefix; return fdp; + } - len = strlen(buf); for (fdp = _PyImport_Filetab; fdp->suffix != NULL; fdp++) { filemode = fdp->mode; if (filemode[0] == 'U') filemode = "r" PY_STDIOTEXTMODE; - strcpy(buf+len, fdp->suffix); - filename = PyUnicode_DecodeFSDefault(buf); - if (filename == NULL) + filename = PyUnicode_FromFormat("%U%s", prefix, fdp->suffix); + if (filename == NULL) { + Py_DECREF(prefix); return NULL; + } if (Py_VerboseFlag > 1) PySys_FormatStderr("# trying %R\n", filename); @@ -1869,11 +1857,13 @@ find_module_path_list(PyObject *fullname, PyObject *name, } match = case_ok(filename, -(Py_ssize_t)strlen(fdp->suffix), name); if (match < 0) { + Py_DECREF(prefix); Py_DECREF(filename); return NULL; } if (match) { - Py_DECREF(filename); + Py_DECREF(prefix); + *p_path = filename; *p_fp = fp; return fdp; } @@ -1882,6 +1872,7 @@ find_module_path_list(PyObject *fullname, PyObject *name, fclose(fp); fp = NULL; } + Py_DECREF(prefix); } PyErr_Format(PyExc_ImportError, "No module named %U", name); @@ -1897,32 +1888,35 @@ find_module_path_list(PyObject *fullname, PyObject *name, - otherwise, call find_module_path_list() with search_path_list (if not NULL) or sys.path + fullname can be NULL, but only if p_loader is NULL. + Return: - &fd_builtin (C_BUILTIN) if it is a builtin - &fd_frozen (PY_FROZEN) if it is frozen - - &fd_package (PKG_DIRECTORY) and write the filename into *buf + - &fd_package (PKG_DIRECTORY) and write the filename into *p_path if it is a package - &importhookdescr (IMP_HOOK) and write the loader into *p_loader if a importer loader was found - a file descriptor (PY_SOURCE, PY_COMPILED, C_EXTENSION, PY_RESOURCE or PY_CODERESOURCE: see _PyImport_Filetab), write the filename into - *buf and the pointer to the open file into *p_fp + *p_path and the pointer to the open file into *p_fp - NULL on error - By default, write an empty string into *buf, and *p_fp and *p_loader (if - set) are set to NULL. Eg. *buf is an empty string for a builtin package. */ + By default, *p_path, *p_fp and *p_loader (if set) are set to NULL. + Eg. *p_path is set to NULL for a builtin package. +*/ static struct filedescr * find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, - char *buf, size_t buflen, FILE **p_fp, PyObject **p_loader) + PyObject **p_path, FILE **p_fp, PyObject **p_loader) { Py_ssize_t i, npath; static struct filedescr fd_frozen = {"", "", PY_FROZEN}; static struct filedescr fd_builtin = {"", "", C_BUILTIN}; PyObject *path_hooks, *path_importer_cache; - *buf = '\0'; + *p_path = NULL; *p_fp = NULL; if (p_loader != NULL) *p_loader = NULL; @@ -1975,20 +1969,12 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, #ifdef MS_COREDLL FILE *fp; struct filedescr *fdp; - PyObject *filename, *filename_bytes; #endif if (is_builtin(name)) return &fd_builtin; #ifdef MS_COREDLL - fp = _PyWin_FindRegisteredModule(name, &fdp, &filename); + fp = _PyWin_FindRegisteredModule(name, &fdp, p_path); if (fp != NULL) { - filename_bytes = PyUnicode_EncodeFSDefault(filename); - Py_DECREF(filename); - if (filename_bytes == NULL) - return NULL; - strncpy(buf, PyBytes_AS_STRING(filename_bytes), buflen); - buf[buflen-1] = '\0'; - Py_DECREF(filename_bytes); *p_fp = fp; return fdp; } @@ -2022,8 +2008,7 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, return find_module_path_list(fullname, name, search_path_list, path_hooks, path_importer_cache, - buf, buflen, - p_fp, p_loader); + p_path, p_fp, p_loader); } /* case_bytes(char* buf, Py_ssize_t len, Py_ssize_t namelen, char* name) @@ -3185,7 +3170,6 @@ import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) { PyObject *modules = PyImport_GetModuleDict(); PyObject *m = NULL, *bufobj, *path_list, *loader; - char buf[MAXPATHLEN+1]; struct filedescr *fdp; FILE *fp; @@ -3211,7 +3195,7 @@ import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) } fdp = find_module(fullname, subname, path_list, - buf, MAXPATHLEN+1, &fp, &loader); + &bufobj, &fp, &loader); Py_XDECREF(path_list); if (fdp == NULL) { if (!PyErr_ExceptionMatches(PyExc_ImportError)) @@ -3220,13 +3204,8 @@ import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) Py_INCREF(Py_None); return Py_None; } - bufobj = PyUnicode_DecodeFSDefault(buf); - if (bufobj != NULL) { - m = load_module(fullname, fp, bufobj, fdp->type, loader); - Py_DECREF(bufobj); - } - else - m = NULL; + m = load_module(fullname, fp, bufobj, fdp->type, loader); + Py_XDECREF(bufobj); Py_XDECREF(loader); if (fp) fclose(fp); @@ -3249,7 +3228,6 @@ PyImport_ReloadModule(PyObject *m) PyInterpreterState *interp = PyThreadState_Get()->interp; PyObject *modules_reloading = interp->modules_reloading; PyObject *modules = PyImport_GetModuleDict(); - char buf[MAXPATHLEN+1]; PyObject *path_list = NULL, *loader = NULL, *existing_m = NULL; PyObject *nameobj, *bufobj, *subnameobj; Py_UNICODE *name, *subname; @@ -3324,7 +3302,7 @@ PyImport_ReloadModule(PyObject *m) if (subnameobj == NULL) goto error; fdp = find_module(nameobj, subnameobj, path_list, - buf, MAXPATHLEN+1, &fp, &loader); + &bufobj, &fp, &loader); Py_DECREF(subnameobj); Py_XDECREF(path_list); @@ -3333,13 +3311,8 @@ PyImport_ReloadModule(PyObject *m) goto error; } - bufobj = PyUnicode_DecodeFSDefault(buf); - if (bufobj != NULL) { - newm = load_module(nameobj, fp, bufobj, fdp->type, loader); - Py_DECREF(bufobj); - } - else - newm = NULL; + newm = load_module(nameobj, fp, bufobj, fdp->type, loader); + Py_XDECREF(bufobj); Py_XDECREF(loader); if (fp) @@ -3509,17 +3482,15 @@ call_find_module(PyObject *name, PyObject *path_list) PyObject *fob, *ret; PyObject *pathobj; struct filedescr *fdp; - char pathname[MAXPATHLEN+1]; FILE *fp; int fd = -1; char *found_encoding = NULL; char *encoding = NULL; - pathname[0] = '\0'; if (path_list == Py_None) path_list = NULL; fdp = find_module(NULL, name, path_list, - pathname, MAXPATHLEN+1, &fp, NULL); + &pathobj, &fp, NULL); if (fdp == NULL) return NULL; if (fp != NULL) { @@ -3535,14 +3506,17 @@ call_find_module(PyObject *name, PyObject *path_list) memory. */ found_encoding = PyTokenizer_FindEncoding(fd); lseek(fd, 0, 0); /* Reset position */ - if (found_encoding == NULL && PyErr_Occurred()) + if (found_encoding == NULL && PyErr_Occurred()) { + Py_XDECREF(pathobj); return NULL; + } encoding = (found_encoding != NULL) ? found_encoding : (char*)PyUnicode_GetDefaultEncoding(); } - fob = PyFile_FromFd(fd, pathname, fdp->mode, -1, + fob = PyFile_FromFd(fd, NULL, fdp->mode, -1, (char*)encoding, NULL, NULL, 1); if (fob == NULL) { + Py_XDECREF(pathobj); close(fd); PyMem_FREE(found_encoding); return NULL; @@ -3552,7 +3526,10 @@ call_find_module(PyObject *name, PyObject *path_list) fob = Py_None; Py_INCREF(fob); } - pathobj = PyUnicode_DecodeFSDefault(pathname); + if (pathobj == NULL) { + Py_INCREF(Py_None); + pathobj = Py_None; + } ret = Py_BuildValue("NN(ssi)", fob, pathobj, fdp->suffix, fdp->mode, fdp->type); PyMem_FREE(found_encoding); -- cgit v1.2.1 From 8784da32ff0f704a48459abf452eba8345f0d053 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 14:53:28 -0400 Subject: Issue #3080: imp.new_module() uses Unicode --- Python/import.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 5dcbf17710..3237ddc84e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3787,10 +3787,10 @@ imp_load_package(PyObject *self, PyObject *args) static PyObject * imp_new_module(PyObject *self, PyObject *args) { - char *name; - if (!PyArg_ParseTuple(args, "s:new_module", &name)) + PyObject *name; + if (!PyArg_ParseTuple(args, "U:new_module", &name)) return NULL; - return PyModule_New(name); + return PyModule_NewObject(name); } static PyObject * -- cgit v1.2.1 From 24cf6f1d864c6c51a299ed32ba9e1fe8701a4b54 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 15:05:12 -0400 Subject: Fix imp.cache_from_source() if the directory name contains a dot If the directory name contains a dot but not the filename, don't strip at the dot. --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 3237ddc84e..e5cb627003 100644 --- a/Python/import.c +++ b/Python/import.c @@ -943,12 +943,12 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug) Py_UNICODE_strcat(buf, CACHEDIR_UNICODE); i += Py_UNICODE_strlen(CACHEDIR_UNICODE) - 1; buf[i++] = sep; - buf[i++] = '\0'; + buf[i] = '\0'; /* Add the base filename, but remove the .py or .pyw extension, since the tag name must go before the extension. */ Py_UNICODE_strcat(buf, pathname + save); - pos = Py_UNICODE_strrchr(buf, '.'); + pos = Py_UNICODE_strrchr(buf + i, '.'); if (pos != NULL) *++pos = '\0'; Py_UNICODE_strcat(buf, PYC_TAG_UNICODE); -- cgit v1.2.1 From 0b5200ff91987d460004543f52dc7443f57a6b36 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 15:15:47 -0400 Subject: Issue #3080: Use repr() to format the module name on error --- Python/import.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index e5cb627003..65944473cc 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1431,7 +1431,7 @@ load_source_module(PyObject *name, PyObject *pathname, FILE *fp) goto error; if (Py_VerboseFlag) PySys_FormatStderr("import %U # from %R\n", - name, pathname); + name, pathname); if (cpathname != NULL) { PyObject *ro = PySys_GetObject("dont_write_bytecode"); if (ro == NULL || !PyObject_IsTrue(ro)) @@ -1517,8 +1517,8 @@ load_package(PyObject *name, PyObject *pathname) if (m == NULL) return NULL; if (Py_VerboseFlag) - PySys_FormatStderr("import %U # directory %U\n", - name, pathname); + PySys_FormatStderr("import %U # directory %R\n", + name, pathname); file = get_sourcefile(pathname); if (file == NULL) return NULL; @@ -1875,7 +1875,7 @@ find_module_path_list(PyObject *fullname, PyObject *name, Py_DECREF(prefix); } PyErr_Format(PyExc_ImportError, - "No module named %U", name); + "No module named %R", name); return NULL; } @@ -2366,7 +2366,7 @@ load_module(PyObject *name, FILE *fp, PyObject *pathname, int type, PyObject *lo default: PyErr_Format(PyExc_ImportError, - "Don't know how to import %U (type code %d)", + "Don't know how to import %R (type code %d)", name, type); m = NULL; @@ -3251,7 +3251,7 @@ PyImport_ReloadModule(PyObject *m) return NULL; if (m != PyDict_GetItem(modules, nameobj)) { PyErr_Format(PyExc_ImportError, - "reload(): module %U not in sys.modules", + "reload(): module %R not in sys.modules", nameobj); Py_DECREF(nameobj); return NULL; @@ -3286,7 +3286,7 @@ PyImport_ReloadModule(PyObject *m) parent = PyDict_GetItem(modules, parentname); if (parent == NULL) { PyErr_Format(PyExc_ImportError, - "reload(): parent %U not in sys.modules", + "reload(): parent %R not in sys.modules", parentname); Py_DECREF(parentname); goto error; -- cgit v1.2.1 From 67a022a7f5cdf458fa1e76d7b5482fec1784890f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 15:54:52 -0400 Subject: Issue #3080: Add PyImport_ImportModuleLevelObject() function Use it for the builtin __import__ function. --- Python/import.c | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 65944473cc..7f9042ac01 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2753,25 +2753,37 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, } PyObject * -PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, - PyObject *fromlist, int level) +PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, + PyObject *locals, PyObject *fromlist, + int level) { - PyObject *nameobj, *result; - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; + PyObject *mod; _PyImport_AcquireLock(); - result = import_module_level(nameobj, globals, locals, fromlist, level); - Py_DECREF(nameobj); + mod = import_module_level(name, globals, locals, fromlist, level); if (_PyImport_ReleaseLock() < 0) { - Py_XDECREF(result); + Py_XDECREF(mod); PyErr_SetString(PyExc_RuntimeError, "not holding the import lock"); return NULL; } - return result; + return mod; +} + +PyObject * +PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, + PyObject *fromlist, int level) +{ + PyObject *nameobj, *mod; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + mod = PyImport_ImportModuleLevelObject(nameobj, globals, locals, + fromlist, level); + Py_DECREF(nameobj); + return mod; } + /* Return the package that an import is being performed in. If globals comes from the module foo.bar.bat (not itself a package), this returns the sys.modules entry for foo.bar. If globals is from a package's __init__.py, -- cgit v1.2.1 From 83ee336336ccdc6b1fcc97ce0f693409ecf4b60a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 20 Mar 2011 04:28:55 +0100 Subject: Issue #3080: Fix case_ok() using case_bytes() Invert name and namelen arguments. --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 7f9042ac01..0194d89956 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2208,8 +2208,8 @@ case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) match = case_bytes( PyBytes_AS_STRING(filebytes), PyBytes_GET_SIZE(filebytes) + prefix_delta, - PyBytes_AS_STRING(namebytes), - PyBytes_GET_SIZE(namebytes)); + PyBytes_GET_SIZE(namebytes), + PyBytes_AS_STRING(namebytes)); Py_DECREF(filebytes); Py_DECREF(namebytes); return match; -- cgit v1.2.1 From f8715e031dd6628c290bfc9341b092063449a6f8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 20 Mar 2011 04:58:29 +0100 Subject: Issue #3080: Fix call to case_ok() in find_init_module() --- Python/import.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 0194d89956..38960b165d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2237,8 +2237,8 @@ find_init_module(PyObject *directory) if (filename == NULL) return -1; if (_Py_stat(filename, &statbuf) == 0) { - /* 9=len("/__init__") */ - match = case_ok(filename, 9, initstr); + /* 3=len(".py") */ + match = case_ok(filename, -3, initstr); if (match < 0) { Py_DECREF(filename); return -1; @@ -2255,8 +2255,8 @@ find_init_module(PyObject *directory) if (filename == NULL) return -1; if (_Py_stat(filename, &statbuf) == 0) { - /* 9=len("/__init__") */ - match = case_ok(filename, 9, initstr); + /* 4=len(".pyc") */ + match = case_ok(filename, -4, initstr); if (match < 0) { Py_DECREF(filename); return -1; -- cgit v1.2.1 From 6d71d6018e93b53f8af5ed0d268a4c834a5cdda2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 20 Mar 2011 22:37:17 +0100 Subject: Issue #3080: imp.load_module() accepts None for the module path imp.find_module() returns None as module path for builtin and frozen builtins. --- Python/import.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 38960b165d..abc3906719 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3744,17 +3744,22 @@ imp_load_source(PyObject *self, PyObject *args) static PyObject * imp_load_module(PyObject *self, PyObject *args) { - PyObject *name, *fob, *pathname, *ret; + PyObject *name, *fob, *pathname, *pathname_obj, *ret; char *suffix; /* Unused */ char *mode; int type; FILE *fp; - if (!PyArg_ParseTuple(args, "UOO&(ssi):load_module", - &name, &fob, - PyUnicode_FSDecoder, &pathname, - &suffix, &mode, &type)) + if (!PyArg_ParseTuple(args, "UOO(ssi):load_module", + &name, &fob, &pathname_obj, &suffix, &mode, &type)) return NULL; + if (pathname_obj != Py_None) { + if (!PyUnicode_FSDecoder(pathname_obj, &pathname)) + return NULL; + } + else + pathname = NULL; + if (*mode) { /* Mode must start with 'r' or 'U' and must not contain '+'. Implicit in this test is the assumption that the mode @@ -3763,7 +3768,7 @@ imp_load_module(PyObject *self, PyObject *args) if (!(*mode == 'r' || *mode == 'U') || strchr(mode, '+')) { PyErr_Format(PyExc_ValueError, "invalid file open mode %.200s", mode); - Py_DECREF(pathname); + Py_XDECREF(pathname); return NULL; } } @@ -3772,12 +3777,12 @@ imp_load_module(PyObject *self, PyObject *args) else { fp = get_file(NULL, fob, mode); if (fp == NULL) { - Py_DECREF(pathname); + Py_XDECREF(pathname); return NULL; } } ret = load_module(name, fp, pathname, type, NULL); - Py_DECREF(pathname); + Py_XDECREF(pathname); if (fp) fclose(fp); return ret; -- cgit v1.2.1 From 05292378fc049f9a5ffa9480d0ad27b25f2c5528 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2011 01:22:27 +0100 Subject: Issue #3080: On DJGPP, case_bytes() returns -1 to signal an error if the file cannot be found. --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index abc3906719..3b8ce931d9 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2084,7 +2084,7 @@ case_bytes(char *buf, Py_ssize_t len, Py_ssize_t namelen, const char *name) PyErr_Format(PyExc_NameError, "Can't find file for module %.100s\n(filename %.300s)", name, buf); - return 0; + return -1; } return strncmp(ffblk.ff_name, name, namelen) == 0; -- cgit v1.2.1 From aad56d7f47263bddcaca2765d03df11d81bc0926 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2011 10:46:35 +0100 Subject: Issue #11630, issue #3080: Fix refleak introduced by ef2b6305d395 --- Python/import.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 3b8ce931d9..907ccd797c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3705,12 +3705,15 @@ imp_load_dynamic(PyObject *self, PyObject *args) return NULL; if (fob != NULL) { fp = get_file(NULL, fob, "r"); - if (fp == NULL) + if (fp == NULL) { + Py_DECREF(pathname); return NULL; + } } else fp = NULL; mod = _PyImport_LoadDynamicModule(name, pathname, fp); + Py_DECREF(pathname); if (fp) fclose(fp); return mod; -- cgit v1.2.1 From 725c21b3a4aec4426ccaa4c0425ee4e777a9eb35 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 23 Mar 2011 16:14:42 -0700 Subject: Make importlib compatible with __import__ by "fixing" code.co_filename paths. __import__ does a little trick when importing from bytecode by back-patching the co_filename paths to point to the file location where the code object was loaded from, *not* where the code object was originally created. This allows co_filename to point to a valid path. Problem is that co_filename is immutable from Python, so a private function -- imp._fix_co_filename() -- had to be introduced in order to get things working properly. Originally the plan was to add a file argument to marshal.loads(), but that failed as the algorithm used by __import__ is not fully recursive as one might expect, so to be fully backwards-compatible the code used by __import__ needed to be exposed. This closes issue #6811 by taking a different approach than outlined in the issue. --- Python/import.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 907ccd797c..b074b834e9 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1374,6 +1374,32 @@ update_compiled_module(PyCodeObject *co, PyObject *newname) Py_DECREF(oldname); } +static PyObject * +imp_fix_co_filename(PyObject *self, PyObject *args) +{ + PyObject *co; + PyObject *file_path; + + if (!PyArg_ParseTuple(args, "OO:_fix_co_filename", &co, &file_path)) + return NULL; + + if (!PyCode_Check(co)) { + PyErr_SetString(PyExc_TypeError, + "first argument must be a code object"); + return NULL; + } + + if (!PyUnicode_Check(file_path)) { + PyErr_SetString(PyExc_TypeError, + "second argument must be a string"); + return NULL; + } + + update_compiled_module((PyCodeObject*)co, file_path); + + Py_RETURN_NONE; +} + /* Load a source module from a given file and return its module object WITH INCREMENTED REFERENCE COUNT. If there's a matching byte-compiled file, use that instead. */ @@ -3976,6 +4002,7 @@ static PyMethodDef imp_methods[] = { #endif {"load_package", imp_load_package, METH_VARARGS}, {"load_source", imp_load_source, METH_VARARGS}, + {"_fix_co_filename", imp_fix_co_filename, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; -- cgit v1.2.1 From f41c2fdbb05ad29ad7c2a6a65dfac2e9d919f022 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 5 Apr 2011 01:48:03 +0200 Subject: Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. --- Python/import.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index b074b834e9..4159a8ece6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -124,12 +124,12 @@ static const Py_UNICODE PYC_TAG_UNICODE[] = { /* See _PyImport_FixupExtensionObject() below */ static PyObject *extensions = NULL; +/* Function from Parser/tokenizer.c */ +extern char * PyTokenizer_FindEncodingFilename(int, PyObject *); + /* This table is defined in config.c: */ extern struct _inittab _PyImport_Inittab[]; -/* Method from Parser/tokenizer.c */ -extern char * PyTokenizer_FindEncoding(int); - struct _inittab *PyImport_Inittab = _PyImport_Inittab; /* these tables define the module suffixes that Python recognizes */ @@ -3540,9 +3540,9 @@ call_find_module(PyObject *name, PyObject *path_list) } if (fd != -1) { if (strchr(fdp->mode, 'b') == NULL) { - /* PyTokenizer_FindEncoding() returns PyMem_MALLOC'ed + /* PyTokenizer_FindEncodingFilename() returns PyMem_MALLOC'ed memory. */ - found_encoding = PyTokenizer_FindEncoding(fd); + found_encoding = PyTokenizer_FindEncodingFilename(fd, pathobj); lseek(fd, 0, 0); /* Reset position */ if (found_encoding == NULL && PyErr_Occurred()) { Py_XDECREF(pathobj); -- cgit v1.2.1 From 0cd7388f789c312864700c151157a939f07cc090 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 20 Apr 2011 03:27:51 +0200 Subject: Close #11619: write_compiled_module() doesn't encode the filename Reimplement open_exclusive() using _wopen() to avoid encoding the filename to the filesystem encoding: use the Unicode version of the Windows API. --- Python/import.c | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 4159a8ece6..5360d57c51 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1198,6 +1198,7 @@ parse_source_module(PyObject *pathname, FILE *fp) /* Helper to open a bytecode file for writing in exclusive mode */ +#ifndef MS_WINDOWS static FILE * open_exclusive(char *filename, mode_t mode) { @@ -1228,6 +1229,7 @@ open_exclusive(char *filename, mode_t mode) return fopen(filename, "wb"); #endif } +#endif /* Write a compiled module to a file, placing the time of last @@ -1250,7 +1252,12 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, S_IWUSR | S_IWGRP | S_IWOTH); PyObject *dirbytes; #endif - PyObject *cpathbytes, *dirname; +#ifdef MS_WINDOWS + int fd; +#else + PyObject *cpathbytes; +#endif + PyObject *dirname; Py_UNICODE *dirsep; int res, ok; @@ -1294,6 +1301,16 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, } Py_DECREF(dirname); +#ifdef MS_WINDOWS + (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname)); + fd = _wopen(PyUnicode_AS_UNICODE(cpathname), + O_EXCL | O_CREAT | O_WRONLY | O_TRUNC | O_BINARY, + mode); + if (0 <= fd) + fp = fdopen(fd, "wb"); + else + fp = NULL; +#else cpathbytes = PyUnicode_EncodeFSDefault(cpathname); if (cpathbytes == NULL) { PyErr_Clear(); @@ -1301,11 +1318,14 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, } fp = open_exclusive(PyBytes_AS_STRING(cpathbytes), mode); +#endif if (fp == NULL) { if (Py_VerboseFlag) PySys_FormatStderr( "# can't create %R\n", cpathname); +#ifndef MS_WINDOWS Py_DECREF(cpathbytes); +#endif return; } PyMarshal_WriteLongToFile(pyc_magic, fp, Py_MARSHAL_VERSION); @@ -1321,11 +1341,13 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname)); #else (void) unlink(PyBytes_AS_STRING(cpathbytes)); -#endif Py_DECREF(cpathbytes); +#endif return; } +#ifndef MS_WINDOWS Py_DECREF(cpathbytes); +#endif /* Now write the true mtime */ fseek(fp, 4L, 0); assert(mtime < LONG_MAX); -- cgit v1.2.1 From bf30ea7ad386f8851207ea9d410a3215b03d31af Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Wed, 25 May 2011 09:29:00 -0500 Subject: make PyImport_ImportModuleLevel's first arg const like similiar functions (closes #12173) --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 5360d57c51..bfb976c77c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2818,7 +2818,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } PyObject * -PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, +PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) { PyObject *nameobj, *mod; -- cgit v1.2.1 From 54ecf696bb489a653324f5b8a20018495b8f0142 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 26 May 2011 13:47:08 +0200 Subject: Remove useless assignments Warnings found by the the Clang Static Analyzer. --- Python/import.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index bfb976c77c..1f28d2233d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1733,7 +1733,6 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, Py_UNICODE buf[MAXPATHLEN+1]; Py_ssize_t buflen = MAXPATHLEN+1; PyObject *path_unicode, *filename; - const Py_UNICODE *base; Py_ssize_t len; struct stat statbuf; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; @@ -1751,7 +1750,6 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, else return 0; - base = PyUnicode_AS_UNICODE(path_unicode); len = PyUnicode_GET_SIZE(path_unicode); if (len + 2 + PyUnicode_GET_SIZE(name) + MAXSUFFIXSIZE >= buflen) { Py_DECREF(path_unicode); @@ -2275,12 +2273,10 @@ case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) static int find_init_module(PyObject *directory) { - size_t len; struct stat statbuf; PyObject *filename; int match; - len = PyUnicode_GET_SIZE(directory); filename = PyUnicode_FromFormat("%U%c__init__.py", directory, SEP); if (filename == NULL) return -1; -- cgit v1.2.1 From dbea271885eccda0db2e5248fd932cc56ec8a495 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Fri, 3 Jun 2011 17:50:16 -0500 Subject: some horrible preprocessing tricks to automatically update the tag --- Python/import.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 1f28d2233d..aaf649f858 100644 --- a/Python/import.c +++ b/Python/import.c @@ -110,8 +110,12 @@ typedef unsigned short mode_t; TAG and PYC_TAG_UNICODE must change for each major Python release. The magic number will take care of any bytecode changes that occur during development. */ +#define QUOTE(arg) #arg +#define STRIFY(name) QUOTE(name) +#define MAJOR STRIFY(PY_MAJOR_VERSION) +#define MINOR STRIFY(PY_MINOR_VERSION) #define MAGIC (3180 | ((long)'\r'<<16) | ((long)'\n'<<24)) -#define TAG "cpython-32" +#define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" static const Py_UNICODE CACHEDIR_UNICODE[] = { '_', '_', 'p', 'y', 'c', 'a', 'c', 'h', 'e', '_', '_', '\0'}; @@ -119,7 +123,11 @@ static const Py_UNICODE CACHEDIR_UNICODE[] = { static long pyc_magic = MAGIC; static const char *pyc_tag = TAG; static const Py_UNICODE PYC_TAG_UNICODE[] = { - 'c', 'p', 'y', 't', 'h', 'o', 'n', '-', '3', '2', '\0'}; + 'c', 'p', 'y', 't', 'h', 'o', 'n', '-', PY_MAJOR_VERSION + 48, PY_MINOR_VERSION + 48, '\0'}; +#undef QUOTE +#undef STRIFY +#undef MAJOR +#undef MINOR /* See _PyImport_FixupExtensionObject() below */ static PyObject *extensions = NULL; -- cgit v1.2.1 From 18d2c7a85fe49dbac1177fffb6517b8093f8e23c Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 19 Jun 2011 19:54:45 -0500 Subject: bump magic for super closure change --- Python/import.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index aaf649f858..9ec8cd8b84 100644 --- a/Python/import.c +++ b/Python/import.c @@ -102,6 +102,7 @@ typedef unsigned short mode_t; Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR) tag: cpython-32 Python 3.2a2 3180 (add DELETE_DEREF) + Python 3.3a0 3190 __class__ super closure changed */ /* MAGIC must change whenever the bytecode emitted by the compiler may no @@ -114,7 +115,7 @@ typedef unsigned short mode_t; #define STRIFY(name) QUOTE(name) #define MAJOR STRIFY(PY_MAJOR_VERSION) #define MINOR STRIFY(PY_MINOR_VERSION) -#define MAGIC (3180 | ((long)'\r'<<16) | ((long)'\n'<<24)) +#define MAGIC (3190 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" static const Py_UNICODE CACHEDIR_UNICODE[] = { -- cgit v1.2.1 From 1a17e5818c9c58989ffdd4d0e2931db9c66579a0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 20 Jun 2011 15:01:10 +0200 Subject: find_module_path_list() fails if _Py_fopen() failed and raised an exception (UnicodeEncodeError). --- Python/import.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 9ec8cd8b84..de5dc0ca41 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1908,6 +1908,10 @@ find_module_path_list(PyObject *fullname, PyObject *name, fp = _Py_fopen(filename, filemode); if (fp == NULL) { Py_DECREF(filename); + if (PyErr_Occurred()) { + Py_DECREF(prefix); + return NULL; + } continue; } match = case_ok(filename, -(Py_ssize_t)strlen(fdp->suffix), name); -- cgit v1.2.1 From fab579dcbba91dc35f5f2b25fcf221736fff5e4b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 20 Jun 2011 15:16:55 +0200 Subject: call_find_module() handles dup() failure: raise an OSError exception --- Python/import.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index de5dc0ca41..19e975a593 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3567,6 +3567,10 @@ call_find_module(PyObject *name, PyObject *path_list) if (fd != -1) fd = dup(fd); fclose(fp); + if (fd == -1) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } fp = NULL; } if (fd != -1) { -- cgit v1.2.1 From 9ce343f33c35b14fe616a15cea33f8f84ccbff85 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 15 Sep 2011 19:50:01 +0200 Subject: import.c: remove now useless arbitrary limit --- Python/import.c | 6 ------ 1 file changed, 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 7902721a92..adfd2cc5df 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1980,12 +1980,6 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, if (p_loader != NULL) *p_loader = NULL; - if (PyUnicode_GET_SIZE(name) > MAXPATHLEN) { - PyErr_SetString(PyExc_OverflowError, - "module name is too long"); - return NULL; - } - /* sys.meta_path import hook */ if (p_loader != NULL) { PyObject *meta_path; -- cgit v1.2.1 From d1d013c01c268d869597b35cbcd8b5d7c5baf2ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Wed, 28 Sep 2011 07:41:54 +0200 Subject: Implement PEP 393. --- Python/import.c | 335 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 190 insertions(+), 145 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 24df985d46..4afdb7058c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -118,12 +118,12 @@ typedef unsigned short mode_t; #define MAGIC (3190 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" -static const Py_UNICODE CACHEDIR_UNICODE[] = { +static const Py_UCS4 CACHEDIR_UNICODE[] = { '_', '_', 'p', 'y', 'c', 'a', 'c', 'h', 'e', '_', '_', '\0'}; /* Current magic word and string tag as globals. */ static long pyc_magic = MAGIC; static const char *pyc_tag = TAG; -static const Py_UNICODE PYC_TAG_UNICODE[] = { +static const Py_UCS4 PYC_TAG_UNICODE[] = { 'c', 'p', 'y', 't', 'h', 'o', 'n', '-', PY_MAJOR_VERSION + 48, PY_MINOR_VERSION + 48, '\0'}; #undef QUOTE #undef STRIFY @@ -762,7 +762,7 @@ remove_module(PyObject *name) static PyObject * get_sourcefile(PyObject *filename); static PyObject *make_source_pathname(PyObject *pathname); -static PyObject* make_compiled_pathname(Py_UNICODE *pathname, int debug); +static PyObject* make_compiled_pathname(PyObject *pathname, int debug); /* Execute a code object in a module and return the module object * WITH INCREMENTED REFERENCE COUNT. If an error occurs, name is @@ -886,10 +886,10 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, /* Like strrchr(string, '/') but searches for the rightmost of either SEP or ALTSEP, if the latter is defined. */ -static Py_UNICODE* -rightmost_sep(Py_UNICODE *s) +static Py_UCS4* +rightmost_sep(Py_UCS4 *s) { - Py_UNICODE *found, c; + Py_UCS4 *found, c; for (found = NULL; (c = *s); s++) { if (c == SEP #ifdef ALTSEP @@ -912,15 +912,21 @@ rightmost_sep(Py_UNICODE *s) foo.py -> __pycache__/foo..pyc */ static PyObject* -make_compiled_pathname(Py_UNICODE *pathname, int debug) +make_compiled_pathname(PyObject *pathstr, int debug) { - Py_UNICODE buf[MAXPATHLEN]; + Py_UCS4 *pathname; + Py_UCS4 buf[MAXPATHLEN]; size_t buflen = (size_t)MAXPATHLEN; - size_t len = Py_UNICODE_strlen(pathname); + size_t len; size_t i, save; - Py_UNICODE *pos; + Py_UCS4 *pos; int sep = SEP; + pathname = PyUnicode_AsUCS4Copy(pathstr); + if (!pathname) + return NULL; + len = Py_UCS4_strlen(pathname); + /* Sanity check that the buffer has roughly enough space to hold what will eventually be the full path to the compiled file. The 5 extra bytes include the slash afer __pycache__, the two extra dots, the @@ -930,8 +936,10 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug) sanity check before writing the extension to ensure we do not overflow the buffer. */ - if (len + Py_UNICODE_strlen(CACHEDIR_UNICODE) + Py_UNICODE_strlen(PYC_TAG_UNICODE) + 5 > buflen) + if (len + Py_UCS4_strlen(CACHEDIR_UNICODE) + Py_UCS4_strlen(PYC_TAG_UNICODE) + 5 > buflen) { + PyMem_Free(pathname); return NULL; + } /* Find the last path separator and copy everything from the start of the source string up to and including the separator. @@ -943,24 +951,28 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug) else { sep = *pos; i = pos - pathname + 1; - Py_UNICODE_strncpy(buf, pathname, i); + Py_UCS4_strncpy(buf, pathname, i); } save = i; buf[i++] = '\0'; /* Add __pycache__/ */ - Py_UNICODE_strcat(buf, CACHEDIR_UNICODE); - i += Py_UNICODE_strlen(CACHEDIR_UNICODE) - 1; + Py_UCS4_strcat(buf, CACHEDIR_UNICODE); + i += Py_UCS4_strlen(CACHEDIR_UNICODE) - 1; buf[i++] = sep; buf[i] = '\0'; /* Add the base filename, but remove the .py or .pyw extension, since the tag name must go before the extension. */ - Py_UNICODE_strcat(buf, pathname + save); - pos = Py_UNICODE_strrchr(buf + i, '.'); + Py_UCS4_strcat(buf, pathname + save); + pos = Py_UCS4_strrchr(buf + i, '.'); if (pos != NULL) *++pos = '\0'; - Py_UNICODE_strcat(buf, PYC_TAG_UNICODE); + + /* pathname is not used from here on. */ + PyMem_Free(pathname); + + Py_UCS4_strcat(buf, PYC_TAG_UNICODE); /* The length test above assumes that we're only adding one character to the end of what would normally be the extension. What if there is no extension, or the string ends in '.' or '.p', and otherwise @@ -1010,7 +1022,7 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug) #if 0 printf("strlen(buf): %d; buflen: %d\n", (int)strlen(buf), (int)buflen); #endif - len = Py_UNICODE_strlen(buf); + len = Py_UCS4_strlen(buf); if (len + 5 > buflen) return NULL; buf[len] = '.'; len++; @@ -1018,7 +1030,7 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug) buf[len] = 'y'; len++; buf[len] = debug ? 'c' : 'o'; len++; assert(len <= buflen); - return PyUnicode_FromUnicode(buf, len); + return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, len); } @@ -1033,14 +1045,16 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug) static PyObject* make_source_pathname(PyObject *pathobj) { - Py_UNICODE buf[MAXPATHLEN]; - Py_UNICODE *pathname; - Py_UNICODE *left, *right, *dot0, *dot1, sep; + Py_UCS4 buf[MAXPATHLEN]; + Py_UCS4 *pathname; + Py_UCS4 *left, *right, *dot0, *dot1, sep; size_t i, j; - if (PyUnicode_GET_SIZE(pathobj) > MAXPATHLEN) + if (PyUnicode_GET_LENGTH(pathobj) > MAXPATHLEN) + return NULL; + pathname = PyUnicode_AsUCS4Copy(pathobj); + if (!pathname) return NULL; - pathname = PyUnicode_AS_UNICODE(pathobj); /* Look back two slashes from the end. In between these two slashes must be the string __pycache__ or this is not a PEP 3147 style @@ -1057,31 +1071,35 @@ make_source_pathname(PyObject *pathobj) left = pathname; else left++; - if (right-left != Py_UNICODE_strlen(CACHEDIR_UNICODE) || - Py_UNICODE_strncmp(left, CACHEDIR_UNICODE, right-left) != 0) - return NULL; + if (right-left != Py_UCS4_strlen(CACHEDIR_UNICODE) || + Py_UCS4_strncmp(left, CACHEDIR_UNICODE, right-left) != 0) + goto error; /* Now verify that the path component to the right of the last slash has two dots in it. */ - if ((dot0 = Py_UNICODE_strchr(right + 1, '.')) == NULL) - return NULL; - if ((dot1 = Py_UNICODE_strchr(dot0 + 1, '.')) == NULL) - return NULL; + if ((dot0 = Py_UCS4_strchr(right + 1, '.')) == NULL) + goto error; + if ((dot1 = Py_UCS4_strchr(dot0 + 1, '.')) == NULL) + goto error; /* Too many dots? */ - if (Py_UNICODE_strchr(dot1 + 1, '.') != NULL) - return NULL; + if (Py_UCS4_strchr(dot1 + 1, '.') != NULL) + goto error; /* This is a PEP 3147 path. Start by copying everything from the start of pathname up to and including the leftmost slash. Then copy the file's basename, removing the magic tag and adding a .py suffix. */ - Py_UNICODE_strncpy(buf, pathname, (i=left-pathname)); - Py_UNICODE_strncpy(buf+i, right+1, (j=dot0-right)); + Py_UCS4_strncpy(buf, pathname, (i=left-pathname)); + Py_UCS4_strncpy(buf+i, right+1, (j=dot0-right)); buf[i+j] = 'p'; buf[i+j+1] = 'y'; - return PyUnicode_FromUnicode(buf, i+j+2); + PyMem_Free(pathname); + return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, i+j+2); + error: + PyMem_Free(pathname); + return NULL; } /* Given a pathname for a Python source file, its time of last @@ -1250,6 +1268,7 @@ static void write_compiled_module(PyCodeObject *co, PyObject *cpathname, struct stat *srcstat) { + Py_UCS4 *cpathname_ucs4; FILE *fp; time_t mtime = srcstat->st_mtime; #ifdef MS_WINDOWS /* since Windows uses different permissions */ @@ -1267,18 +1286,23 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, PyObject *cpathbytes; #endif PyObject *dirname; - Py_UNICODE *dirsep; + Py_UCS4 *dirsep; int res, ok; /* Ensure that the __pycache__ directory exists. */ - dirsep = rightmost_sep(PyUnicode_AS_UNICODE(cpathname)); + cpathname_ucs4 = PyUnicode_AsUCS4Copy(cpathname); + if (!cpathname_ucs4) + return; + dirsep = rightmost_sep(cpathname_ucs4); if (dirsep == NULL) { if (Py_VerboseFlag) PySys_FormatStderr("# no %s path found %R\n", CACHEDIR, cpathname); return; } - dirname = PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(cpathname), - dirsep - PyUnicode_AS_UNICODE(cpathname)); + dirname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + cpathname_ucs4, + dirsep - cpathname_ucs4); + PyMem_Free(cpathname_ucs4); if (dirname == NULL) { PyErr_Clear(); return; @@ -1461,9 +1485,7 @@ load_source_module(PyObject *name, PyObject *pathname, FILE *fp) goto error; } #endif - cpathname = make_compiled_pathname( - PyUnicode_AS_UNICODE(pathname), - !Py_OptimizeFlag); + cpathname = make_compiled_pathname(pathname, !Py_OptimizeFlag); if (cpathname != NULL) fpc = check_compiled_module(pathname, st.st_mtime, cpathname); @@ -1512,16 +1534,18 @@ static PyObject * get_sourcefile(PyObject *filename) { Py_ssize_t len; - Py_UNICODE *fileuni; + Py_UCS4 *fileuni; PyObject *py; struct stat statbuf; - len = PyUnicode_GET_SIZE(filename); + len = PyUnicode_GET_LENGTH(filename); if (len == 0) Py_RETURN_NONE; /* don't match *.pyc or *.pyo? */ - fileuni = PyUnicode_AS_UNICODE(filename); + fileuni = PyUnicode_AsUCS4Copy(filename); + if (!fileuni) + return NULL; if (len < 5 || fileuni[len-4] != '.' || (fileuni[len-3] != 'p' && fileuni[len-3] != 'P') @@ -1535,7 +1559,7 @@ get_sourcefile(PyObject *filename) py = make_source_pathname(filename); if (py == NULL) { PyErr_Clear(); - py = PyUnicode_FromUnicode(fileuni, len - 1); + py = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, fileuni, len - 1); } if (py == NULL) goto error; @@ -1548,6 +1572,7 @@ get_sourcefile(PyObject *filename) error: PyErr_Clear(); unchanged: + PyMem_Free(fileuni); Py_INCREF(filename); return filename; } @@ -1739,8 +1764,7 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, PyObject *path_hooks, PyObject *path_importer_cache, PyObject **p_path, PyObject **p_loader, struct filedescr **p_fd) { - Py_UNICODE buf[MAXPATHLEN+1]; - Py_ssize_t buflen = MAXPATHLEN+1; + Py_UCS4 buf[MAXPATHLEN+1]; PyObject *path_unicode, *filename; Py_ssize_t len; struct stat statbuf; @@ -1759,15 +1783,15 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, else return 0; - len = PyUnicode_GET_SIZE(path_unicode); - if (len + 2 + PyUnicode_GET_SIZE(name) + MAXSUFFIXSIZE >= buflen) { + len = PyUnicode_GET_LENGTH(path_unicode); + if (!PyUnicode_AsUCS4(path_unicode, buf, PY_ARRAY_LENGTH(buf), 1)) { Py_DECREF(path_unicode); - return 0; /* Too long */ + PyErr_Clear(); + return 0; } - Py_UNICODE_strcpy(buf, PyUnicode_AS_UNICODE(path_unicode)); Py_DECREF(path_unicode); - if (Py_UNICODE_strlen(buf) != len) + if (Py_UCS4_strlen(buf) != len) return 0; /* path contains '\0' */ /* sys.path_hooks import hook */ @@ -1804,10 +1828,14 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, #endif ) buf[len++] = SEP; - Py_UNICODE_strcpy(buf+len, PyUnicode_AS_UNICODE(name)); - len += PyUnicode_GET_SIZE(name); + if (!PyUnicode_AsUCS4(name, buf+len, PY_ARRAY_LENGTH(buf)-len, 1)) { + PyErr_Clear(); + return 0; + } + len += PyUnicode_GET_LENGTH(name); - filename = PyUnicode_FromUnicode(buf, len); + filename = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buf, len); if (filename == NULL) return -1; @@ -1989,6 +2017,12 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, if (p_loader != NULL) *p_loader = NULL; + if (PyUnicode_GET_LENGTH(name) > MAXPATHLEN) { + PyErr_SetString(PyExc_OverflowError, + "module name is too long"); + return NULL; + } + /* sys.meta_path import hook */ if (p_loader != NULL) { PyObject *meta_path; @@ -2704,7 +2738,7 @@ static PyObject *get_parent(PyObject *globals, int level); static PyObject *load_next(PyObject *mod, PyObject *altmod, PyObject *inputname, PyObject **p_outputname, - Py_UNICODE *buf, Py_ssize_t *p_buflen, + Py_UCS4 *buf, Py_ssize_t *p_buflen, Py_ssize_t bufsize); static int mark_miss(PyObject *name); static int ensure_fromlist(PyObject *mod, PyObject *fromlist, @@ -2718,37 +2752,47 @@ static PyObject * import_module_level(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) { - Py_UNICODE buf[MAXPATHLEN+1]; + Py_UCS4 buf[MAXPATHLEN+1]; Py_ssize_t buflen; Py_ssize_t bufsize = MAXPATHLEN+1; PyObject *parent, *head, *next, *tail, *inputname, *outputname; PyObject *parent_name, *ensure_name; - const Py_UNICODE *nameunicode; + Py_ssize_t sep, altsep; - nameunicode = PyUnicode_AS_UNICODE(name); + if (PyUnicode_READY(name)) + return NULL; - if (Py_UNICODE_strchr(nameunicode, SEP) != NULL + sep = PyUnicode_FindChar(name, SEP, 0, PyUnicode_GET_LENGTH(name), 1); + if (sep == -2) + return NULL; #ifdef ALTSEP - || Py_UNICODE_strchr(nameunicode, ALTSEP) != NULL + altsep = PyUnicode_FindChar(name, ALTSEP, 0, PyUnicode_GET_LENGTH(name), 1); + if (altsep == -2) + return NULL; +#else + altsep = -1; #endif - ) { + if (sep != -1 || altsep != -1) + { PyErr_SetString(PyExc_ImportError, "Import by filename is not supported."); return NULL; } parent = get_parent(globals, &parent_name, level); - if (parent == NULL) + if (parent == NULL) { return NULL; + } - buflen = PyUnicode_GET_SIZE(parent_name); - if (buflen+1 > bufsize) { + if (PyUnicode_READY(parent_name)) + return NULL; + buflen = PyUnicode_GET_LENGTH(parent_name); + if (!PyUnicode_AsUCS4(parent_name, buf, PY_ARRAY_LENGTH(buf), 1)) { Py_DECREF(parent_name); PyErr_SetString(PyExc_ValueError, "Module name too long"); return NULL; } - Py_UNICODE_strcpy(buf, PyUnicode_AS_UNICODE(parent_name)); Py_DECREF(parent_name); head = load_next(parent, level < 0 ? Py_None : parent, name, &outputname, @@ -2799,7 +2843,8 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, Py_DECREF(head); - ensure_name = PyUnicode_FromUnicode(buf, Py_UNICODE_strlen(buf)); + ensure_name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buf, Py_UCS4_strlen(buf)); if (ensure_name == NULL) { Py_DECREF(tail); return NULL; @@ -2859,8 +2904,6 @@ PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals static PyObject * get_parent(PyObject *globals, PyObject **p_name, int level) { - Py_UNICODE name[MAXPATHLEN+1]; - const Py_ssize_t bufsize = MAXPATHLEN+1; PyObject *nameobj; static PyObject *namestr = NULL; @@ -2897,7 +2940,7 @@ get_parent(PyObject *globals, PyObject **p_name, int level) "__package__ set to non-string"); return NULL; } - if (PyUnicode_GET_SIZE(pkgname) == 0) { + if (PyUnicode_GET_LENGTH(pkgname) == 0) { if (level > 0) { PyErr_SetString(PyExc_ValueError, "Attempted relative import in non-package"); @@ -2905,12 +2948,8 @@ get_parent(PyObject *globals, PyObject **p_name, int level) } goto return_none; } - if (PyUnicode_GET_SIZE(pkgname)+1 > bufsize) { - PyErr_SetString(PyExc_ValueError, - "Package name too long"); - return NULL; - } - Py_UNICODE_strcpy(name, PyUnicode_AS_UNICODE(pkgname)); + Py_INCREF(pkgname); + nameobj = pkgname; } else { /* __package__ not set, so figure it out and set it */ modname = PyDict_GetItem(globals, namestr); @@ -2922,74 +2961,71 @@ get_parent(PyObject *globals, PyObject **p_name, int level) /* __path__ is set, so modname is already the package name */ int error; - if (PyUnicode_GET_SIZE(modname)+1 > bufsize) { - PyErr_SetString(PyExc_ValueError, - "Module name too long"); - return NULL; - } - Py_UNICODE_strcpy(name, PyUnicode_AS_UNICODE(modname)); error = PyDict_SetItem(globals, pkgstr, modname); if (error) { PyErr_SetString(PyExc_ValueError, "Could not set __package__"); return NULL; } + Py_INCREF(modname); + nameobj = modname; } else { /* Normal module, so work out the package name if any */ - Py_UNICODE *start = PyUnicode_AS_UNICODE(modname); - Py_UNICODE *lastdot = Py_UNICODE_strrchr(start, '.'); Py_ssize_t len; - int error; - if (lastdot == NULL && level > 0) { - PyErr_SetString(PyExc_ValueError, - "Attempted relative import in non-package"); + len = PyUnicode_FindChar(modname, '.', + 0, PyUnicode_GET_LENGTH(modname), -1); + if (len == -2) return NULL; - } - if (lastdot == NULL) { - error = PyDict_SetItem(globals, pkgstr, Py_None); - if (error) { + if (len < 0) { + if (level > 0) { + PyErr_SetString(PyExc_ValueError, + "Attempted relative import in non-package"); + return NULL; + } + if (PyDict_SetItem(globals, pkgstr, Py_None)) { PyErr_SetString(PyExc_ValueError, "Could not set __package__"); return NULL; } goto return_none; } - len = lastdot - start; - if (len+1 > bufsize) { + pkgname = PyUnicode_Substring(modname, 0, len); + if (pkgname == NULL) + return NULL; + if (PyDict_SetItem(globals, pkgstr, pkgname)) { + Py_DECREF(pkgname); PyErr_SetString(PyExc_ValueError, - "Module name too long"); + "Could not set __package__"); return NULL; } - Py_UNICODE_strncpy(name, start, len); - name[len] = '\0'; - pkgname = PyUnicode_FromUnicode(name, len); - if (pkgname == NULL) { + nameobj = pkgname; + } + } + if (level > 1) { + Py_ssize_t dot, end = PyUnicode_GET_LENGTH(nameobj); + PyObject *newname; + while (--level > 0) { + dot = PyUnicode_FindChar(nameobj, '.', 0, end, -1); + if (dot == -2) { + Py_DECREF(nameobj); return NULL; } - error = PyDict_SetItem(globals, pkgstr, pkgname); - Py_DECREF(pkgname); - if (error) { + if (dot < 0) { + Py_DECREF(nameobj); PyErr_SetString(PyExc_ValueError, - "Could not set __package__"); + "Attempted relative import beyond " + "toplevel package"); return NULL; } + end = dot; } - } - while (--level > 0) { - Py_UNICODE *dot = Py_UNICODE_strrchr(name, '.'); - if (dot == NULL) { - PyErr_SetString(PyExc_ValueError, - "Attempted relative import beyond " - "toplevel package"); + newname = PyUnicode_Substring(nameobj, 0, end); + Py_DECREF(nameobj); + if (newname == NULL) return NULL; - } - *dot = '\0'; + nameobj = newname; } - nameobj = PyUnicode_FromUnicode(name, Py_UNICODE_strlen(name)); - if (nameobj == NULL) - return NULL; - modules = PyImport_GetModuleDict(); parent = PyDict_GetItem(modules, nameobj); if (parent == NULL) { @@ -3021,7 +3057,7 @@ get_parent(PyObject *globals, PyObject **p_name, int level) If this is violated... Who cares? */ return_none: - nameobj = PyUnicode_FromUnicode(NULL, 0); + nameobj = PyUnicode_New(0, 0); if (nameobj == NULL) return NULL; *p_name = nameobj; @@ -3032,28 +3068,28 @@ return_none: static PyObject * load_next(PyObject *mod, PyObject *altmod, PyObject *inputname, PyObject **p_outputname, - Py_UNICODE *buf, Py_ssize_t *p_buflen, Py_ssize_t bufsize) + Py_UCS4 *buf, Py_ssize_t *p_buflen, Py_ssize_t bufsize) { - const Py_UNICODE *dot; + Py_UCS4 *dot; Py_ssize_t len; - Py_UNICODE *p; + Py_UCS4 *p; PyObject *fullname, *name, *result, *mark_name; - const Py_UNICODE *nameuni; + const Py_UCS4 *nameuni; *p_outputname = NULL; - if (PyUnicode_GET_SIZE(inputname) == 0) { + if (PyUnicode_GET_LENGTH(inputname) == 0) { /* completely empty module name should only happen in 'from . import' (or '__import__("")')*/ Py_INCREF(mod); return mod; } - nameuni = PyUnicode_AS_UNICODE(inputname); + nameuni = PyUnicode_AsUCS4Copy(inputname); if (nameuni == NULL) return NULL; - dot = Py_UNICODE_strchr(nameuni, '.'); + dot = Py_UCS4_strchr(nameuni, '.'); if (dot != NULL) { len = dot - nameuni; if (len == 0) { @@ -3063,7 +3099,7 @@ load_next(PyObject *mod, PyObject *altmod, } } else - len = PyUnicode_GET_SIZE(inputname); + len = PyUnicode_GET_LENGTH(inputname); if (*p_buflen+len+1 >= bufsize) { PyErr_SetString(PyExc_ValueError, @@ -3076,14 +3112,16 @@ load_next(PyObject *mod, PyObject *altmod, *p++ = '.'; *p_buflen += 1; } - Py_UNICODE_strncpy(p, nameuni, len); + Py_UCS4_strncpy(p, nameuni, len); p[len] = '\0'; *p_buflen += len; - fullname = PyUnicode_FromUnicode(buf, *p_buflen); + fullname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buf, *p_buflen); if (fullname == NULL) return NULL; - name = PyUnicode_FromUnicode(p, len); + name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + p, len); if (name == NULL) { Py_DECREF(fullname); return NULL; @@ -3096,7 +3134,8 @@ load_next(PyObject *mod, PyObject *altmod, result = import_submodule(altmod, name, name); Py_DECREF(name); if (result != NULL && result != Py_None) { - mark_name = PyUnicode_FromUnicode(buf, *p_buflen); + mark_name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buf, *p_buflen); if (mark_name == NULL) { Py_DECREF(result); return NULL; @@ -3107,7 +3146,7 @@ load_next(PyObject *mod, PyObject *altmod, return NULL; } Py_DECREF(mark_name); - Py_UNICODE_strncpy(buf, nameuni, len); + Py_UCS4_strncpy(buf, nameuni, len); buf[len] = '\0'; *p_buflen = len; } @@ -3125,7 +3164,8 @@ load_next(PyObject *mod, PyObject *altmod, } if (dot != NULL) { - *p_outputname = PyUnicode_FromUnicode(dot+1, Py_UNICODE_strlen(dot+1)); + *p_outputname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + dot+1, Py_UCS4_strlen(dot+1)); if (*p_outputname == NULL) { Py_DECREF(result); return NULL; @@ -3166,7 +3206,7 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *name, Py_DECREF(item); return 0; } - if (PyUnicode_AS_UNICODE(item)[0] == '*') { + if (PyUnicode_READ_CHAR(item, 0) == '*') { PyObject *all; Py_DECREF(item); /* See if the package defines __all__ */ @@ -3304,7 +3344,7 @@ PyImport_ReloadModule(PyObject *m) PyObject *modules = PyImport_GetModuleDict(); PyObject *path_list = NULL, *loader = NULL, *existing_m = NULL; PyObject *nameobj, *bufobj, *subnameobj; - Py_UNICODE *name, *subname; + Py_UCS4 *name = NULL, *subname; struct filedescr *fdp; FILE *fp = NULL; PyObject *newm = NULL; @@ -3321,7 +3361,7 @@ PyImport_ReloadModule(PyObject *m) return NULL; } nameobj = PyModule_GetNameObject(m); - if (nameobj == NULL) + if (nameobj == NULL || PyUnicode_READY(nameobj) == -1) return NULL; if (m != PyDict_GetItem(modules, nameobj)) { PyErr_Format(PyExc_ImportError, @@ -3343,8 +3383,12 @@ PyImport_ReloadModule(PyObject *m) return NULL; } - name = PyUnicode_AS_UNICODE(nameobj); - subname = Py_UNICODE_strrchr(name, '.'); + name = PyUnicode_AsUCS4Copy(nameobj); + if (!name) { + Py_DECREF(nameobj); + return NULL; + } + subname = Py_UCS4_strrchr(name, '.'); if (subname == NULL) { Py_INCREF(nameobj); subnameobj = nameobj; @@ -3353,7 +3397,8 @@ PyImport_ReloadModule(PyObject *m) PyObject *parentname, *parent; Py_ssize_t len; len = subname - name; - parentname = PyUnicode_FromUnicode(name, len); + parentname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + name, len); if (parentname == NULL) { goto error; } @@ -3370,8 +3415,9 @@ PyImport_ReloadModule(PyObject *m) if (path_list == NULL) PyErr_Clear(); subname++; - len = PyUnicode_GET_SIZE(nameobj) - (len + 1); - subnameobj = PyUnicode_FromUnicode(subname, len); + len = PyUnicode_GET_LENGTH(nameobj) - (len + 1); + subnameobj = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + subname, len); } if (subnameobj == NULL) goto error; @@ -3403,6 +3449,7 @@ PyImport_ReloadModule(PyObject *m) error: imp_modules_reloading_clear(); Py_DECREF(nameobj); + PyMem_Free(name); return newm; } @@ -3910,9 +3957,7 @@ imp_cache_from_source(PyObject *self, PyObject *args, PyObject *kws) return NULL; } - cpathname = make_compiled_pathname( - PyUnicode_AS_UNICODE(pathname), - debug); + cpathname = make_compiled_pathname(pathname, debug); Py_DECREF(pathname); if (cpathname == NULL) { @@ -4105,7 +4150,7 @@ NullImporter_init(NullImporter *self, PyObject *args, PyObject *kwds) &pathobj)) return -1; - if (PyUnicode_GET_SIZE(pathobj) == 0) { + if (PyUnicode_GET_LENGTH(pathobj) == 0) { PyErr_SetString(PyExc_ImportError, "empty pathname"); return -1; } -- cgit v1.2.1 From 593041f3e6ac869b37c7da4355b69b1adc173b37 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Wed, 28 Sep 2011 21:49:49 +0200 Subject: Rename new macros to conform to naming rules (function macros have "Py" prefix, not "PY"). --- Python/import.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 4afdb7058c..9f38061723 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1784,7 +1784,7 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, return 0; len = PyUnicode_GET_LENGTH(path_unicode); - if (!PyUnicode_AsUCS4(path_unicode, buf, PY_ARRAY_LENGTH(buf), 1)) { + if (!PyUnicode_AsUCS4(path_unicode, buf, Py_ARRAY_LENGTH(buf), 1)) { Py_DECREF(path_unicode); PyErr_Clear(); return 0; @@ -1828,7 +1828,7 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, #endif ) buf[len++] = SEP; - if (!PyUnicode_AsUCS4(name, buf+len, PY_ARRAY_LENGTH(buf)-len, 1)) { + if (!PyUnicode_AsUCS4(name, buf+len, Py_ARRAY_LENGTH(buf)-len, 1)) { PyErr_Clear(); return 0; } @@ -2787,7 +2787,7 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, if (PyUnicode_READY(parent_name)) return NULL; buflen = PyUnicode_GET_LENGTH(parent_name); - if (!PyUnicode_AsUCS4(parent_name, buf, PY_ARRAY_LENGTH(buf), 1)) { + if (!PyUnicode_AsUCS4(parent_name, buf, Py_ARRAY_LENGTH(buf), 1)) { Py_DECREF(parent_name); PyErr_SetString(PyExc_ValueError, "Module name too long"); -- cgit v1.2.1 From 2ab1e534bf781097c59588983df13f737c912576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sat, 1 Oct 2011 16:35:40 +0200 Subject: Issue 13085: Fix some memory leaks. Patch by Stefan Krah. --- Python/import.c | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 9f38061723..901b733480 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1564,8 +1564,10 @@ get_sourcefile(PyObject *filename) if (py == NULL) goto error; - if (_Py_stat(py, &statbuf) == 0 && S_ISREG(statbuf.st_mode)) + if (_Py_stat(py, &statbuf) == 0 && S_ISREG(statbuf.st_mode)) { + PyMem_Free(fileuni); return py; + } Py_DECREF(py); goto unchanged; @@ -3074,7 +3076,7 @@ load_next(PyObject *mod, PyObject *altmod, Py_ssize_t len; Py_UCS4 *p; PyObject *fullname, *name, *result, *mark_name; - const Py_UCS4 *nameuni; + Py_UCS4 *nameuni; *p_outputname = NULL; @@ -3095,7 +3097,7 @@ load_next(PyObject *mod, PyObject *altmod, if (len == 0) { PyErr_SetString(PyExc_ValueError, "Empty module name"); - return NULL; + goto error; } } else @@ -3104,7 +3106,7 @@ load_next(PyObject *mod, PyObject *altmod, if (*p_buflen+len+1 >= bufsize) { PyErr_SetString(PyExc_ValueError, "Module name too long"); - return NULL; + goto error; } p = buf + *p_buflen; @@ -3119,12 +3121,12 @@ load_next(PyObject *mod, PyObject *altmod, fullname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, *p_buflen); if (fullname == NULL) - return NULL; + goto error; name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, p, len); if (name == NULL) { Py_DECREF(fullname); - return NULL; + goto error; } result = import_submodule(mod, name, fullname); Py_DECREF(fullname); @@ -3138,12 +3140,12 @@ load_next(PyObject *mod, PyObject *altmod, buf, *p_buflen); if (mark_name == NULL) { Py_DECREF(result); - return NULL; + goto error; } if (mark_miss(mark_name) != 0) { Py_DECREF(result); Py_DECREF(mark_name); - return NULL; + goto error; } Py_DECREF(mark_name); Py_UCS4_strncpy(buf, nameuni, len); @@ -3154,13 +3156,13 @@ load_next(PyObject *mod, PyObject *altmod, else Py_DECREF(name); if (result == NULL) - return NULL; + goto error; if (result == Py_None) { Py_DECREF(result); PyErr_Format(PyExc_ImportError, "No module named %R", inputname); - return NULL; + goto error; } if (dot != NULL) { @@ -3168,11 +3170,17 @@ load_next(PyObject *mod, PyObject *altmod, dot+1, Py_UCS4_strlen(dot+1)); if (*p_outputname == NULL) { Py_DECREF(result); - return NULL; + goto error; } } +out: + PyMem_Free(nameuni); return result; + +error: + PyMem_Free(nameuni); + return NULL; } static int -- cgit v1.2.1 From 7a98f5f5ced85e78dc79169bfaeaa1b3adfa0596 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 2 Oct 2011 13:19:16 -0400 Subject: remove unused label --- Python/import.c | 1 - 1 file changed, 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 901b733480..9f094c0da0 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3174,7 +3174,6 @@ load_next(PyObject *mod, PyObject *altmod, } } -out: PyMem_Free(nameuni); return result; -- cgit v1.2.1 From 0aeb34c31cc7bfc0c0fd5595b017e3f9dc0a93ef Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 6 Oct 2011 02:39:42 +0200 Subject: Fix find_module_path(): make the string ready --- Python/import.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 9f094c0da0..5f84ac2f74 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1785,6 +1785,9 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, else return 0; + if (PyUnicode_READY(path_unicode)) + return -1; + len = PyUnicode_GET_LENGTH(path_unicode); if (!PyUnicode_AsUCS4(path_unicode, buf, Py_ARRAY_LENGTH(buf), 1)) { Py_DECREF(path_unicode); -- cgit v1.2.1 From dcacb24f386ae1b107462181af42b6826ec15fb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 9 Oct 2011 10:38:36 +0200 Subject: =?UTF-8?q?Add=20API=20for=20static=20strings,=20primarily=20good?= =?UTF-8?q?=20for=20identifiers.=20Thanks=20to=20Konrad=20Sch=C3=B6bel=20a?= =?UTF-8?q?nd=20Jasper=20Schulz=20for=20helping=20with=20the=20mass-editin?= =?UTF-8?q?g.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Python/import.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 5f84ac2f74..6eca90a0f7 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1801,6 +1801,7 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, /* sys.path_hooks import hook */ if (p_loader != NULL) { + _Py_identifier(find_module); PyObject *importer; importer = get_path_importer(path_importer_cache, @@ -1811,8 +1812,8 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, /* Note: importer is a borrowed reference */ if (importer != Py_None) { PyObject *loader; - loader = PyObject_CallMethod(importer, - "find_module", "O", fullname); + loader = _PyObject_CallMethodId(importer, + &PyId_find_module, "O", fullname); if (loader == NULL) return -1; /* error */ if (loader != Py_None) { @@ -2030,6 +2031,7 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, /* sys.meta_path import hook */ if (p_loader != NULL) { + _Py_identifier(find_module); PyObject *meta_path; meta_path = PySys_GetObject("meta_path"); @@ -2044,7 +2046,7 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, for (i = 0; i < npath; i++) { PyObject *loader; PyObject *hook = PyList_GetItem(meta_path, i); - loader = PyObject_CallMethod(hook, "find_module", + loader = _PyObject_CallMethodId(hook, &PyId_find_module, "OO", fullname, search_path_list != NULL ? search_path_list : Py_None); @@ -2454,12 +2456,13 @@ load_module(PyObject *name, FILE *fp, PyObject *pathname, int type, PyObject *lo break; case IMP_HOOK: { + _Py_identifier(load_module); if (loader == NULL) { PyErr_SetString(PyExc_ImportError, "import hook without loader"); return NULL; } - m = PyObject_CallMethod(loader, "load_module", "O", name); + m = _PyObject_CallMethodId(loader, &PyId_load_module, "O", name); break; } -- cgit v1.2.1 From c3eb93fe1ce5f5ff2666fb9f34a6fd7c46279554 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Mon, 10 Oct 2011 18:11:30 +0200 Subject: Use identifier API for PyObject_GetAttrString. --- Python/import.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 6eca90a0f7..3d6c6ed1d4 100644 --- a/Python/import.c +++ b/Python/import.c @@ -154,7 +154,7 @@ static const struct filedescr _PyImport_StandardFiletab[] = { }; static PyObject *initstr = NULL; - +_Py_identifier(__path__); /* Initialize things */ @@ -248,8 +248,9 @@ _PyImportHooks_Init(void) PySys_WriteStderr("# can't import zipimport\n"); } else { - PyObject *zipimporter = PyObject_GetAttrString(zimpimport, - "zipimporter"); + _Py_identifier(zipimporter); + PyObject *zipimporter = _PyObject_GetAttrId(zimpimport, + &PyId_zipimporter); Py_DECREF(zimpimport); if (zipimporter == NULL) { PyErr_Clear(); /* No zipimporter object -- okay */ @@ -3203,7 +3204,7 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *name, PyObject *fullname; Py_ssize_t fromlist_len; - if (!PyObject_HasAttrString(mod, "__path__")) + if (!_PyObject_HasAttrId(mod, &PyId___path__)) return 1; fromlist_len = PySequence_Size(fromlist); @@ -3221,11 +3222,12 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *name, } if (PyUnicode_READ_CHAR(item, 0) == '*') { PyObject *all; + _Py_identifier(__all__); Py_DECREF(item); /* See if the package defines __all__ */ if (recursive) continue; /* Avoid endless recursion */ - all = PyObject_GetAttrString(mod, "__all__"); + all = _PyObject_GetAttrId(mod, &PyId___all__); if (all == NULL) PyErr_Clear(); else { @@ -3313,7 +3315,7 @@ import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) if (mod == Py_None) path_list = NULL; else { - path_list = PyObject_GetAttrString(mod, "__path__"); + path_list = _PyObject_GetAttrId(mod, &PyId___path__); if (path_list == NULL) { PyErr_Clear(); Py_INCREF(Py_None); @@ -3424,7 +3426,7 @@ PyImport_ReloadModule(PyObject *m) goto error; } Py_DECREF(parentname); - path_list = PyObject_GetAttrString(parent, "__path__"); + path_list = _PyObject_GetAttrId(parent, &PyId___path__); if (path_list == NULL) PyErr_Clear(); subname++; -- cgit v1.2.1 From 9fde12ef0a3cbe4260897de13f054ec630456992 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 11 Oct 2011 21:55:01 +0200 Subject: Use PyUnicode_AsUnicodeAndSize() instead of PyUnicode_GET_SIZE() --- Python/import.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 3d6c6ed1d4..6f564a65da 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2282,6 +2282,8 @@ case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) WIN32_FIND_DATAW data; HANDLE h; int cmp; + wchar_t *wname; + Py_ssizet wname_len; if (Py_GETENV("PYTHONCASEOK") != NULL) return 1; @@ -2294,9 +2296,12 @@ case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) return 0; } FindClose(h); - cmp = wcsncmp(data.cFileName, - PyUnicode_AS_UNICODE(name), - PyUnicode_GET_SIZE(name)); + + wname = PyUnicode_AsUnicodeAndSize(name, &wname_len); + if (wname == NULL) + return -1; + + cmp = wcsncmp(data.cFileName, wname, wname_len); return cmp == 0; #elif defined(USE_CASE_OK_BYTES) int match; -- cgit v1.2.1 From f716b4e95cf727dc9fe4a592c38e86b73ca16c74 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 11 Oct 2011 22:27:13 +0200 Subject: Fix typo in import.c --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 6f564a65da..d937cc165a 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2283,7 +2283,7 @@ case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) HANDLE h; int cmp; wchar_t *wname; - Py_ssizet wname_len; + Py_ssize_t wname_len; if (Py_GETENV("PYTHONCASEOK") != NULL) return 1; -- cgit v1.2.1 From f5ad3b280b43227eb0e3fa63d89490a58ba9c28b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Fri, 14 Oct 2011 10:20:37 +0200 Subject: Rename _Py_identifier to _Py_IDENTIFIER. --- Python/import.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index d937cc165a..6e3e0f181e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -154,7 +154,7 @@ static const struct filedescr _PyImport_StandardFiletab[] = { }; static PyObject *initstr = NULL; -_Py_identifier(__path__); +_Py_IDENTIFIER(__path__); /* Initialize things */ @@ -248,7 +248,7 @@ _PyImportHooks_Init(void) PySys_WriteStderr("# can't import zipimport\n"); } else { - _Py_identifier(zipimporter); + _Py_IDENTIFIER(zipimporter); PyObject *zipimporter = _PyObject_GetAttrId(zimpimport, &PyId_zipimporter); Py_DECREF(zimpimport); @@ -1802,7 +1802,7 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, /* sys.path_hooks import hook */ if (p_loader != NULL) { - _Py_identifier(find_module); + _Py_IDENTIFIER(find_module); PyObject *importer; importer = get_path_importer(path_importer_cache, @@ -2032,7 +2032,7 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, /* sys.meta_path import hook */ if (p_loader != NULL) { - _Py_identifier(find_module); + _Py_IDENTIFIER(find_module); PyObject *meta_path; meta_path = PySys_GetObject("meta_path"); @@ -2462,7 +2462,7 @@ load_module(PyObject *name, FILE *fp, PyObject *pathname, int type, PyObject *lo break; case IMP_HOOK: { - _Py_identifier(load_module); + _Py_IDENTIFIER(load_module); if (loader == NULL) { PyErr_SetString(PyExc_ImportError, "import hook without loader"); @@ -3227,7 +3227,7 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *name, } if (PyUnicode_READ_CHAR(item, 0) == '*') { PyObject *all; - _Py_identifier(__all__); + _Py_IDENTIFIER(__all__); Py_DECREF(item); /* See if the package defines __all__ */ if (recursive) -- cgit v1.2.1 From 5263d9e0be0bf742276426378f96f8b6e6687872 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 17 Oct 2011 19:28:44 +0200 Subject: Issue #13146: Writing a pyc file is now atomic under POSIX. --- Python/import.c | 45 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 6e3e0f181e..0db7e43d29 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1284,7 +1284,8 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, #ifdef MS_WINDOWS int fd; #else - PyObject *cpathbytes; + PyObject *cpathbytes, *cpathbytes_tmp; + Py_ssize_t cpathbytes_len; #endif PyObject *dirname; Py_UCS4 *dirsep; @@ -1345,13 +1346,25 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, else fp = NULL; #else + /* Under POSIX, we first write to a tmp file and then take advantage + of atomic renaming. */ cpathbytes = PyUnicode_EncodeFSDefault(cpathname); if (cpathbytes == NULL) { PyErr_Clear(); return; } + cpathbytes_len = PyBytes_GET_SIZE(cpathbytes); + cpathbytes_tmp = PyBytes_FromStringAndSize(NULL, cpathbytes_len + 4); + if (cpathbytes_tmp == NULL) { + Py_DECREF(cpathbytes); + PyErr_Clear(); + return; + } + memcpy(PyBytes_AS_STRING(cpathbytes_tmp), PyBytes_AS_STRING(cpathbytes), + cpathbytes_len); + memcpy(PyBytes_AS_STRING(cpathbytes_tmp) + cpathbytes_len, ".tmp", 4); - fp = open_exclusive(PyBytes_AS_STRING(cpathbytes), mode); + fp = open_exclusive(PyBytes_AS_STRING(cpathbytes_tmp), mode); #endif if (fp == NULL) { if (Py_VerboseFlag) @@ -1359,6 +1372,7 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, "# can't create %R\n", cpathname); #ifndef MS_WINDOWS Py_DECREF(cpathbytes); + Py_DECREF(cpathbytes_tmp); #endif return; } @@ -1366,6 +1380,11 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, /* First write a 0 for mtime */ PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION); PyMarshal_WriteObjectToFile((PyObject *)co, fp, Py_MARSHAL_VERSION); + fflush(fp); + /* Now write the true mtime */ + fseek(fp, 4L, 0); + assert(mtime < LONG_MAX); + PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION); if (fflush(fp) != 0 || ferror(fp)) { if (Py_VerboseFlag) PySys_FormatStderr("# can't write %R\n", cpathname); @@ -1374,20 +1393,28 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, #ifdef MS_WINDOWS (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname)); #else - (void) unlink(PyBytes_AS_STRING(cpathbytes)); + (void) unlink(PyBytes_AS_STRING(cpathbytes_tmp)); Py_DECREF(cpathbytes); + Py_DECREF(cpathbytes_tmp); #endif return; } + fclose(fp); + /* Under POSIX, do an atomic rename */ #ifndef MS_WINDOWS + if (rename(PyBytes_AS_STRING(cpathbytes_tmp), + PyBytes_AS_STRING(cpathbytes))) { + if (Py_VerboseFlag) + PySys_FormatStderr("# can't write %R\n", cpathname); + /* Don't keep tmp file */ + unlink(PyBytes_AS_STRING(cpathbytes_tmp)); + Py_DECREF(cpathbytes); + Py_DECREF(cpathbytes_tmp); + return; + } Py_DECREF(cpathbytes); + Py_DECREF(cpathbytes_tmp); #endif - /* Now write the true mtime */ - fseek(fp, 4L, 0); - assert(mtime < LONG_MAX); - PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION); - fflush(fp); - fclose(fp); if (Py_VerboseFlag) PySys_FormatStderr("# wrote %R\n", cpathname); } -- cgit v1.2.1 From 787a4931e7a65741deb81c532dfb184af13a0e20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 23 Oct 2011 17:29:08 +0200 Subject: Reformulate make_compiled_pathname in terms of unicode objects. --- Python/import.c | 177 +++++++++++++++++++------------------------------------- 1 file changed, 61 insertions(+), 116 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 0db7e43d29..1101c920f5 100644 --- a/Python/import.c +++ b/Python/import.c @@ -904,6 +904,25 @@ rightmost_sep(Py_UCS4 *s) return found; } +/* Like rightmost_sep, but operate on unicode objects. */ +static Py_ssize_t +rightmost_sep_obj(PyObject* o) +{ + Py_ssize_t found, i; + Py_UCS4 c; + for (found = -1, i = 0; i < PyUnicode_GET_LENGTH(o); i++) { + c = PyUnicode_READ_CHAR(o, i); + if (c == SEP +#ifdef ALTSEP + || c == ALTSEP +#endif + ) + { + found = i; + } + } + return found; +} /* Given a pathname for a Python source file, fill a buffer with the pathname for the corresponding compiled file. Return the pathname @@ -915,123 +934,49 @@ rightmost_sep(Py_UCS4 *s) static PyObject* make_compiled_pathname(PyObject *pathstr, int debug) { - Py_UCS4 *pathname; - Py_UCS4 buf[MAXPATHLEN]; - size_t buflen = (size_t)MAXPATHLEN; - size_t len; - size_t i, save; - Py_UCS4 *pos; - int sep = SEP; - - pathname = PyUnicode_AsUCS4Copy(pathstr); - if (!pathname) - return NULL; - len = Py_UCS4_strlen(pathname); - - /* Sanity check that the buffer has roughly enough space to hold what - will eventually be the full path to the compiled file. The 5 extra - bytes include the slash afer __pycache__, the two extra dots, the - extra trailing character ('c' or 'o') and null. This isn't exact - because the contents of the buffer can affect how many actual - characters of the string get into the buffer. We'll do a final - sanity check before writing the extension to ensure we do not - overflow the buffer. - */ - if (len + Py_UCS4_strlen(CACHEDIR_UNICODE) + Py_UCS4_strlen(PYC_TAG_UNICODE) + 5 > buflen) { - PyMem_Free(pathname); - return NULL; - } - - /* Find the last path separator and copy everything from the start of - the source string up to and including the separator. - */ - pos = rightmost_sep(pathname); - if (pos == NULL) { - i = 0; - } - else { - sep = *pos; - i = pos - pathname + 1; - Py_UCS4_strncpy(buf, pathname, i); - } - - save = i; - buf[i++] = '\0'; - /* Add __pycache__/ */ - Py_UCS4_strcat(buf, CACHEDIR_UNICODE); - i += Py_UCS4_strlen(CACHEDIR_UNICODE) - 1; - buf[i++] = sep; - buf[i] = '\0'; - /* Add the base filename, but remove the .py or .pyw extension, since - the tag name must go before the extension. - */ - Py_UCS4_strcat(buf, pathname + save); - pos = Py_UCS4_strrchr(buf + i, '.'); - if (pos != NULL) - *++pos = '\0'; - - /* pathname is not used from here on. */ - PyMem_Free(pathname); - - Py_UCS4_strcat(buf, PYC_TAG_UNICODE); - /* The length test above assumes that we're only adding one character - to the end of what would normally be the extension. What if there - is no extension, or the string ends in '.' or '.p', and otherwise - fills the buffer? By appending 4 more characters onto the string - here, we could overrun the buffer. - - As a simple example, let's say buflen=32 and the input string is - 'xxx.py'. strlen() would be 6 and the test above would yield: - - (6 + 11 + 10 + 5 == 32) > 32 - - which is false and so the name mangling would continue. This would - be fine because we'd end up with this string in buf: - - __pycache__/xxx.cpython-32.pyc\0 - - strlen(of that) == 30 + the nul fits inside a 32 character buffer. - We can even handle an input string of say 'xxxxx' above because - that's (5 + 11 + 10 + 5 == 31) > 32 which is also false. Name - mangling that yields: - - __pycache__/xxxxxcpython-32.pyc\0 - - which is 32 characters including the nul, and thus fits in the - buffer. However, an input string of 'xxxxxx' would yield a result - string of: - - __pycache__/xxxxxxcpython-32.pyc\0 - - which is 33 characters long (including the nul), thus overflowing - the buffer, even though the first test would fail, i.e.: the input - string is also 6 characters long, so 32 > 32 is false. - - The reason the first test fails but we still overflow the buffer is - that the test above only expects to add one extra character to be - added to the extension, and here we're adding three (pyc). We - don't add the first dot, so that reclaims one of expected - positions, leaving us overflowing by 1 byte (3 extra - 1 reclaimed - dot - 1 expected extra == 1 overflowed). - - The best we can do is ensure that we still have enough room in the - target buffer before we write the extension. Because it's always - only the extension that can cause the overflow, and never the other - path bytes we've written, it's sufficient to just do one more test - here. Still, the assertion that follows can't hurt. - */ -#if 0 - printf("strlen(buf): %d; buflen: %d\n", (int)strlen(buf), (int)buflen); -#endif - len = Py_UCS4_strlen(buf); - if (len + 5 > buflen) + PyObject *result; + Py_ssize_t fname, ext, len, i, pos, taglen; + Py_ssize_t pycache_len = sizeof("__pycache__/") - 1; + int kind; + void *data; + + /* Compute the output string size. */ + len = PyUnicode_GET_LENGTH(pathstr); + /* If there is no separator, this returns -1, so + lastsep will be 0. */ + fname = rightmost_sep_obj(pathstr) + 1; + ext = fname - 1; + for(i = fname; i < len; i++) + if (PyUnicode_READ_CHAR(pathstr, i) == '.') + ext = i + 1; + if (ext < fname) + /* No dot in filename; use entire filename */ + ext = len; + + /* result = pathstr[:fname] + "__pycache__" + SEP + + pathstr[fname:ext] + tag + ".py[co]" */ + taglen = strlen(pyc_tag); + result = PyUnicode_New(ext + pycache_len + taglen + 4, + PyUnicode_MAX_CHAR_VALUE(pathstr)); + if (!result) return NULL; - buf[len] = '.'; len++; - buf[len] = 'p'; len++; - buf[len] = 'y'; len++; - buf[len] = debug ? 'c' : 'o'; len++; - assert(len <= buflen); - return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, len); + kind = PyUnicode_KIND(result); + data = PyUnicode_DATA(result); + PyUnicode_CopyCharacters(result, 0, pathstr, 0, fname); + pos = fname; + for (i = 0; i < pycache_len - 1; i++) + PyUnicode_WRITE(kind, data, pos++, "__pycache__"[i]); + PyUnicode_WRITE(kind, data, pos++, SEP); + PyUnicode_CopyCharacters(result, pos, pathstr, + fname, ext - fname); + pos += ext - fname; + for (i = 0; pyc_tag[i]; i++) + PyUnicode_WRITE(kind, data, pos++, pyc_tag[i]); + PyUnicode_WRITE(kind, data, pos++, '.'); + PyUnicode_WRITE(kind, data, pos++, 'p'); + PyUnicode_WRITE(kind, data, pos++, 'y'); + PyUnicode_WRITE(kind, data, pos++, debug ? 'c' : 'o'); + return result; } -- cgit v1.2.1 From 17bab0f4a5891ac06360ae32cc1b1f9ab746cf52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 23 Oct 2011 17:35:46 +0200 Subject: Add ready checks for make_compiled_pathname. --- Python/import.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 1101c920f5..5dcaf3a40e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -929,7 +929,10 @@ rightmost_sep_obj(PyObject* o) for the compiled file, or NULL if there's no space in the buffer. Doesn't set an exception. - foo.py -> __pycache__/foo..pyc */ + foo.py -> __pycache__/foo..pyc + + pathstr is assumed to be "ready". +*/ static PyObject* make_compiled_pathname(PyObject *pathstr, int debug) @@ -1458,6 +1461,8 @@ load_source_module(PyObject *name, PyObject *pathname, FILE *fp) goto error; } #endif + if (PyUnicode_READY(pathname) < 0) + return NULL; cpathname = make_compiled_pathname(pathname, !Py_OptimizeFlag); if (cpathname != NULL) @@ -3949,6 +3954,9 @@ imp_cache_from_source(PyObject *self, PyObject *args, PyObject *kws) return NULL; } + if (PyUnicode_READY(pathname) < 0) + return NULL; + cpathname = make_compiled_pathname(pathname, debug); Py_DECREF(pathname); -- cgit v1.2.1 From ff4c4b6f318abc223cbae8920ee2286d111d8bce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 23 Oct 2011 18:05:06 +0200 Subject: Rewrite make_source_pathname using Unicode API. --- Python/import.c | 79 ++++++++++++++++++++++++++++++--------------------------- 1 file changed, 41 insertions(+), 38 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 5dcaf3a40e..a20871efcc 100644 --- a/Python/import.c +++ b/Python/import.c @@ -906,11 +906,11 @@ rightmost_sep(Py_UCS4 *s) /* Like rightmost_sep, but operate on unicode objects. */ static Py_ssize_t -rightmost_sep_obj(PyObject* o) +rightmost_sep_obj(PyObject* o, Py_ssize_t start, Py_ssize_t end) { Py_ssize_t found, i; Py_UCS4 c; - for (found = -1, i = 0; i < PyUnicode_GET_LENGTH(o); i++) { + for (found = -1, i = start; i < end; i++) { c = PyUnicode_READ_CHAR(o, i); if (c == SEP #ifdef ALTSEP @@ -947,7 +947,7 @@ make_compiled_pathname(PyObject *pathstr, int debug) len = PyUnicode_GET_LENGTH(pathstr); /* If there is no separator, this returns -1, so lastsep will be 0. */ - fname = rightmost_sep_obj(pathstr) + 1; + fname = rightmost_sep_obj(pathstr, 0, len) + 1; ext = fname - 1; for(i = fname; i < len; i++) if (PyUnicode_READ_CHAR(pathstr, i) == '.') @@ -992,63 +992,66 @@ make_compiled_pathname(PyObject *pathstr, int debug) (...)/__pycache__/foo..pyc -> (...)/foo.py */ static PyObject* -make_source_pathname(PyObject *pathobj) +make_source_pathname(PyObject *path) { - Py_UCS4 buf[MAXPATHLEN]; - Py_UCS4 *pathname; - Py_UCS4 *left, *right, *dot0, *dot1, sep; - size_t i, j; + Py_ssize_t left, right, dot0, dot1, len; + Py_ssize_t i, j; + PyObject *result; + int kind; + void *data; - if (PyUnicode_GET_LENGTH(pathobj) > MAXPATHLEN) - return NULL; - pathname = PyUnicode_AsUCS4Copy(pathobj); - if (!pathname) + len = PyUnicode_GET_LENGTH(path); + if (len > MAXPATHLEN) return NULL; /* Look back two slashes from the end. In between these two slashes must be the string __pycache__ or this is not a PEP 3147 style path. It's possible for there to be only one slash. */ - right = rightmost_sep(pathname); - if (right == NULL) + right = rightmost_sep_obj(path, 0, len); + if (right == -1) return NULL; - sep = *right; - *right = '\0'; - left = rightmost_sep(pathname); - *right = sep; - if (left == NULL) - left = pathname; + left = rightmost_sep_obj(path, 0, right); + if (left == -1) + left = 0; else left++; - if (right-left != Py_UCS4_strlen(CACHEDIR_UNICODE) || - Py_UCS4_strncmp(left, CACHEDIR_UNICODE, right-left) != 0) - goto error; + if (right-left != sizeof(CACHEDIR)-1) + return NULL; + for (i = 0; i < sizeof(CACHEDIR)-1; i++) + if (PyUnicode_READ_CHAR(path, left+i) != CACHEDIR[i]) + return NULL; /* Now verify that the path component to the right of the last slash has two dots in it. */ - if ((dot0 = Py_UCS4_strchr(right + 1, '.')) == NULL) - goto error; - if ((dot1 = Py_UCS4_strchr(dot0 + 1, '.')) == NULL) - goto error; + dot0 = PyUnicode_FindChar(path, '.', right+1, len, 1); + if (dot0 < 0) + return NULL; + dot1 = PyUnicode_FindChar(path, '.', dot0+1, len, 1); + if (dot1 < 0) + return NULL; /* Too many dots? */ - if (Py_UCS4_strchr(dot1 + 1, '.') != NULL) - goto error; + if (PyUnicode_FindChar(path, '.', dot1+1, len, 1) != -1) + return NULL; /* This is a PEP 3147 path. Start by copying everything from the start of pathname up to and including the leftmost slash. Then copy the file's basename, removing the magic tag and adding a .py suffix. */ - Py_UCS4_strncpy(buf, pathname, (i=left-pathname)); - Py_UCS4_strncpy(buf+i, right+1, (j=dot0-right)); - buf[i+j] = 'p'; - buf[i+j+1] = 'y'; - PyMem_Free(pathname); - return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, i+j+2); - error: - PyMem_Free(pathname); - return NULL; + result = PyUnicode_New(left + (dot0-right) + 2, + PyUnicode_MAX_CHAR_VALUE(path)); + if (!result) + return NULL; + kind = PyUnicode_KIND(result); + data = PyUnicode_DATA(result); + PyUnicode_CopyCharacters(result, 0, path, 0, (i = left)); + PyUnicode_CopyCharacters(result, left, path, right+1, + (j = dot0-right)); + PyUnicode_WRITE(kind, data, i+j, 'p'); + PyUnicode_WRITE(kind, data, i+j+1, 'y'); + return result; } /* Given a pathname for a Python source file, its time of last -- cgit v1.2.1 From c7e9dd65640704b9ef0174a6c5fc9d82fbebf492 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 23 Oct 2011 18:08:20 +0200 Subject: Drop unused macros. Use CACHEDIR consistently. --- Python/import.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index a20871efcc..ebaaeacebd 100644 --- a/Python/import.c +++ b/Python/import.c @@ -108,8 +108,8 @@ typedef unsigned short mode_t; /* MAGIC must change whenever the bytecode emitted by the compiler may no longer be understood by older implementations of the eval loop (usually due to the addition of new opcodes) - TAG and PYC_TAG_UNICODE must change for each major Python release. The magic - number will take care of any bytecode changes that occur during development. + TAG must change for each major Python release. The magic number will take + care of any bytecode changes that occur during development. */ #define QUOTE(arg) #arg #define STRIFY(name) QUOTE(name) @@ -118,13 +118,9 @@ typedef unsigned short mode_t; #define MAGIC (3190 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" -static const Py_UCS4 CACHEDIR_UNICODE[] = { - '_', '_', 'p', 'y', 'c', 'a', 'c', 'h', 'e', '_', '_', '\0'}; /* Current magic word and string tag as globals. */ static long pyc_magic = MAGIC; static const char *pyc_tag = TAG; -static const Py_UCS4 PYC_TAG_UNICODE[] = { - 'c', 'p', 'y', 't', 'h', 'o', 'n', '-', PY_MAJOR_VERSION + 48, PY_MINOR_VERSION + 48, '\0'}; #undef QUOTE #undef STRIFY #undef MAJOR @@ -939,7 +935,7 @@ make_compiled_pathname(PyObject *pathstr, int debug) { PyObject *result; Py_ssize_t fname, ext, len, i, pos, taglen; - Py_ssize_t pycache_len = sizeof("__pycache__/") - 1; + Py_ssize_t pycache_len = sizeof(CACHEDIR) - 1; int kind; void *data; @@ -968,7 +964,7 @@ make_compiled_pathname(PyObject *pathstr, int debug) PyUnicode_CopyCharacters(result, 0, pathstr, 0, fname); pos = fname; for (i = 0; i < pycache_len - 1; i++) - PyUnicode_WRITE(kind, data, pos++, "__pycache__"[i]); + PyUnicode_WRITE(kind, data, pos++, CACHEDIR[i]); PyUnicode_WRITE(kind, data, pos++, SEP); PyUnicode_CopyCharacters(result, pos, pathstr, fname, ext - fname); -- cgit v1.2.1 From 012400c9fe4281ff97e7dee256cd2a05649a54ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 23 Oct 2011 18:41:56 +0200 Subject: Fix off-by-one error. --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index ebaaeacebd..736ce7717c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -955,7 +955,7 @@ make_compiled_pathname(PyObject *pathstr, int debug) /* result = pathstr[:fname] + "__pycache__" + SEP + pathstr[fname:ext] + tag + ".py[co]" */ taglen = strlen(pyc_tag); - result = PyUnicode_New(ext + pycache_len + taglen + 4, + result = PyUnicode_New(ext + pycache_len + 1 + taglen + 4, PyUnicode_MAX_CHAR_VALUE(pathstr)); if (!result) return NULL; @@ -963,7 +963,7 @@ make_compiled_pathname(PyObject *pathstr, int debug) data = PyUnicode_DATA(result); PyUnicode_CopyCharacters(result, 0, pathstr, 0, fname); pos = fname; - for (i = 0; i < pycache_len - 1; i++) + for (i = 0; i < pycache_len; i++) PyUnicode_WRITE(kind, data, pos++, CACHEDIR[i]); PyUnicode_WRITE(kind, data, pos++, SEP); PyUnicode_CopyCharacters(result, pos, pathstr, -- cgit v1.2.1 From 9d5ef173055b3857a637b716f029b0e81d172c29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 23 Oct 2011 18:45:16 +0200 Subject: Rewrite find_module_path using unicode API. --- Python/import.c | 83 +++++++++++++++++++++++++++++++++------------------------ 1 file changed, 48 insertions(+), 35 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 736ce7717c..e21ae39359 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1743,11 +1743,11 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, PyObject *path_hooks, PyObject *path_importer_cache, PyObject **p_path, PyObject **p_loader, struct filedescr **p_fd) { - Py_UCS4 buf[MAXPATHLEN+1]; - PyObject *path_unicode, *filename; - Py_ssize_t len; + PyObject *path_unicode, *filename = NULL; + Py_ssize_t len, pos; struct stat statbuf; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; + int result, addsep; if (PyUnicode_Check(path)) { Py_INCREF(path); @@ -1766,15 +1766,10 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, return -1; len = PyUnicode_GET_LENGTH(path_unicode); - if (!PyUnicode_AsUCS4(path_unicode, buf, Py_ARRAY_LENGTH(buf), 1)) { - Py_DECREF(path_unicode); - PyErr_Clear(); - return 0; + if (PyUnicode_FindChar(path_unicode, 0, 0, len, 1) != -1) { + result = 0; + goto out; /* path contains '\0' */ } - Py_DECREF(path_unicode); - - if (Py_UCS4_strlen(buf) != len) - return 0; /* path contains '\0' */ /* sys.path_hooks import hook */ if (p_loader != NULL) { @@ -1784,43 +1779,54 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, importer = get_path_importer(path_importer_cache, path_hooks, path); if (importer == NULL) { - return -1; + result = -1; + goto out; } /* Note: importer is a borrowed reference */ if (importer != Py_None) { PyObject *loader; loader = _PyObject_CallMethodId(importer, &PyId_find_module, "O", fullname); - if (loader == NULL) - return -1; /* error */ + if (loader == NULL) { + result = -1; /* error */ + goto out; + } if (loader != Py_None) { /* a loader was found */ *p_loader = loader; *p_fd = &importhookdescr; - return 2; + result = 2; + goto out; } Py_DECREF(loader); - return 0; + result = 0; + goto out; } } /* no hook was found, use builtin import */ - if (len > 0 && buf[len-1] != SEP + addsep = 0; + if (len > 0 && PyUnicode_READ_CHAR(path_unicode, len-1) != SEP #ifdef ALTSEP - && buf[len-1] != ALTSEP + && PyUnicode_READ_CHAR(path_unicode, len-1) != ALTSEP #endif ) - buf[len++] = SEP; - if (!PyUnicode_AsUCS4(name, buf+len, Py_ARRAY_LENGTH(buf)-len, 1)) { - PyErr_Clear(); - return 0; - } - len += PyUnicode_GET_LENGTH(name); - - filename = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - buf, len); - if (filename == NULL) - return -1; + addsep = 1; + filename = PyUnicode_New(len + PyUnicode_GET_LENGTH(name) + addsep, + Py_MAX(PyUnicode_MAX_CHAR_VALUE(path_unicode), + PyUnicode_MAX_CHAR_VALUE(name))); + if (filename == NULL) { + result = -1; + goto out; + } + PyUnicode_CopyCharacters(filename, 0, path_unicode, 0, len); + pos = len; + if (addsep) + PyUnicode_WRITE(PyUnicode_KIND(filename), + PyUnicode_DATA(filename), + pos++, SEP); + PyUnicode_CopyCharacters(filename, pos, name, 0, + PyUnicode_GET_LENGTH(name)); /* Check for package import (buf holds a directory name, and there's an __init__ module in that directory */ @@ -1832,14 +1838,16 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, match = case_ok(filename, 0, name); if (match < 0) { - Py_DECREF(filename); - return -1; + result = -1; + goto out; } if (match) { /* case matches */ if (find_init_module(filename)) { /* and has __init__.py */ *p_path = filename; + filename = NULL; *p_fd = &fd_package; - return 2; + result = 2; + goto out; } else { int err; @@ -1847,15 +1855,20 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, "Not importing directory %R: missing __init__.py", filename); if (err) { - Py_DECREF(filename); - return -1; + result = -1; + goto out; } } } } #endif *p_path = filename; - return 1; + filename = NULL; + result = 1; + out: + Py_DECREF(path_unicode); + Py_XDECREF(filename); + return result; } /* Find a module in search_path_list. For each path, try -- cgit v1.2.1 From 9ab76944255d2498c50789b1d0582aec1e417ad9 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 25 Oct 2011 00:21:02 +0200 Subject: Fix test_imp failure under Windows --- Python/import.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index e21ae39359..3842c63c2e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -938,12 +938,19 @@ make_compiled_pathname(PyObject *pathstr, int debug) Py_ssize_t pycache_len = sizeof(CACHEDIR) - 1; int kind; void *data; + Py_UCS4 lastsep; /* Compute the output string size. */ len = PyUnicode_GET_LENGTH(pathstr); /* If there is no separator, this returns -1, so - lastsep will be 0. */ + fname will be 0. */ fname = rightmost_sep_obj(pathstr, 0, len) + 1; + /* Windows: re-use the last separator character (/ or \\) when + appending the __pycache__ path. */ + if (fname > 0) + lastsep = PyUnicode_READ_CHAR(pathstr, fname -1); + else + lastsep = SEP; ext = fname - 1; for(i = fname; i < len; i++) if (PyUnicode_READ_CHAR(pathstr, i) == '.') @@ -965,7 +972,7 @@ make_compiled_pathname(PyObject *pathstr, int debug) pos = fname; for (i = 0; i < pycache_len; i++) PyUnicode_WRITE(kind, data, pos++, CACHEDIR[i]); - PyUnicode_WRITE(kind, data, pos++, SEP); + PyUnicode_WRITE(kind, data, pos++, lastsep); PyUnicode_CopyCharacters(result, pos, pathstr, fname, ext - fname); pos += ext - fname; -- cgit v1.2.1 From 7336fba07c5e34404a0d72c4b24e78a3d83ac1cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 30 Oct 2011 09:07:07 +0100 Subject: Port PyImport_ReloadModule to Unicode API. --- Python/import.c | 60 +++++++++++++++++++++++++-------------------------------- 1 file changed, 26 insertions(+), 34 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 3842c63c2e..294eb2bb33 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3359,8 +3359,8 @@ PyImport_ReloadModule(PyObject *m) PyObject *modules_reloading = interp->modules_reloading; PyObject *modules = PyImport_GetModuleDict(); PyObject *path_list = NULL, *loader = NULL, *existing_m = NULL; - PyObject *nameobj, *bufobj, *subnameobj; - Py_UCS4 *name = NULL, *subname; + PyObject *name, *bufobj, *subname; + Py_ssize_t subname_start; struct filedescr *fdp; FILE *fp = NULL; PyObject *newm = NULL; @@ -3376,45 +3376,39 @@ PyImport_ReloadModule(PyObject *m) "reload() argument must be module"); return NULL; } - nameobj = PyModule_GetNameObject(m); - if (nameobj == NULL || PyUnicode_READY(nameobj) == -1) + name = PyModule_GetNameObject(m); + if (name == NULL || PyUnicode_READY(name) == -1) return NULL; - if (m != PyDict_GetItem(modules, nameobj)) { + if (m != PyDict_GetItem(modules, name)) { PyErr_Format(PyExc_ImportError, "reload(): module %R not in sys.modules", - nameobj); - Py_DECREF(nameobj); + name); + Py_DECREF(name); return NULL; } - existing_m = PyDict_GetItem(modules_reloading, nameobj); + existing_m = PyDict_GetItem(modules_reloading, name); if (existing_m != NULL) { /* Due to a recursive reload, this module is already being reloaded. */ - Py_DECREF(nameobj); + Py_DECREF(name); Py_INCREF(existing_m); return existing_m; } - if (PyDict_SetItem(modules_reloading, nameobj, m) < 0) { - Py_DECREF(nameobj); + if (PyDict_SetItem(modules_reloading, name, m) < 0) { + Py_DECREF(name); return NULL; } - name = PyUnicode_AsUCS4Copy(nameobj); - if (!name) { - Py_DECREF(nameobj); - return NULL; - } - subname = Py_UCS4_strrchr(name, '.'); - if (subname == NULL) { - Py_INCREF(nameobj); - subnameobj = nameobj; + subname_start = PyUnicode_FindChar(name, '.', 0, + PyUnicode_GET_LENGTH(name), -1); + if (subname_start == -1) { + Py_INCREF(name); + subname = name; } else { PyObject *parentname, *parent; Py_ssize_t len; - len = subname - name; - parentname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - name, len); + parentname = PyUnicode_Substring(name, 0, subname_start); if (parentname == NULL) { goto error; } @@ -3430,16 +3424,15 @@ PyImport_ReloadModule(PyObject *m) path_list = _PyObject_GetAttrId(parent, &PyId___path__); if (path_list == NULL) PyErr_Clear(); - subname++; - len = PyUnicode_GET_LENGTH(nameobj) - (len + 1); - subnameobj = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - subname, len); + subname_start++; + len = PyUnicode_GET_LENGTH(name) - (subname_start + 1); + subname = PyUnicode_Substring(name, subname_start, len); } - if (subnameobj == NULL) + if (subname == NULL) goto error; - fdp = find_module(nameobj, subnameobj, path_list, + fdp = find_module(name, subname, path_list, &bufobj, &fp, &loader); - Py_DECREF(subnameobj); + Py_DECREF(subname); Py_XDECREF(path_list); if (fdp == NULL) { @@ -3447,7 +3440,7 @@ PyImport_ReloadModule(PyObject *m) goto error; } - newm = load_module(nameobj, fp, bufobj, fdp->type, loader); + newm = load_module(name, fp, bufobj, fdp->type, loader); Py_XDECREF(bufobj); Py_XDECREF(loader); @@ -3459,13 +3452,12 @@ PyImport_ReloadModule(PyObject *m) * going to return NULL in this case regardless of whether * replacing name succeeds, so the return value is ignored. */ - PyDict_SetItem(modules, nameobj, m); + PyDict_SetItem(modules, name, m); } error: imp_modules_reloading_clear(); - Py_DECREF(nameobj); - PyMem_Free(name); + Py_DECREF(name); return newm; } -- cgit v1.2.1 From 2593ff3b46728f698ea5a5307fcc6263012b358b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 30 Oct 2011 23:50:02 +0100 Subject: Port import_module_level to Unicode API. --- Python/import.c | 164 +++++++++++++++++++++----------------------------------- 1 file changed, 61 insertions(+), 103 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 8e3aa4b921..03e9986126 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2753,8 +2753,7 @@ static PyObject *get_parent(PyObject *globals, int level); static PyObject *load_next(PyObject *mod, PyObject *altmod, PyObject *inputname, PyObject **p_outputname, - Py_UCS4 *buf, Py_ssize_t *p_buflen, - Py_ssize_t bufsize); + PyObject **p_prefix); static int mark_miss(PyObject *name); static int ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *buf, int recursive); @@ -2767,11 +2766,11 @@ static PyObject * import_module_level(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) { - Py_UCS4 buf[MAXPATHLEN+1]; - Py_ssize_t buflen; - Py_ssize_t bufsize = MAXPATHLEN+1; - PyObject *parent, *head, *next, *tail, *inputname, *outputname; - PyObject *parent_name, *ensure_name; + PyObject *parent, *next, *inputname, *outputname; + PyObject *head = NULL; + PyObject *tail = NULL; + PyObject *prefix = NULL; + PyObject *result = NULL; Py_ssize_t sep, altsep; if (PyUnicode_READY(name)) @@ -2794,26 +2793,18 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, return NULL; } - parent = get_parent(globals, &parent_name, level); + parent = get_parent(globals, &prefix, level); if (parent == NULL) { return NULL; } - if (PyUnicode_READY(parent_name)) + if (PyUnicode_READY(prefix)) return NULL; - buflen = PyUnicode_GET_LENGTH(parent_name); - if (!PyUnicode_AsUCS4(parent_name, buf, Py_ARRAY_LENGTH(buf), 1)) { - Py_DECREF(parent_name); - PyErr_SetString(PyExc_ValueError, - "Module name too long"); - return NULL; - } - Py_DECREF(parent_name); head = load_next(parent, level < 0 ? Py_None : parent, name, &outputname, - buf, &buflen, bufsize); + &prefix); if (head == NULL) - return NULL; + goto out; tail = head; Py_INCREF(tail); @@ -2822,13 +2813,11 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, while (1) { inputname = outputname; next = load_next(tail, tail, inputname, &outputname, - buf, &buflen, bufsize); - Py_DECREF(tail); - Py_DECREF(inputname); - if (next == NULL) { - Py_DECREF(head); - return NULL; - } + &prefix); + Py_CLEAR(tail); + Py_CLEAR(inputname); + if (next == NULL) + goto out; tail = next; if (outputname == NULL) { @@ -2840,10 +2829,8 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, /* If tail is Py_None, both get_parent and load_next found an empty module name: someone called __import__("") or doctored faulty bytecode */ - Py_DECREF(tail); - Py_DECREF(head); PyErr_SetString(PyExc_ValueError, "Empty module name"); - return NULL; + goto out; } if (fromlist != NULL) { @@ -2852,26 +2839,21 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, } if (fromlist == NULL) { - Py_DECREF(tail); - return head; - } - - Py_DECREF(head); - - ensure_name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - buf, Py_UCS4_strlen(buf)); - if (ensure_name == NULL) { - Py_DECREF(tail); - return NULL; - } - if (!ensure_fromlist(tail, fromlist, ensure_name, 0)) { - Py_DECREF(tail); - Py_DECREF(ensure_name); - return NULL; + result = head; + Py_INCREF(result); + goto out; } - Py_DECREF(ensure_name); - return tail; + if (!ensure_fromlist(tail, fromlist, prefix, 0)) + goto out; + + result = tail; + Py_INCREF(result); + out: + Py_XDECREF(head); + Py_XDECREF(tail); + Py_XDECREF(prefix); + return result; } PyObject * @@ -3083,91 +3065,67 @@ return_none: static PyObject * load_next(PyObject *mod, PyObject *altmod, PyObject *inputname, PyObject **p_outputname, - Py_UCS4 *buf, Py_ssize_t *p_buflen, Py_ssize_t bufsize) + PyObject **p_prefix) { - Py_UCS4 *dot; + Py_ssize_t dot; Py_ssize_t len; - Py_UCS4 *p; - PyObject *fullname, *name, *result, *mark_name; - Py_UCS4 *nameuni; + PyObject *fullname, *name = NULL, *result; *p_outputname = NULL; - if (PyUnicode_GET_LENGTH(inputname) == 0) { + len = PyUnicode_GET_LENGTH(inputname); + if (len == 0) { /* completely empty module name should only happen in 'from . import' (or '__import__("")')*/ Py_INCREF(mod); return mod; } - nameuni = PyUnicode_AsUCS4Copy(inputname); - if (nameuni == NULL) - return NULL; - dot = Py_UCS4_strchr(nameuni, '.'); - if (dot != NULL) { - len = dot - nameuni; + dot = PyUnicode_FindChar(inputname, '.', 0, len, 1); + if (dot >= 0) { + len = dot; if (len == 0) { PyErr_SetString(PyExc_ValueError, "Empty module name"); goto error; } } - else - len = PyUnicode_GET_LENGTH(inputname); - if (*p_buflen+len+1 >= bufsize) { - PyErr_SetString(PyExc_ValueError, - "Module name too long"); + /* name = inputname[:len] */ + name = PyUnicode_Substring(inputname, 0, len); + if (name == NULL) goto error; - } - p = buf + *p_buflen; - if (p != buf) { - *p++ = '.'; - *p_buflen += 1; + if (PyUnicode_GET_LENGTH(*p_prefix)) { + /* fullname = prefix + "." + name */ + fullname = PyUnicode_FromFormat("%U.%U", *p_prefix, name); + if (fullname == NULL) + goto error; } - Py_UCS4_strncpy(p, nameuni, len); - p[len] = '\0'; - *p_buflen += len; - - fullname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - buf, *p_buflen); - if (fullname == NULL) - goto error; - name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - p, len); - if (name == NULL) { - Py_DECREF(fullname); - goto error; + else { + fullname = name; + Py_INCREF(fullname); } + result = import_submodule(mod, name, fullname); - Py_DECREF(fullname); + Py_DECREF(*p_prefix); + /* Transfer reference. */ + *p_prefix = fullname; if (result == Py_None && altmod != mod) { Py_DECREF(result); /* Here, altmod must be None and mod must not be None */ result = import_submodule(altmod, name, name); - Py_DECREF(name); if (result != NULL && result != Py_None) { - mark_name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - buf, *p_buflen); - if (mark_name == NULL) { + if (mark_miss(*p_prefix) != 0) { Py_DECREF(result); goto error; } - if (mark_miss(mark_name) != 0) { - Py_DECREF(result); - Py_DECREF(mark_name); - goto error; - } - Py_DECREF(mark_name); - Py_UCS4_strncpy(buf, nameuni, len); - buf[len] = '\0'; - *p_buflen = len; + Py_DECREF(*p_prefix); + *p_prefix = name; + Py_INCREF(*p_prefix); } } - else - Py_DECREF(name); if (result == NULL) goto error; @@ -3178,20 +3136,20 @@ load_next(PyObject *mod, PyObject *altmod, goto error; } - if (dot != NULL) { - *p_outputname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - dot+1, Py_UCS4_strlen(dot+1)); + if (dot >= 0) { + *p_outputname = PyUnicode_Substring(inputname, dot+1, + PyUnicode_GET_LENGTH(inputname)); if (*p_outputname == NULL) { Py_DECREF(result); goto error; } } - PyMem_Free(nameuni); + Py_DECREF(name); return result; error: - PyMem_Free(nameuni); + Py_XDECREF(name); return NULL; } -- cgit v1.2.1 From 0c820a9c9584c838de7ba3abea17c76a03bd405b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Mon, 31 Oct 2011 20:47:31 +0100 Subject: Issue #13303: Fix a race condition in the bytecode file creation. --- Python/import.c | 50 ++++++++------------------------------------------ 1 file changed, 8 insertions(+), 42 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 03e9986126..701041b2f2 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1183,42 +1183,6 @@ parse_source_module(PyObject *pathname, FILE *fp) return co; } -/* Helper to open a bytecode file for writing in exclusive mode */ - -#ifndef MS_WINDOWS -static FILE * -open_exclusive(char *filename, mode_t mode) -{ -#if defined(O_EXCL)&&defined(O_CREAT)&&defined(O_WRONLY)&&defined(O_TRUNC) - /* Use O_EXCL to avoid a race condition when another process tries to - write the same file. When that happens, our open() call fails, - which is just fine (since it's only a cache). - XXX If the file exists and is writable but the directory is not - writable, the file will never be written. Oh well. - */ - int fd; - (void) unlink(filename); - fd = open(filename, O_EXCL|O_CREAT|O_WRONLY|O_TRUNC -#ifdef O_BINARY - |O_BINARY /* necessary for Windows */ -#endif -#ifdef __VMS - , mode, "ctxt=bin", "shr=nil" -#else - , mode -#endif - ); - if (fd < 0) - return NULL; - return fdopen(fd, "wb"); -#else - /* Best we can do -- on Windows this can't happen anyway */ - return fopen(filename, "wb"); -#endif -} -#endif - - /* Write a compiled module to a file, placing the time of last modification of its source into the header. Errors are ignored, if a write error occurs an attempt is made to @@ -1234,15 +1198,13 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, #ifdef MS_WINDOWS /* since Windows uses different permissions */ mode_t mode = srcstat->st_mode & ~S_IEXEC; #else - mode_t mode = srcstat->st_mode & ~S_IXUSR & ~S_IXGRP & ~S_IXOTH; mode_t dirmode = (srcstat->st_mode | S_IXUSR | S_IXGRP | S_IXOTH | S_IWUSR | S_IWGRP | S_IWOTH); PyObject *dirbytes; #endif -#ifdef MS_WINDOWS int fd; -#else +#ifndef MS_WINDOWS PyObject *cpathbytes, *cpathbytes_tmp; Py_ssize_t cpathbytes_len; #endif @@ -1313,7 +1275,7 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, return; } cpathbytes_len = PyBytes_GET_SIZE(cpathbytes); - cpathbytes_tmp = PyBytes_FromStringAndSize(NULL, cpathbytes_len + 4); + cpathbytes_tmp = PyBytes_FromStringAndSize(NULL, cpathbytes_len + 6); if (cpathbytes_tmp == NULL) { Py_DECREF(cpathbytes); PyErr_Clear(); @@ -1321,9 +1283,13 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, } memcpy(PyBytes_AS_STRING(cpathbytes_tmp), PyBytes_AS_STRING(cpathbytes), cpathbytes_len); - memcpy(PyBytes_AS_STRING(cpathbytes_tmp) + cpathbytes_len, ".tmp", 4); + memcpy(PyBytes_AS_STRING(cpathbytes_tmp) + cpathbytes_len, "XXXXXX", 6); - fp = open_exclusive(PyBytes_AS_STRING(cpathbytes_tmp), mode); + fd = mkstemp(PyBytes_AS_STRING(cpathbytes_tmp)); + if (0 <= fd) + fp = fdopen(fd, "wb"); + else + fp = NULL; #endif if (fp == NULL) { if (Py_VerboseFlag) -- cgit v1.2.1 From 17ec0be2a18cb0c8d6dc40f0bf0205642e47c31d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Thu, 10 Nov 2011 19:12:29 +0100 Subject: Issue #13303: Fix bytecode file default permission. --- Python/import.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 5c33b6be57..ae1101e4ec 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1202,12 +1202,10 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, S_IXUSR | S_IXGRP | S_IXOTH | S_IWUSR | S_IWGRP | S_IWOTH); PyObject *dirbytes; -#endif - int fd; -#ifndef MS_WINDOWS PyObject *cpathbytes, *cpathbytes_tmp; Py_ssize_t cpathbytes_len; #endif + int fd; PyObject *dirname; Py_UCS4 *dirsep; int res, ok; @@ -1275,7 +1273,7 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, return; } cpathbytes_len = PyBytes_GET_SIZE(cpathbytes); - cpathbytes_tmp = PyBytes_FromStringAndSize(NULL, cpathbytes_len + 6); + cpathbytes_tmp = PyBytes_FromStringAndSize(NULL, cpathbytes_len + 4); if (cpathbytes_tmp == NULL) { Py_DECREF(cpathbytes); PyErr_Clear(); @@ -1283,9 +1281,10 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, } memcpy(PyBytes_AS_STRING(cpathbytes_tmp), PyBytes_AS_STRING(cpathbytes), cpathbytes_len); - memcpy(PyBytes_AS_STRING(cpathbytes_tmp) + cpathbytes_len, "XXXXXX", 6); + memcpy(PyBytes_AS_STRING(cpathbytes_tmp) + cpathbytes_len, ".tmp", 4); - fd = mkstemp(PyBytes_AS_STRING(cpathbytes_tmp)); + fd = open(PyBytes_AS_STRING(cpathbytes_tmp), + O_CREAT | O_EXCL | O_WRONLY, 0666); if (0 <= fd) fp = fdopen(fd, "wb"); else -- cgit v1.2.1 From 1a238237f0304a03e73e464b069a2a4aec4beb3c Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 15 Nov 2011 19:15:19 +0100 Subject: Issue #13392: Writing a pyc file should now be atomic under Windows as well. --- Python/import.c | 48 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 10 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index ae1101e4ec..3c3e504e28 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1197,6 +1197,8 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, time_t mtime = srcstat->st_mtime; #ifdef MS_WINDOWS /* since Windows uses different permissions */ mode_t mode = srcstat->st_mode & ~S_IEXEC; + PyObject *cpathname_tmp; + Py_ssize_t cpathname_len; #else mode_t dirmode = (srcstat->st_mode | S_IXUSR | S_IXGRP | S_IXOTH | @@ -1255,18 +1257,29 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, } Py_DECREF(dirname); + /* We first write to a tmp file and then take advantage + of atomic renaming (which *should* be true even under Windows). */ #ifdef MS_WINDOWS - (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname)); - fd = _wopen(PyUnicode_AS_UNICODE(cpathname), - O_EXCL | O_CREAT | O_WRONLY | O_TRUNC | O_BINARY, - mode); + cpathname_len = PyUnicode_GET_LENGTH(cpathname); + cpathname_tmp = PyUnicode_New(cpathname_len + 4, + PyUnicode_MAX_CHAR_VALUE(cpathname)); + if (cpathname_tmp == NULL) { + PyErr_Clear(); + return; + } + PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 0, '.'); + PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 1, 't'); + PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 2, 'm'); + PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 3, 'p'); + (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname_tmp)); + fd = _wopen(PyUnicode_AS_UNICODE(cpathname_tmp), + O_EXCL | O_CREAT | O_WRONLY | O_BINARY, + mode); if (0 <= fd) fp = fdopen(fd, "wb"); else fp = NULL; #else - /* Under POSIX, we first write to a tmp file and then take advantage - of atomic renaming. */ cpathbytes = PyUnicode_EncodeFSDefault(cpathname); if (cpathbytes == NULL) { PyErr_Clear(); @@ -1294,7 +1307,9 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, if (Py_VerboseFlag) PySys_FormatStderr( "# can't create %R\n", cpathname); -#ifndef MS_WINDOWS +#ifdef MS_WINDOWS + Py_DECREF(cpathname_tmp); +#else Py_DECREF(cpathbytes); Py_DECREF(cpathbytes_tmp); #endif @@ -1315,7 +1330,8 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, /* Don't keep partial file */ fclose(fp); #ifdef MS_WINDOWS - (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname)); + (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname_tmp)); + Py_DECREF(cpathname_tmp); #else (void) unlink(PyBytes_AS_STRING(cpathbytes_tmp)); Py_DECREF(cpathbytes); @@ -1324,8 +1340,20 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, return; } fclose(fp); - /* Under POSIX, do an atomic rename */ -#ifndef MS_WINDOWS + /* Do a (hopefully) atomic rename */ +#ifdef MS_WINDOWS + if (!MoveFileExW(PyUnicode_AS_UNICODE(cpathname_tmp), + PyUnicode_AS_UNICODE(cpathname), + MOVEFILE_REPLACE_EXISTING)) { + if (Py_VerboseFlag) + PySys_FormatStderr("# can't write %R\n", cpathname); + /* Don't keep tmp file */ + (void) DeleteFileW(PyUnicode_AS_UNICODE(cpathname_tmp)); + Py_DECREF(cpathname_tmp); + return; + } + Py_DECREF(cpathname_tmp); +#else if (rename(PyBytes_AS_STRING(cpathbytes_tmp), PyBytes_AS_STRING(cpathbytes))) { if (Py_VerboseFlag) -- cgit v1.2.1 From 1a11faf52590fae1e61f304835b27c42ed21b1dd Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 15 Nov 2011 20:40:55 +0100 Subject: Fix regression under Windows following b75b41237380 (from issue #13392) --- Python/import.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 3c3e504e28..64197d7a48 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1267,6 +1267,11 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, PyErr_Clear(); return; } + if (PyUnicode_CopyCharacters(cpathname_tmp, 0, + cpathname, 0, cpathname_len) < 0) { + PyErr_Clear(); + return; + } PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 0, '.'); PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 1, 't'); PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 2, 'm'); -- cgit v1.2.1 From 23ad9bf93bf81d89142e55ca1cfb04f9c3ffc837 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 15 Nov 2011 22:27:32 +0100 Subject: Use PyUnicode_FromFormat() to create the temporary file name. Also, as in importlib, append the id of an object to make the file name pseudo-random. --- Python/import.c | 41 ++++++++++++++--------------------------- 1 file changed, 14 insertions(+), 27 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 64197d7a48..b1eca64f4d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1195,17 +1195,15 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, Py_UCS4 *cpathname_ucs4; FILE *fp; time_t mtime = srcstat->st_mtime; + PyObject *cpathname_tmp; #ifdef MS_WINDOWS /* since Windows uses different permissions */ mode_t mode = srcstat->st_mode & ~S_IEXEC; - PyObject *cpathname_tmp; - Py_ssize_t cpathname_len; #else mode_t dirmode = (srcstat->st_mode | S_IXUSR | S_IXGRP | S_IXOTH | S_IWUSR | S_IWGRP | S_IWOTH); PyObject *dirbytes; PyObject *cpathbytes, *cpathbytes_tmp; - Py_ssize_t cpathbytes_len; #endif int fd; PyObject *dirname; @@ -1258,24 +1256,18 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, Py_DECREF(dirname); /* We first write to a tmp file and then take advantage - of atomic renaming (which *should* be true even under Windows). */ -#ifdef MS_WINDOWS - cpathname_len = PyUnicode_GET_LENGTH(cpathname); - cpathname_tmp = PyUnicode_New(cpathname_len + 4, - PyUnicode_MAX_CHAR_VALUE(cpathname)); + of atomic renaming (which *should* be true even under Windows). + As in importlib, we use id(something) to generate a pseudo-random + filename. mkstemp() can't be used since it doesn't allow specifying + the file access permissions. + */ + cpathname_tmp = PyUnicode_FromFormat("%U.%zd", + cpathname, (Py_ssize_t) co); if (cpathname_tmp == NULL) { PyErr_Clear(); return; } - if (PyUnicode_CopyCharacters(cpathname_tmp, 0, - cpathname, 0, cpathname_len) < 0) { - PyErr_Clear(); - return; - } - PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 0, '.'); - PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 1, 't'); - PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 2, 'm'); - PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 3, 'p'); +#ifdef MS_WINDOWS (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname_tmp)); fd = _wopen(PyUnicode_AS_UNICODE(cpathname_tmp), O_EXCL | O_CREAT | O_WRONLY | O_BINARY, @@ -1285,22 +1277,17 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, else fp = NULL; #else - cpathbytes = PyUnicode_EncodeFSDefault(cpathname); - if (cpathbytes == NULL) { + cpathbytes_tmp = PyUnicode_EncodeFSDefault(cpathname_tmp); + Py_DECREF(cpathname_tmp); + if (cpathbytes_tmp == NULL) { PyErr_Clear(); return; } - cpathbytes_len = PyBytes_GET_SIZE(cpathbytes); - cpathbytes_tmp = PyBytes_FromStringAndSize(NULL, cpathbytes_len + 4); - if (cpathbytes_tmp == NULL) { - Py_DECREF(cpathbytes); + cpathbytes = PyUnicode_EncodeFSDefault(cpathname); + if (cpathbytes == NULL) { PyErr_Clear(); return; } - memcpy(PyBytes_AS_STRING(cpathbytes_tmp), PyBytes_AS_STRING(cpathbytes), - cpathbytes_len); - memcpy(PyBytes_AS_STRING(cpathbytes_tmp) + cpathbytes_len, ".tmp", 4); - fd = open(PyBytes_AS_STRING(cpathbytes_tmp), O_CREAT | O_EXCL | O_WRONLY, 0666); if (0 <= fd) -- cgit v1.2.1 From 84acdc7c2fbb43c63eab2518445c5a7771755037 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 17 Nov 2011 00:45:54 +0100 Subject: Catch PyUnicode_AS_UNICODE() errors --- Python/import.c | 47 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 14 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index b1eca64f4d..6c6d1f697a 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1198,6 +1198,7 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, PyObject *cpathname_tmp; #ifdef MS_WINDOWS /* since Windows uses different permissions */ mode_t mode = srcstat->st_mode & ~S_IEXEC; + wchar_t *wdirname, *wpathname, *wpathname_tmp; #else mode_t dirmode = (srcstat->st_mode | S_IXUSR | S_IXGRP | S_IXOTH | @@ -1230,7 +1231,12 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, } #ifdef MS_WINDOWS - res = CreateDirectoryW(PyUnicode_AS_UNICODE(dirname), NULL); + wdirname = PyUnicode_AsUnicode(dirname); + if (wdirname == NULL) { + PyErr_Clear(); + return; + } + res = CreateDirectoryW(wdirname, NULL); ok = (res != 0); if (!ok && GetLastError() == ERROR_ALREADY_EXISTS) ok = 1; @@ -1268,8 +1274,19 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, return; } #ifdef MS_WINDOWS - (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname_tmp)); - fd = _wopen(PyUnicode_AS_UNICODE(cpathname_tmp), + wpathname = PyUnicode_AsUnicode(cpathname); + if (wpathname == NULL) { + PyErr_Clear(); + return; + } + wpathname_tmp = PyUnicode_AsUnicode(cpathname_tmp); + if (wpathname_tmp == NULL) { + PyErr_Clear(); + return; + } + + (void)DeleteFileW(wpathname_tmp); + fd = _wopen(wpathname_tmp, O_EXCL | O_CREAT | O_WRONLY | O_BINARY, mode); if (0 <= fd) @@ -1322,7 +1339,7 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, /* Don't keep partial file */ fclose(fp); #ifdef MS_WINDOWS - (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname_tmp)); + (void)DeleteFileW(wpathname_tmp); Py_DECREF(cpathname_tmp); #else (void) unlink(PyBytes_AS_STRING(cpathbytes_tmp)); @@ -1334,13 +1351,11 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, fclose(fp); /* Do a (hopefully) atomic rename */ #ifdef MS_WINDOWS - if (!MoveFileExW(PyUnicode_AS_UNICODE(cpathname_tmp), - PyUnicode_AS_UNICODE(cpathname), - MOVEFILE_REPLACE_EXISTING)) { + if (!MoveFileExW(wpathname_tmp, wpathname, MOVEFILE_REPLACE_EXISTING)) { if (Py_VerboseFlag) PySys_FormatStderr("# can't write %R\n", cpathname); /* Don't keep tmp file */ - (void) DeleteFileW(PyUnicode_AS_UNICODE(cpathname_tmp)); + (void) DeleteFileW(wpathname_tmp); Py_DECREF(cpathname_tmp); return; } @@ -1819,10 +1834,10 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, PyUnicode_CopyCharacters(filename, 0, path_unicode, 0, len); pos = len; if (addsep) - PyUnicode_WRITE(PyUnicode_KIND(filename), + PyUnicode_WRITE(PyUnicode_KIND(filename), PyUnicode_DATA(filename), pos++, SEP); - PyUnicode_CopyCharacters(filename, pos, name, 0, + PyUnicode_CopyCharacters(filename, pos, name, 0, PyUnicode_GET_LENGTH(name)); /* Check for package import (buf holds a directory name, @@ -2268,18 +2283,22 @@ case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) WIN32_FIND_DATAW data; HANDLE h; int cmp; - wchar_t *wname; + wchar_t *wfilename, *wname; Py_ssize_t wname_len; if (Py_GETENV("PYTHONCASEOK") != NULL) return 1; - h = FindFirstFileW(PyUnicode_AS_UNICODE(filename), &data); + wfilename = PyUnicode_AsUnicode(filename); + if (wfilename == NULL) + return -1; + + h = FindFirstFileW(wfilename, &data); if (h == INVALID_HANDLE_VALUE) { PyErr_Format(PyExc_NameError, "Can't find file for module %R\n(filename %R)", name, filename); - return 0; + return -1; } FindClose(h); @@ -2831,7 +2850,7 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, if (!ensure_fromlist(tail, fromlist, prefix, 0)) goto out; - + result = tail; Py_INCREF(result); out: -- cgit v1.2.1 From 949c3f094c436fea35325980527d63d0d9eabd21 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 25 Nov 2011 18:56:07 +0100 Subject: PEP 3155 / issue #13448: Qualified name for classes and functions. --- Python/import.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 6c6d1f697a..7199230098 100644 --- a/Python/import.c +++ b/Python/import.c @@ -103,6 +103,7 @@ typedef unsigned short mode_t; tag: cpython-32 Python 3.2a2 3180 (add DELETE_DEREF) Python 3.3a0 3190 __class__ super closure changed + Python 3.3a0 3200 (__qualname__ added) */ /* MAGIC must change whenever the bytecode emitted by the compiler may no @@ -115,7 +116,7 @@ typedef unsigned short mode_t; #define STRIFY(name) QUOTE(name) #define MAJOR STRIFY(PY_MAJOR_VERSION) #define MINOR STRIFY(PY_MINOR_VERSION) -#define MAGIC (3190 | ((long)'\r'<<16) | ((long)'\n'<<24)) +#define MAGIC (3200 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" /* Current magic word and string tag as globals. */ -- cgit v1.2.1