diff options
author | Giampaolo Rodola <g.rodola@gmail.com> | 2017-05-04 01:55:21 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-05-04 01:55:21 +0200 |
commit | 91f26bb3bd08b5bf1db3d185274986962c8bd15a (patch) | |
tree | 3f4642d9a2eefd5d429f0658715cfa0b7b70c9cc | |
parent | e6f5f498d508335f359cd2d01a1098afcbcb1b6c (diff) | |
parent | 04111d9a048006ecb24ca7a7a92e84bb138e4618 (diff) | |
download | psutil-91f26bb3bd08b5bf1db3d185274986962c8bd15a.tar.gz |
Merge pull request #1052 from giampaolo/1040-fix-unicode
#1040 fix unicode
-rw-r--r-- | HISTORY.rst | 17 | ||||
-rw-r--r-- | docs/index.rst | 38 | ||||
-rw-r--r-- | psutil/_psutil_bsd.c | 83 | ||||
-rw-r--r-- | psutil/_psutil_common.c | 19 | ||||
-rw-r--r-- | psutil/_psutil_common.h | 4 | ||||
-rw-r--r-- | psutil/_psutil_linux.c | 47 | ||||
-rw-r--r-- | psutil/_psutil_osx.c | 107 | ||||
-rw-r--r-- | psutil/_psutil_sunos.c | 81 | ||||
-rw-r--r-- | psutil/_psutil_windows.c | 107 | ||||
-rw-r--r-- | psutil/_pswindows.py | 58 | ||||
-rw-r--r-- | psutil/arch/bsd/freebsd.c | 29 | ||||
-rw-r--r-- | psutil/arch/bsd/freebsd_socks.c | 24 | ||||
-rw-r--r-- | psutil/arch/bsd/netbsd.c | 11 | ||||
-rw-r--r-- | psutil/arch/bsd/netbsd_socks.c | 10 | ||||
-rw-r--r-- | psutil/arch/bsd/openbsd.c | 8 | ||||
-rw-r--r-- | psutil/arch/osx/process_info.c | 11 | ||||
-rw-r--r-- | psutil/arch/windows/services.c | 95 | ||||
-rw-r--r-- | psutil/tests/test_unicode.py | 81 | ||||
-rwxr-xr-x | psutil/tests/test_windows.py | 15 |
19 files changed, 486 insertions, 359 deletions
diff --git a/HISTORY.rst b/HISTORY.rst index c193dfc8..9a3b1793 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -15,6 +15,7 @@ Process.as_dict(): "attrs" and "ad_value". With this you can iterate over all processes in one shot without needing to catch NoSuchProcess and do list/dict comprehensions. +- 1040_: implemented full unicode support. **Bug fixes** @@ -29,15 +30,31 @@ properly handle unicode paths and may raise UnicodeDecodeError. - 1033_: [OSX, FreeBSD] memory leak for net_connections() and Process.connections() when retrieving UNIX sockets (kind='unix'). +- 1040_: fixed many unicode related issues such as UnicodeDecodeError on + Python 3 + UNIX and invalid encoded data on Windows. +- 1046_: [Windows] disk_partitions() on Windows overrides user's SetErrorMode. +- 1047_: [Windows] Process username(): memory leak in case exception is thrown. +- 1048_: [Windows] users()'s host field report an invalid IP address. + +**Porting notes** + - 1039_: returned types consolidation: - Windows / Process.cpu_times(): fields #3 and #4 were int instead of float - Linux / FreeBSD: connections('unix'): raddr is now set to "" instead of None - OpenBSD: connections('unix'): laddr and raddr are now set to "" instead of None +- 1040_: all strings are encoded by using OS fs encoding. +- 1040_: the following Windows APIs returned unicode and now they return str: + - Process.memory_maps().path + - WindosService.bin_path() + - WindosService.description() + - WindosService.display_name() + - WindosService.username() - 1046_: [Windows] disk_partitions() on Windows overrides user's SetErrorMode. - 1047_: [Windows] Process username(): memory leak in case exception is thrown. - 1050_: [Windows] Process.memory_maps memory() leaks memory. +>>>>>>> master *2017-04-10* diff --git a/docs/index.rst b/docs/index.rst index 16301958..509d5f9c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2239,6 +2239,44 @@ Constants >>> if psutil.version_info >= (4, 5): ... pass +---- + +Unicode +======= + +Starting from version 5.3.0 psutil +`fully supports unicode <https://github.com/giampaolo/psutil/issues/1040>`__. +The notes below apply to *any* method returning a string such as +:meth:`Process.exe` or :meth:`Process.cwd`, including non-filesystem related +methods such as :meth:`Process.username`: + +* all strings are encoded by using the OS filesystem encoding which varies + depending on the platform you're on (e.g. UTF-8 on Linux, mbcs on Win) +* no API call is supposed to crash with ``UnicodeDecodeError`` +* instead, in case of badly encoded data returned by the OS, the following error handlers are used to replace the bad characters in the string: + * Python 2: ``"replace"`` + * Python 3: ``"surrogatescape"`` on POSIX and ``"replace"`` on Windows +* on Python 2 all APIs return bytes (``str`` type), never ``unicode`` +* on Python 2 you can go back to unicode by doing: + +.. code-block:: python + + >>> unicode(p.exe(), sys.getdefaultencoding(), errors="replace") + +Example which filters processes with a funky name working with Python 2 and 3:: + + # -*- coding: utf-8 -*- + import psutil, sys + + PY3 = sys.version_info[0] == 2 + LOOKFOR = u"ƒőő" + for proc in psutil.process_iter(attrs=['name']): + name = proc.info['name'] + if not PY3: + name = unicode(name, sys.getdefaultencoding(), errors="replace") + if LOOKFOR == name: + print("process %s found" % p) + Recipes ======= diff --git a/psutil/_psutil_bsd.c b/psutil/_psutil_bsd.c index 537df151..43189a21 100644 --- a/psutil/_psutil_bsd.c +++ b/psutil/_psutil_bsd.c @@ -215,11 +215,7 @@ psutil_proc_oneshot_info(PyObject *self, PyObject *args) { #elif defined(PSUTIL_OPENBSD) || defined(PSUTIL_NETBSD) sprintf(str, "%s", kp.p_comm); #endif -#if PY_MAJOR_VERSION >= 3 py_name = PyUnicode_DecodeFSDefault(str); -#else - py_name = Py_BuildValue("s", str); -#endif if (! py_name) { // Likely a decoding error. We don't want to fail the whole // operation. The python module may retry with proc_name(). @@ -372,12 +368,7 @@ psutil_proc_name(PyObject *self, PyObject *args) { #elif defined(PSUTIL_OPENBSD) || defined(PSUTIL_NETBSD) sprintf(str, "%s", kp.p_comm); #endif - -#if PY_MAJOR_VERSION >= 3 return PyUnicode_DecodeFSDefault(str); -#else - return Py_BuildValue("s", str); -#endif } @@ -472,6 +463,7 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { struct kinfo_file *kif; kinfo_proc kipp; PyObject *py_tuple = NULL; + PyObject *py_path = NULL; PyObject *py_retlist = PyList_New(0); if (py_retlist == NULL) @@ -507,12 +499,16 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { // XXX - it appears path is not exposed in the kinfo_file struct. path = ""; #endif + py_path = PyUnicode_DecodeFSDefault(path); + if (! py_path) + goto error; if (regular == 1) { - py_tuple = Py_BuildValue("(si)", path, fd); + py_tuple = Py_BuildValue("(Oi)", py_path, fd); if (py_tuple == NULL) goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_path); Py_DECREF(py_tuple); } } @@ -546,6 +542,8 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { struct statfs *fs = NULL; #endif PyObject *py_retlist = PyList_New(0); + PyObject *py_dev = NULL; + PyObject *py_mountp = NULL; PyObject *py_tuple = NULL; if (py_retlist == NULL) @@ -658,15 +656,23 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { if (flags & MNT_NODEVMTIME) strlcat(opts, ",nodevmtime", sizeof(opts)); #endif - py_tuple = Py_BuildValue("(ssss)", - fs[i].f_mntfromname, // device - fs[i].f_mntonname, // mount point + py_dev = PyUnicode_DecodeFSDefault(fs[i].f_mntfromname); + if (! py_dev) + goto error; + py_mountp = PyUnicode_DecodeFSDefault(fs[i].f_mntonname); + if (! py_mountp) + goto error; + py_tuple = Py_BuildValue("(OOss)", + py_dev, // device + py_mountp, // mount point fs[i].f_fstypename, // fs type opts); // options if (!py_tuple) goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_dev); + Py_DECREF(py_mountp); Py_DECREF(py_tuple); } @@ -674,6 +680,8 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { return py_retlist; error: + Py_XDECREF(py_dev); + Py_XDECREF(py_mountp); Py_XDECREF(py_tuple); Py_DECREF(py_retlist); if (fs != NULL) @@ -783,6 +791,9 @@ error: static PyObject * psutil_users(PyObject *self, PyObject *args) { PyObject *py_retlist = PyList_New(0); + PyObject *py_username = NULL; + PyObject *py_tty = NULL; + PyObject *py_hostname = NULL; PyObject *py_tuple = NULL; if (py_retlist == NULL) @@ -801,12 +812,21 @@ psutil_users(PyObject *self, PyObject *args) { while (fread(&ut, sizeof(ut), 1, fp) == 1) { if (*ut.ut_name == '\0') continue; + py_username = PyUnicode_DecodeFSDefault(ut.ut_name); + if (! py_username) + goto error; + py_tty = PyUnicode_DecodeFSDefault(ut.ut_line); + if (! py_tty) + goto error; + py_hostname = PyUnicode_DecodeFSDefault(ut.ut_host); + if (! py_hostname) + goto error; py_tuple = Py_BuildValue( - "(sssfi)", - ut.ut_name, // username - ut.ut_line, // tty - ut.ut_host, // hostname - (float)ut.ut_time, // start time + "(OOOfi)", + py_username, // username + py_tty, // tty + py_hostname, // hostname + (float)ut.ut_time, // start time #ifdef PSUTIL_OPENBSD -1 // process id (set to None later) #else @@ -821,22 +841,33 @@ psutil_users(PyObject *self, PyObject *args) { fclose(fp); goto error; } + Py_DECREF(py_username); + Py_DECREF(py_tty); + Py_DECREF(py_hostname); Py_DECREF(py_tuple); } fclose(fp); #else struct utmpx *utx; - setutxent(); while ((utx = getutxent()) != NULL) { if (utx->ut_type != USER_PROCESS) continue; + py_username = PyUnicode_DecodeFSDefault(utx->ut_user); + if (! py_username) + goto error; + py_tty = PyUnicode_DecodeFSDefault(utx->ut_line); + if (! py_tty) + goto error; + py_hostname = PyUnicode_DecodeFSDefault(utx->ut_host); + if (! py_hostname) + goto error; py_tuple = Py_BuildValue( - "(sssfi)", - utx->ut_user, // username - utx->ut_line, // tty - utx->ut_host, // hostname + "(OOOfi)", + py_username, // username + py_tty, // tty + py_hostname, // hostname (float)utx->ut_tv.tv_sec, // start time #ifdef PSUTIL_OPENBSD -1 // process id (set to None later) @@ -853,6 +884,9 @@ psutil_users(PyObject *self, PyObject *args) { endutxent(); goto error; } + Py_DECREF(py_username); + Py_DECREF(py_tty); + Py_DECREF(py_hostname); Py_DECREF(py_tuple); } @@ -861,6 +895,9 @@ psutil_users(PyObject *self, PyObject *args) { return py_retlist; error: + Py_XDECREF(py_username); + Py_XDECREF(py_tty); + Py_XDECREF(py_hostname); Py_XDECREF(py_tuple); Py_DECREF(py_retlist); return NULL; diff --git a/psutil/_psutil_common.c b/psutil/_psutil_common.c index c8d736e8..bcbd623b 100644 --- a/psutil/_psutil_common.c +++ b/psutil/_psutil_common.c @@ -34,3 +34,22 @@ AccessDenied(void) { Py_XDECREF(exc); return NULL; } + + +/* + * Backport of unicode FS APIs from Python 3. + * On Python 2 we just return a plain byte string + * which is never supposed to raise decoding errors. + * See: https://github.com/giampaolo/psutil/issues/1040 + */ +#if PY_MAJOR_VERSION < 3 +PyObject * +PyUnicode_DecodeFSDefault(char *s) { + return PyString_FromString(s); +} + +PyObject * +PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size) { + return PyString_FromStringAndSize(s, size); +} +#endif
\ No newline at end of file diff --git a/psutil/_psutil_common.h b/psutil/_psutil_common.h index 43021a72..0507458a 100644 --- a/psutil/_psutil_common.h +++ b/psutil/_psutil_common.h @@ -8,3 +8,7 @@ PyObject* AccessDenied(void); PyObject* NoSuchProcess(void); +#if PY_MAJOR_VERSION < 3 +PyObject* PyUnicode_DecodeFSDefault(char *s); +PyObject* PyUnicode_DecodeFSDefaultAndSize(char *s, Py_ssize_t size); +#endif
\ No newline at end of file diff --git a/psutil/_psutil_linux.c b/psutil/_psutil_linux.c index 9abe44e0..1a96fea0 100644 --- a/psutil/_psutil_linux.c +++ b/psutil/_psutil_linux.c @@ -195,8 +195,10 @@ static PyObject * psutil_disk_partitions(PyObject *self, PyObject *args) { FILE *file = NULL; struct mntent *entry; - PyObject *py_retlist = PyList_New(0); + PyObject *py_dev = NULL; + PyObject *py_mountp = NULL; PyObject *py_tuple = NULL; + PyObject *py_retlist = PyList_New(0); if (py_retlist == NULL) return NULL; @@ -215,15 +217,23 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { PyErr_Format(PyExc_RuntimeError, "getmntent() syscall failed"); goto error; } - py_tuple = Py_BuildValue("(ssss)", - entry->mnt_fsname, // device - entry->mnt_dir, // mount point + py_dev = PyUnicode_DecodeFSDefault(entry->mnt_fsname); + if (! py_dev) + goto error; + py_mountp = PyUnicode_DecodeFSDefault(entry->mnt_dir); + if (! py_mountp) + goto error; + py_tuple = Py_BuildValue("(OOss)", + py_dev, // device + py_mountp, // mount point entry->mnt_type, // fs type entry->mnt_opts); // options if (! py_tuple) goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_dev); + Py_DECREF(py_mountp); Py_DECREF(py_tuple); } endmntent(file); @@ -232,6 +242,8 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { error: if (file != NULL) endmntent(file); + Py_XDECREF(py_dev); + Py_XDECREF(py_mountp); Py_XDECREF(py_tuple); Py_DECREF(py_retlist); return NULL; @@ -439,6 +451,9 @@ psutil_users(PyObject *self, PyObject *args) { struct utmp *ut; PyObject *py_retlist = PyList_New(0); PyObject *py_tuple = NULL; + PyObject *py_username = NULL; + PyObject *py_tty = NULL; + PyObject *py_hostname = NULL; PyObject *py_user_proc = NULL; if (py_retlist == NULL) @@ -451,11 +466,20 @@ psutil_users(PyObject *self, PyObject *args) { py_user_proc = Py_True; else py_user_proc = Py_False; + py_username = PyUnicode_DecodeFSDefault(ut->ut_user); + if (! py_username) + goto error; + py_tty = PyUnicode_DecodeFSDefault(ut->ut_line); + if (! py_tty) + goto error; + py_hostname = PyUnicode_DecodeFSDefault(ut->ut_host); + if (! py_hostname) + goto error; py_tuple = Py_BuildValue( - "(sssfOi)", - ut->ut_user, // username - ut->ut_line, // tty - ut->ut_host, // hostname + "(OOOfOi)", + py_username, // username + py_tty, // tty + py_username, // hostname (float)ut->ut_tv.tv_sec, // tstamp py_user_proc, // (bool) user process ut->ut_pid // process id @@ -464,14 +488,19 @@ psutil_users(PyObject *self, PyObject *args) { goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_username); + Py_DECREF(py_tty); + Py_DECREF(py_hostname); Py_DECREF(py_tuple); } endutent(); return py_retlist; error: + Py_XDECREF(py_username); + Py_XDECREF(py_tty); + Py_XDECREF(py_hostname); Py_XDECREF(py_tuple); - Py_XDECREF(py_user_proc); Py_DECREF(py_retlist); endutent(); return NULL; diff --git a/psutil/_psutil_osx.c b/psutil/_psutil_osx.c index db1f997a..a831441a 100644 --- a/psutil/_psutil_osx.c +++ b/psutil/_psutil_osx.c @@ -138,11 +138,7 @@ psutil_proc_kinfo_oneshot(PyObject *self, PyObject *args) { if (psutil_get_kinfo_proc(pid, &kp) == -1) return NULL; -#if PY_MAJOR_VERSION >= 3 py_name = PyUnicode_DecodeFSDefault(kp.kp_proc.p_comm); -#else - py_name = Py_BuildValue("s", kp.kp_proc.p_comm); -#endif if (! py_name) { // Likely a decoding error. We don't want to fail the whole // operation. The python module may retry with proc_name(). @@ -224,11 +220,7 @@ psutil_proc_name(PyObject *self, PyObject *args) { return NULL; if (psutil_get_kinfo_proc(pid, &kp) == -1) return NULL; -#if PY_MAJOR_VERSION >= 3 return PyUnicode_DecodeFSDefault(kp.kp_proc.p_comm); -#else - return Py_BuildValue("s", kp.kp_proc.p_comm); -#endif } @@ -249,11 +241,7 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { return NULL; } -#if PY_MAJOR_VERSION >= 3 return PyUnicode_DecodeFSDefault(pathinfo.pvi_cdir.vip_path); -#else - return Py_BuildValue("s", pathinfo.pvi_cdir.vip_path); -#endif } @@ -277,11 +265,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { psutil_raise_for_pid(pid, "proc_pidpath() syscall failed"); return NULL; } -#if PY_MAJOR_VERSION >= 3 return PyUnicode_DecodeFSDefault(buf); -#else - return Py_BuildValue("s", buf); -#endif } @@ -337,6 +321,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { vm_size_t size = 0; PyObject *py_tuple = NULL; + PyObject *py_path = NULL; PyObject *py_list = PyList_New(0); if (py_list == NULL) @@ -431,11 +416,14 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { } } + py_path = PyUnicode_DecodeFSDefault(buf); + if (! py_path) + goto error; py_tuple = Py_BuildValue( - "sssIIIIIH", + "ssOIIIIIH", addr_str, // "start-end"address perms, // "rwx" permissions - buf, // path + py_path, // path info.pages_resident * pagesize, // rss info.pages_shared_now_private * pagesize, // private info.pages_swapped_out * pagesize, // swapped @@ -448,6 +436,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { if (PyList_Append(py_list, py_tuple)) goto error; Py_DECREF(py_tuple); + Py_DECREF(py_path); } // increment address for the next map/file @@ -463,6 +452,7 @@ error: if (task != MACH_PORT_NULL) mach_port_deallocate(mach_task_self(), task); Py_XDECREF(py_tuple); + Py_XDECREF(py_path); Py_DECREF(py_list); return NULL; } @@ -863,8 +853,10 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { uint64_t flags; char opts[400]; struct statfs *fs = NULL; - PyObject *py_retlist = PyList_New(0); + PyObject *py_dev = NULL; + PyObject *py_mountp = NULL; PyObject *py_tuple = NULL; + PyObject *py_retlist = PyList_New(0); if (py_retlist == NULL) return NULL; @@ -949,15 +941,24 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { if (flags & MNT_CMDFLAGS) strlcat(opts, ",cmdflags", sizeof(opts)); + py_dev = PyUnicode_DecodeFSDefault(fs[i].f_mntfromname); + if (! py_dev) + goto error; + py_mountp = PyUnicode_DecodeFSDefault(fs[i].f_mntonname); + if (! py_mountp) + goto error; py_tuple = Py_BuildValue( - "(ssss)", fs[i].f_mntfromname, // device - fs[i].f_mntonname, // mount point + "(OOss)", + py_dev, // device + py_mountp, // mount point fs[i].f_fstypename, // fs type opts); // options if (!py_tuple) goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_dev); + Py_DECREF(py_mountp); Py_DECREF(py_tuple); } @@ -965,6 +966,8 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { return py_retlist; error: + Py_XDECREF(py_dev); + Py_XDECREF(py_mountp); Py_XDECREF(py_tuple); Py_DECREF(py_retlist); if (fs != NULL) @@ -1151,11 +1154,7 @@ psutil_proc_open_files(PyObject *self, PyObject *args) { // --- /errors checking // --- construct python list -#if PY_MAJOR_VERSION >= 3 py_path = PyUnicode_DecodeFSDefault(vi.pvip.vip_path); -#else - py_path = Py_BuildValue("s", vi.pvip.vip_path); -#endif if (! py_path) goto error; py_tuple = Py_BuildValue( @@ -1351,28 +1350,14 @@ psutil_proc_connections(PyObject *self, PyObject *args) { Py_DECREF(py_tuple); } else if (family == AF_UNIX) { - // decode laddr -#if PY_MAJOR_VERSION >= 3 - py_laddr = PyUnicode_DecodeFSDefault( -#else - py_laddr = Py_BuildValue("s", -#endif - si.psi.soi_proto.pri_un.unsi_addr.ua_sun.sun_path - ); - if (!py_laddr) - goto error; - - // decode raddr -#if PY_MAJOR_VERSION >= 3 - py_raddr = PyUnicode_DecodeFSDefault( -#else - py_raddr = Py_BuildValue("s", -#endif - si.psi.soi_proto.pri_un.unsi_caddr.ua_sun.sun_path - ); - if (!py_raddr) - goto error; - + py_laddr = PyUnicode_DecodeFSDefault( + si.psi.soi_proto.pri_un.unsi_addr.ua_sun.sun_path); + if (!py_laddr) + goto error; + py_raddr = PyUnicode_DecodeFSDefault( + si.psi.soi_proto.pri_un.unsi_caddr.ua_sun.sun_path); + if (!py_raddr) + goto error; // construct the python list py_tuple = Py_BuildValue( "(iiiOOi)", @@ -1705,19 +1690,31 @@ error: static PyObject * psutil_users(PyObject *self, PyObject *args) { struct utmpx *utx; - PyObject *py_retlist = PyList_New(0); + PyObject *py_username = NULL; + PyObject *py_tty = NULL; + PyObject *py_hostname = NULL; PyObject *py_tuple = NULL; + PyObject *py_retlist = PyList_New(0); if (py_retlist == NULL) return NULL; while ((utx = getutxent()) != NULL) { if (utx->ut_type != USER_PROCESS) continue; + py_username = PyUnicode_DecodeFSDefault(utx->ut_user); + if (! py_username) + goto error; + py_tty = PyUnicode_DecodeFSDefault(utx->ut_line); + if (! py_tty) + goto error; + py_hostname = PyUnicode_DecodeFSDefault(utx->ut_host); + if (! py_hostname) + goto error; py_tuple = Py_BuildValue( - "(sssfi)", - utx->ut_user, // username - utx->ut_line, // tty - utx->ut_host, // hostname + "(OOOfi)", + py_username, // username + py_tty, // tty + py_hostname, // hostname (float)utx->ut_tv.tv_sec, // start time utx->ut_pid // process id ); @@ -1729,6 +1726,9 @@ psutil_users(PyObject *self, PyObject *args) { endutxent(); goto error; } + Py_DECREF(py_username); + Py_DECREF(py_tty); + Py_DECREF(py_hostname); Py_DECREF(py_tuple); } @@ -1736,6 +1736,9 @@ psutil_users(PyObject *self, PyObject *args) { return py_retlist; error: + Py_XDECREF(py_username); + Py_XDECREF(py_tty); + Py_XDECREF(py_hostname); Py_XDECREF(py_tuple); Py_DECREF(py_retlist); return NULL; diff --git a/psutil/_psutil_sunos.c b/psutil/_psutil_sunos.c index 3e2262ff..c205a3ac 100644 --- a/psutil/_psutil_sunos.c +++ b/psutil/_psutil_sunos.c @@ -123,6 +123,7 @@ psutil_proc_name_and_args(PyObject *self, PyObject *args) { const char *procfs_path; PyObject *py_name; PyObject *py_args; + PyObject *py_retlist; if (! PyArg_ParseTuple(args, "is", &pid, &procfs_path)) return NULL; @@ -130,17 +131,24 @@ psutil_proc_name_and_args(PyObject *self, PyObject *args) { if (! psutil_file_to_struct(path, (void *)&info, sizeof(info))) return NULL; -#if PY_MAJOR_VERSION >= 3 py_name = PyUnicode_DecodeFSDefault(info.pr_fname); if (!py_name) - return NULL; + goto error; py_args = PyUnicode_DecodeFSDefault(info.pr_psargs); if (!py_args) - return NULL; - return Py_BuildValue("OO", py_name, py_args); -#else - return Py_BuildValue("ss", info.pr_fname, info.pr_psargs); -#endif + goto error; + py_retlist = Py_BuildValue("OO", py_name, py_args); + if (!py_retlist) + goto error; + Py_DECREF(py_name); + Py_DECREF(py_args); + return py_retlist; + +error: + Py_XDECREF(py_name); + Py_XDECREF(py_args); + Py_XDECREF(py_retlist); + return NULL; } @@ -450,9 +458,12 @@ psutil_swap_mem(PyObject *self, PyObject *args) { static PyObject * psutil_users(PyObject *self, PyObject *args) { struct utmpx *ut; - PyObject *py_retlist = PyList_New(0); PyObject *py_tuple = NULL; + PyObject *py_username = NULL; + PyObject *py_tty = NULL; + PyObject *py_hostname = NULL; PyObject *py_user_proc = NULL; + PyObject *py_retlist = PyList_New(0); if (py_retlist == NULL) return NULL; @@ -462,11 +473,20 @@ psutil_users(PyObject *self, PyObject *args) { py_user_proc = Py_True; else py_user_proc = Py_False; + py_username = PyUnicode_DecodeFSDefault(ut->ut_user); + if (! py_username) + goto error; + py_tty = PyUnicode_DecodeFSDefault(ut->ut_line); + if (! py_tty) + goto error; + py_hostname = PyUnicode_DecodeFSDefault(ut->ut_host); + if (! py_hostname) + goto error; py_tuple = Py_BuildValue( - "(sssfOi)", - ut->ut_user, // username - ut->ut_line, // tty - ut->ut_host, // hostname + "(OOOfOi)", + py_username, // username + py_tty, // tty + py_hostname, // hostname (float)ut->ut_tv.tv_sec, // tstamp py_user_proc, // (bool) user process ut->ut_pid // process id @@ -475,6 +495,9 @@ psutil_users(PyObject *self, PyObject *args) { goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_username); + Py_DECREF(py_tty); + Py_DECREF(py_hostname); Py_DECREF(py_tuple); } endutent(); @@ -482,6 +505,9 @@ psutil_users(PyObject *self, PyObject *args) { return py_retlist; error: + Py_XDECREF(py_username); + Py_XDECREF(py_tty); + Py_XDECREF(py_hostname); Py_XDECREF(py_tuple); Py_DECREF(py_retlist); if (ut != NULL) @@ -498,8 +524,10 @@ static PyObject * psutil_disk_partitions(PyObject *self, PyObject *args) { FILE *file; struct mnttab mt; - PyObject *py_retlist = PyList_New(0); + PyObject *py_dev = NULL; + PyObject *py_mountp = NULL; PyObject *py_tuple = NULL; + PyObject *py_retlist = PyList_New(0); if (py_retlist == NULL) return NULL; @@ -511,23 +539,32 @@ psutil_disk_partitions(PyObject *self, PyObject *args) { } while (getmntent(file, &mt) == 0) { + py_dev = PyUnicode_DecodeFSDefault(mt.mnt_special); + if (! py_dev) + goto error; + py_mountp = PyUnicode_DecodeFSDefault(mt.mnt_mountp); + if (! py_mountp) + goto error; py_tuple = Py_BuildValue( - "(ssss)", - mt.mnt_special, // device - mt.mnt_mountp, // mount point + "(OOss)", + py_dev, // device + py_mountp, // mount point mt.mnt_fstype, // fs type mt.mnt_mntopts); // options if (py_tuple == NULL) goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_dev); + Py_DECREF(py_mountp); Py_DECREF(py_tuple); - } fclose(file); return py_retlist; error: + Py_XDECREF(py_dev); + Py_XDECREF(py_mountp); Py_XDECREF(py_tuple); Py_DECREF(py_retlist); if (file != NULL) @@ -668,6 +705,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { const char *procfs_path; PyObject *py_tuple = NULL; + PyObject *py_path = NULL; PyObject *py_retlist = PyList_New(0); if (py_retlist == NULL) @@ -746,12 +784,15 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { } } + py_path = PyUnicode_DecodeFSDefault(name); + if (! py_path) + goto error; py_tuple = Py_BuildValue( - "iisslll", + "iisOlll", p->pr_vaddr, pr_addr_sz, perms, - name, + py_path, (long)p->pr_rss * p->pr_pagesize, (long)p->pr_anon * p->pr_pagesize, (long)p->pr_locked * p->pr_pagesize); @@ -759,6 +800,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_path); Py_DECREF(py_tuple); // increment pointer @@ -773,6 +815,7 @@ error: if (fd != -1) close(fd); Py_XDECREF(py_tuple); + Py_XDECREF(py_path); Py_DECREF(py_retlist); if (xmap != NULL) free(xmap); diff --git a/psutil/_psutil_windows.c b/psutil/_psutil_windows.c index 20d20b82..53deadcc 100644 --- a/psutil/_psutil_windows.c +++ b/psutil/_psutil_windows.c @@ -669,7 +669,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { return PyErr_SetFromWindowsErr(0); } CloseHandle(hProcess); - return Py_BuildValue("u", exe); + return PyUnicode_FromWideChar(exe, wcslen(exe)); } @@ -1309,13 +1309,14 @@ psutil_proc_username(PyObject *self, PyObject *args) { HANDLE tokenHandle = NULL; PTOKEN_USER user = NULL; ULONG bufferSize; - PTSTR name = NULL; + WCHAR *name = NULL; + WCHAR *domainName = NULL; ULONG nameSize; - PTSTR domainName = NULL; ULONG domainNameSize; SID_NAME_USE nameUse; - PTSTR fullName = NULL; - PyObject *py_unicode; + PyObject *py_username; + PyObject *py_domain; + PyObject *py_tuple; if (! PyArg_ParseTuple(args, "l", &pid)) return NULL; @@ -1363,18 +1364,18 @@ psutil_proc_username(PyObject *self, PyObject *args) { nameSize = 0x100; domainNameSize = 0x100; while (1) { - name = malloc(nameSize * sizeof(TCHAR)); + name = malloc(nameSize * sizeof(WCHAR)); if (name == NULL) { PyErr_NoMemory(); goto error; } - domainName = malloc(domainNameSize * sizeof(TCHAR)); + domainName = malloc(domainNameSize * sizeof(WCHAR)); if (domainName == NULL) { PyErr_NoMemory(); goto error; } - if (!LookupAccountSid(NULL, user->User.Sid, name, &nameSize, - domainName, &domainNameSize, &nameUse)) + if (!LookupAccountSidW(NULL, user->User.Sid, name, &nameSize, + domainName, &domainNameSize, &nameUse)) { if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { free(name); @@ -1389,45 +1390,38 @@ psutil_proc_username(PyObject *self, PyObject *args) { break; } - // build the "domain\\username" username string - nameSize = _tcslen(name); - domainNameSize = _tcslen(domainName); - fullName = malloc((domainNameSize + 1 + nameSize + 1) * sizeof(TCHAR)); - if (fullName == NULL) { - PyErr_NoMemory(); + py_domain = PyUnicode_FromWideChar(domainName, wcslen(domainName)); + if (! py_domain) goto error; - } - memcpy(fullName, domainName, domainNameSize); - fullName[domainNameSize] = '\\'; - memcpy(&fullName[domainNameSize + 1], name, nameSize); - fullName[domainNameSize + 1 + nameSize] = '\0'; + py_username = PyUnicode_FromWideChar(name, wcslen(name)); + if (! py_username) + goto error; + py_tuple = Py_BuildValue("OO", py_domain, py_username); + if (! py_tuple) + goto error; + Py_DECREF(py_domain); + Py_DECREF(py_username); -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 - py_unicode = PyUnicode_DecodeLocaleAndSize( - fullName, _tcslen(fullName), "surrogateescape"); -#else - py_unicode = PyUnicode_Decode( - fullName, _tcslen(fullName), Py_FileSystemDefaultEncoding, "replace"); -#endif - free(fullName); free(name); free(domainName); free(user); - return py_unicode; + + return py_tuple; error: if (processHandle != NULL) CloseHandle(processHandle); if (tokenHandle != NULL) CloseHandle(tokenHandle); - if (fullName != NULL) - free(fullName); if (name != NULL) free(name); if (domainName != NULL) free(domainName); if (user != NULL) free(user); + Py_XDECREF(py_domain); + Py_XDECREF(py_username); + Py_XDECREF(py_tuple); return NULL; } @@ -2565,7 +2559,7 @@ error: static PyObject * psutil_users(PyObject *self, PyObject *args) { HANDLE hServer = WTS_CURRENT_SERVER_HANDLE; - LPTSTR buffer_user = NULL; + WCHAR *buffer_user = NULL; LPTSTR buffer_addr = NULL; PWTS_SESSION_INFO sessions = NULL; DWORD count; @@ -2584,7 +2578,7 @@ psutil_users(PyObject *self, PyObject *args) { PyObject *py_retlist = PyList_New(0); PyObject *py_tuple = NULL; PyObject *py_address = NULL; - PyObject *py_buffer_user_encoded = NULL; + PyObject *py_username = NULL; if (py_retlist == NULL) return NULL; @@ -2612,12 +2606,12 @@ psutil_users(PyObject *self, PyObject *args) { // username bytes = 0; - if (WTSQuerySessionInformation(hServer, sessionId, WTSUserName, - &buffer_user, &bytes) == 0) { + if (WTSQuerySessionInformationW(hServer, sessionId, WTSUserName, + &buffer_user, &bytes) == 0) { PyErr_SetFromWindowsErr(0); goto error; } - if (bytes == 1) + if (bytes <= 2) continue; // address @@ -2661,24 +2655,18 @@ psutil_users(PyObject *self, PyObject *args) { station_info.ConnectTime.dwLowDateTime - 116444736000000000LL; unix_time /= 10000000; -#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3 - py_buffer_user_encoded = PyUnicode_DecodeLocaleAndSize( - buffer_user, _tcslen(buffer_user), "surrogateescape"); -#else - py_buffer_user_encoded = PyUnicode_Decode( - buffer_user, _tcslen(buffer_user), Py_FileSystemDefaultEncoding, - "replace"); -#endif - - if (py_buffer_user_encoded == NULL) + py_username = PyUnicode_FromWideChar(buffer_user, wcslen(buffer_user)); + if (py_username == NULL) goto error; - py_tuple = Py_BuildValue("OOd", py_buffer_user_encoded, py_address, + py_tuple = Py_BuildValue("OOd", + py_username, + py_address, (double)unix_time); if (!py_tuple) goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; - Py_XDECREF(py_buffer_user_encoded); + Py_XDECREF(py_username); Py_XDECREF(py_address); Py_XDECREF(py_tuple); } @@ -2690,7 +2678,7 @@ psutil_users(PyObject *self, PyObject *args) { return py_retlist; error: - Py_XDECREF(py_buffer_user_encoded); + Py_XDECREF(py_username); Py_XDECREF(py_tuple); Py_XDECREF(py_address); Py_DECREF(py_retlist); @@ -2862,7 +2850,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { HANDLE hProcess = NULL; PVOID baseAddress; PVOID previousAllocationBase; - CHAR mappedFileName[MAX_PATH]; + LPWSTR mappedFileName[MAX_PATH]; SYSTEM_INFO system_info; LPVOID maxAddr; PyObject *py_retlist = PyList_New(0); @@ -2888,20 +2876,13 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { py_tuple = NULL; if (baseAddress > maxAddr) break; - if (GetMappedFileNameA(hProcess, baseAddress, mappedFileName, + if (GetMappedFileNameW(hProcess, baseAddress, mappedFileName, sizeof(mappedFileName))) { - -#if PY_MAJOR_VERSION >= 3 - py_str = PyUnicode_Decode( - mappedFileName, _tcslen(mappedFileName), - Py_FileSystemDefaultEncoding, "surrogateescape"); -#else - py_str = Py_BuildValue("s", mappedFileName); -#endif + py_str = PyUnicode_FromWideChar(mappedFileName, + wcslen(mappedFileName)); if (py_str == NULL) goto error; - #ifdef _WIN64 py_tuple = Py_BuildValue( "(KsOI)", @@ -3055,11 +3036,7 @@ psutil_net_if_addrs(PyObject *self, PyObject *args) { } *--ptr = '\0'; -#if PY_MAJOR_VERSION >= 3 - py_mac_address = PyUnicode_FromString(buff); -#else - py_mac_address = PyString_FromString(buff); -#endif + py_mac_address = Py_BuildValue("s", buff); if (py_mac_address == NULL) goto error; diff --git a/psutil/_pswindows.py b/psutil/_pswindows.py index 54b094b3..f2cf49e1 100644 --- a/psutil/_pswindows.py +++ b/psutil/_pswindows.py @@ -36,11 +36,12 @@ from ._common import isfile_strict from ._common import parse_environ_block from ._common import sockfam_to_enum from ._common import socktype_to_enum -from ._common import usage_percent from ._common import memoize_when_activated +from ._common import usage_percent from ._compat import long from ._compat import lru_cache from ._compat import PY3 +from ._compat import unicode from ._compat import xrange from ._psutil_windows import ABOVE_NORMAL_PRIORITY_CLASS from ._psutil_windows import BELOW_NORMAL_PRIORITY_CLASS @@ -70,7 +71,8 @@ __extra__all__ = [ # --- globals # ===================================================================== - +FS_ENCODING = sys.getfilesystemencoding() +PY2_ENCODING_ERRS = "replace" CONN_DELETE_TCB = "DELETE_TCB" WAIT_TIMEOUT = 0x00000102 # 258 in decimal ACCESS_DENIED_SET = frozenset([errno.EPERM, errno.EACCES, @@ -181,26 +183,22 @@ def convert_dos_path(s): into: "C:\Windows\systemew\file.txt" """ - if PY3 and not isinstance(s, str): - s = s.decode('utf8') rawdrive = '\\'.join(s.split('\\')[:3]) driveletter = cext.win32_QueryDosDevice(rawdrive) return os.path.join(driveletter, s[len(rawdrive):]) -def py2_strencode(s, encoding=sys.getfilesystemencoding()): - """Encode a string in the given encoding. Falls back on returning - the string as is if it can't be encoded. +def py2_strencode(s): + """Encode a unicode string to a byte string by using the default fs + encoding + "replace" error handler. """ - if PY3 or isinstance(s, str): + if PY3: return s else: - try: - return s.encode(encoding) - except UnicodeEncodeError: - # Filesystem codec failed, return the plain unicode - # string (this should never happen). + if isinstance(s, str): return s + else: + return s.encode(FS_ENCODING, errors=PY2_ENCODING_ERRS) # ===================================================================== @@ -341,9 +339,12 @@ def net_connections(kind, _pid=-1): def net_if_stats(): """Get NIC stats (isup, duplex, speed, mtu).""" - ret = cext.net_if_stats() - for name, items in ret.items(): - name = py2_strencode(name) + ret = {} + rawdict = cext.net_if_stats() + for name, items in rawdict.items(): + if not PY3: + assert isinstance(name, unicode), type(name) + name = py2_strencode(name) isup, duplex, speed, mtu = items if hasattr(_common, 'NicDuplex'): duplex = _common.NicDuplex(duplex) @@ -422,9 +423,9 @@ def users(): def win_service_iter(): - """Return a list of WindowsService instances.""" + """Yields a list of WindowsService instances.""" for name, display_name in cext.winservice_enumerate(): - yield WindowsService(name, display_name) + yield WindowsService(py2_strencode(name), py2_strencode(display_name)) def win_service_get(name): @@ -465,10 +466,10 @@ class WindowsService(object): cext.winservice_query_config(self._name) # XXX - update _self.display_name? return dict( - display_name=display_name, - binpath=binpath, - username=username, - start_type=start_type) + display_name=py2_strencode(display_name), + binpath=py2_strencode(binpath), + username=py2_strencode(username), + start_type=py2_strencode(start_type)) def _query_status(self): with self._wrap_exceptions(): @@ -545,7 +546,7 @@ class WindowsService(object): def description(self): """Service long description.""" - return cext.winservice_query_descr(self.name()) + return py2_strencode(cext.winservice_query_descr(self.name())) # utils @@ -695,7 +696,10 @@ class Process(object): @wrap_exceptions def environ(self): - return parse_environ_block(cext.proc_environ(self.pid)) + ustr = cext.proc_environ(self.pid) + if ustr and not PY3: + assert isinstance(ustr, unicode), type(ustr) + return parse_environ_block(py2_strencode(ustr)) def ppid(self): try: @@ -755,6 +759,9 @@ class Process(object): else: for addr, perm, path, rss in raw: path = convert_dos_path(path) + if not PY3: + assert isinstance(path, unicode), type(path) + path = py2_strencode(path) addr = hex(addr) yield (addr, perm, path, rss) @@ -782,7 +789,8 @@ class Process(object): def username(self): if self.pid in (0, 4): return 'NT AUTHORITY\\SYSTEM' - return cext.proc_username(self.pid) + domain, user = cext.proc_username(self.pid) + return py2_strencode(domain) + '\\' + py2_strencode(user) @wrap_exceptions def create_time(self): diff --git a/psutil/arch/bsd/freebsd.c b/psutil/arch/bsd/freebsd.c index 2188564f..11594b9d 100644 --- a/psutil/arch/bsd/freebsd.c +++ b/psutil/arch/bsd/freebsd.c @@ -238,11 +238,7 @@ psutil_get_cmdline(long pid) { // separator if (argsize > 0) { while (pos < argsize) { -#if PY_MAJOR_VERSION >= 3 py_arg = PyUnicode_DecodeFSDefault(&argstr[pos]); -#else - py_arg = Py_BuildValue("s", &argstr[pos]); -#endif if (!py_arg) goto error; if (PyList_Append(py_retlist, py_arg)) @@ -292,7 +288,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { if (error == -1) { // see: https://github.com/giampaolo/psutil/issues/907 if (errno == ENOENT) - return Py_BuildValue("s", ""); + return PyUnicode_DecodeFSDefault(""); else return PyErr_SetFromErrno(PyExc_OSError); } @@ -306,12 +302,7 @@ psutil_proc_exe(PyObject *self, PyObject *args) { strcpy(pathname, ""); } -#if PY_MAJOR_VERSION >= 3 return PyUnicode_DecodeFSDefault(pathname); -#else - return Py_BuildValue("s", pathname); -#endif - } @@ -564,11 +555,7 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { for (i = 0; i < cnt; i++) { kif = &freep[i]; if (kif->kf_fd == KF_FD_TYPE_CWD) { -#if PY_MAJOR_VERSION >= 3 py_path = PyUnicode_DecodeFSDefault(kif->kf_path); -#else - py_path = Py_BuildValue("s", kif->kf_path); -#endif if (!py_path) goto error; break; @@ -580,7 +567,7 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { * as root we return an empty string instead of AccessDenied. */ if (py_path == NULL) - py_path = Py_BuildValue("s", ""); + py_path = PyUnicode_DecodeFSDefault(""); free(freep); return py_path; @@ -759,12 +746,13 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { int i, cnt; char addr[1000]; char perms[4]; - const char *path; + char *path; struct kinfo_proc kp; struct kinfo_vmentry *freep = NULL; struct kinfo_vmentry *kve; ptrwidth = 2 * sizeof(void *); PyObject *py_tuple = NULL; + PyObject *py_path = NULL; PyObject *py_retlist = PyList_New(0); if (py_retlist == NULL) @@ -833,10 +821,13 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { path = kve->kve_path; } - py_tuple = Py_BuildValue("sssiiii", + py_path = PyUnicode_DecodeFSDefault(path); + if (! py_path) + goto error; + py_tuple = Py_BuildValue("ssOiiii", addr, // "start-end" address perms, // "rwx" permissions - path, // path + py_path, // path kve->kve_resident, // rss kve->kve_private_resident, // private kve->kve_ref_count, // ref count @@ -845,6 +836,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_path); Py_DECREF(py_tuple); } free(freep); @@ -852,6 +844,7 @@ psutil_proc_memory_maps(PyObject *self, PyObject *args) { error: Py_XDECREF(py_tuple); + Py_XDECREF(py_path); Py_DECREF(py_retlist); if (freep != NULL) free(freep); diff --git a/psutil/arch/bsd/freebsd_socks.c b/psutil/arch/bsd/freebsd_socks.c index 521868ba..c7a26323 100644 --- a/psutil/arch/bsd/freebsd_socks.c +++ b/psutil/arch/bsd/freebsd_socks.c @@ -358,8 +358,7 @@ int psutil_gather_unix(int proto, PyObject *py_retlist) { char path[PATH_MAX]; PyObject *py_tuple = NULL; - PyObject *py_laddr = NULL; - PyObject *py_raddr = NULL; + PyObject *py_lpath = NULL; switch (proto) { case SOCK_STREAM: @@ -418,13 +417,23 @@ int psutil_gather_unix(int proto, PyObject *py_retlist) { snprintf(path, sizeof(path), "%.*s", (int)(sun->sun_len - (sizeof(*sun) - sizeof(sun->sun_path))), sun->sun_path); + py_lpath = PyUnicode_DecodeFSDefault(path); + if (! py_lpath) + goto error; - py_tuple = Py_BuildValue("(iiisOii)", -1, AF_UNIX, proto, path, - Py_None, PSUTIL_CONN_NONE, pid); + py_tuple = Py_BuildValue("(iiiOOii)", + -1, + AF_UNIX, + proto, + py_lpath, + Py_None, + PSUTIL_CONN_NONE, + pid); if (!py_tuple) goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_lpath); Py_DECREF(py_tuple); Py_INCREF(Py_None); } @@ -434,8 +443,7 @@ int psutil_gather_unix(int proto, PyObject *py_retlist) { error: Py_XDECREF(py_tuple); - Py_XDECREF(py_laddr); - Py_XDECREF(py_raddr); + Py_XDECREF(py_lpath); free(buf); return 0; } @@ -597,11 +605,7 @@ psutil_proc_connections(PyObject *self, PyObject *args) { (int)(sun->sun_len - (sizeof(*sun) - sizeof(sun->sun_path))), sun->sun_path); -#if PY_MAJOR_VERSION >= 3 py_laddr = PyUnicode_DecodeFSDefault(path); -#else - py_laddr = Py_BuildValue("s", path); -#endif if (! py_laddr) goto error; diff --git a/psutil/arch/bsd/netbsd.c b/psutil/arch/bsd/netbsd.c index 361ab1f9..5748000f 100644 --- a/psutil/arch/bsd/netbsd.c +++ b/psutil/arch/bsd/netbsd.c @@ -119,6 +119,7 @@ kinfo_getfile(pid_t pid, int* cnt) { // https://github.com/giampaolo/psutil/pull/557#issuecomment-171912820 // Current implementation uses /proc instead. // Left here just in case. +/* PyObject * psutil_proc_exe(PyObject *self, PyObject *args) { #if __NetBSD_Version__ >= 799000000 @@ -163,16 +164,12 @@ psutil_proc_exe(PyObject *self, PyObject *args) { strcpy(pathname, ""); } -#if PY_MAJOR_VERSION >= 3 return PyUnicode_DecodeFSDefault(pathname); #else - return Py_BuildValue("s", pathname); -#endif - -#else return Py_BuildValue("s", ""); #endif } +*/ PyObject * psutil_proc_num_threads(PyObject *self, PyObject *args) { @@ -386,11 +383,7 @@ psutil_get_cmdline(pid_t pid) { // separator if (argsize > 0) { while (pos < argsize) { -#if PY_MAJOR_VERSION >= 3 py_arg = PyUnicode_DecodeFSDefault(&argstr[pos]); -#else - py_arg = Py_BuildValue("s", &argstr[pos]); -#endif if (!py_arg) goto error; if (PyList_Append(py_retlist, py_arg)) diff --git a/psutil/arch/bsd/netbsd_socks.c b/psutil/arch/bsd/netbsd_socks.c index d05981d2..06b7c755 100644 --- a/psutil/arch/bsd/netbsd_socks.c +++ b/psutil/arch/bsd/netbsd_socks.c @@ -406,19 +406,17 @@ psutil_net_connections(PyObject *self, PyObject *args) { strcpy(laddr, sun_src->sun_path); strcpy(raddr, sun_dst->sun_path); status = PSUTIL_CONN_NONE; - // TODO: handle unicode - py_laddr = Py_BuildValue("s", laddr); + py_laddr = PyUnicode_DecodeFSDefault(laddr); if (! py_laddr) goto error; - // TODO: handle unicode - py_raddr = Py_BuildValue("s", raddr); + py_raddr = PyUnicode_DecodeFSDefault(raddr); if (! py_raddr) goto error; } // append tuple to list py_tuple = Py_BuildValue( - "(iiiNNii)", + "(iiiOOii)", k->kif->ki_fd, kp->kpcb->ki_family, kp->kpcb->ki_type, @@ -430,6 +428,8 @@ psutil_net_connections(PyObject *self, PyObject *args) { goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; + Py_DECREF(py_laddr); + Py_DECREF(py_raddr); Py_DECREF(py_tuple); } } diff --git a/psutil/arch/bsd/openbsd.c b/psutil/arch/bsd/openbsd.c index fa6edc02..8891c461 100644 --- a/psutil/arch/bsd/openbsd.c +++ b/psutil/arch/bsd/openbsd.c @@ -203,11 +203,7 @@ psutil_get_cmdline(long pid) { goto error; for (p = argv; *p != NULL; p++) { -#if PY_MAJOR_VERSION >= 3 py_arg = PyUnicode_DecodeFSDefault(*p); -#else - py_arg = Py_BuildValue("s", *p); -#endif if (!py_arg) goto error; if (PyList_Append(py_retlist, py_arg)) @@ -435,11 +431,7 @@ psutil_proc_cwd(PyObject *self, PyObject *args) { PyErr_SetFromErrno(PyExc_OSError); return NULL; } -#if PY_MAJOR_VERSION >= 3 return PyUnicode_DecodeFSDefault(path); -#else - return Py_BuildValue("s", path); -#endif } diff --git a/psutil/arch/osx/process_info.c b/psutil/arch/osx/process_info.c index 1c97b69f..7d6861a5 100644 --- a/psutil/arch/osx/process_info.c +++ b/psutil/arch/osx/process_info.c @@ -177,12 +177,8 @@ psutil_get_cmdline(long pid) { goto error; while (arg_ptr < arg_end && nargs > 0) { if (*arg_ptr++ == '\0') { -#if PY_MAJOR_VERSION >= 3 py_arg = PyUnicode_DecodeFSDefault(curr_arg); -#else - py_arg = Py_BuildValue("s", curr_arg); -#endif - if (!py_arg) + if (! py_arg) goto error; if (PyList_Append(py_retlist, py_arg)) goto error; @@ -293,13 +289,8 @@ psutil_get_environ(long pid) { arg_ptr = s + 1; } -#if PY_MAJOR_VERSION >= 3 py_ret = PyUnicode_DecodeFSDefaultAndSize( procenv, arg_ptr - env_start + 1); -#else - py_ret = PyString_FromStringAndSize(procenv, arg_ptr - env_start + 1); -#endif - if (!py_ret) { // XXX: don't want to free() this as per: // https://github.com/giampaolo/psutil/issues/926 diff --git a/psutil/arch/windows/services.c b/psutil/arch/windows/services.c index 85ea6ff4..e82f2887 100644 --- a/psutil/arch/windows/services.c +++ b/psutil/arch/windows/services.c @@ -18,10 +18,9 @@ SC_HANDLE psutil_get_service_handler(char *service_name, DWORD scm_access, DWORD access) { - ENUM_SERVICE_STATUS_PROCESS *lpService = NULL; + ENUM_SERVICE_STATUS_PROCESSW *lpService = NULL; SC_HANDLE sc = NULL; SC_HANDLE hService = NULL; - SERVICE_DESCRIPTION *scd = NULL; sc = OpenSCManager(NULL, NULL, scm_access); if (sc == NULL) { @@ -96,7 +95,7 @@ get_state_string(DWORD state) { */ PyObject * psutil_winservice_enumerate(PyObject *self, PyObject *args) { - ENUM_SERVICE_STATUS_PROCESS *lpService = NULL; + ENUM_SERVICE_STATUS_PROCESSW *lpService = NULL; BOOL ok; SC_HANDLE sc = NULL; DWORD bytesNeeded = 0; @@ -106,7 +105,8 @@ psutil_winservice_enumerate(PyObject *self, PyObject *args) { DWORD i; PyObject *py_retlist = PyList_New(0); PyObject *py_tuple = NULL; - PyObject *py_unicode_display_name = NULL; + PyObject *py_name = NULL; + PyObject *py_display_name = NULL; if (py_retlist == NULL) return NULL; @@ -118,7 +118,7 @@ psutil_winservice_enumerate(PyObject *self, PyObject *args) { } for (;;) { - ok = EnumServicesStatusEx( + ok = EnumServicesStatusExW( sc, SC_ENUM_PROCESS_INFO, SERVICE_WIN32, // XXX - extend this to include drivers etc.? @@ -134,31 +134,31 @@ psutil_winservice_enumerate(PyObject *self, PyObject *args) { if (lpService) free(lpService); dwBytes = bytesNeeded; - lpService = (ENUM_SERVICE_STATUS_PROCESS*)malloc(dwBytes); + lpService = (ENUM_SERVICE_STATUS_PROCESSW*)malloc(dwBytes); } for (i = 0; i < srvCount; i++) { - // Get unicode display name. - py_unicode_display_name = NULL; - py_unicode_display_name = PyUnicode_Decode( - lpService[i].lpDisplayName, - _tcslen(lpService[i].lpDisplayName), - Py_FileSystemDefaultEncoding, - "replace"); - if (py_unicode_display_name == NULL) + // Get unicode name / display name. + py_name = NULL; + py_name = PyUnicode_FromWideChar( + lpService[i].lpServiceName, wcslen(lpService[i].lpServiceName)); + if (py_name == NULL) + goto error; + + py_display_name = NULL; + py_display_name = PyUnicode_FromWideChar( + lpService[i].lpDisplayName, wcslen(lpService[i].lpDisplayName)); + if (py_display_name == NULL) goto error; // Construct the result. - py_tuple = Py_BuildValue( - "(sO)", - lpService[i].lpServiceName, // name - py_unicode_display_name // display_name - ); + py_tuple = Py_BuildValue("(OO)", py_name, py_display_name); if (py_tuple == NULL) goto error; if (PyList_Append(py_retlist, py_tuple)) goto error; - Py_DECREF(py_unicode_display_name); + Py_DECREF(py_display_name); + Py_DECREF(py_name); Py_DECREF(py_tuple); } @@ -168,7 +168,8 @@ psutil_winservice_enumerate(PyObject *self, PyObject *args) { return py_retlist; error: - Py_XDECREF(py_unicode_display_name); + Py_DECREF(py_name); + Py_XDECREF(py_display_name); Py_XDECREF(py_tuple); Py_DECREF(py_retlist); if (sc != NULL) @@ -194,7 +195,7 @@ psutil_winservice_query_config(PyObject *self, PyObject *args) { DWORD bytesNeeded = 0; DWORD resumeHandle = 0; DWORD dwBytes = 0; - QUERY_SERVICE_CONFIG *qsc = NULL; + QUERY_SERVICE_CONFIGW *qsc = NULL; PyObject *py_tuple = NULL; PyObject *py_unicode_display_name = NULL; PyObject *py_unicode_binpath = NULL; @@ -207,45 +208,36 @@ psutil_winservice_query_config(PyObject *self, PyObject *args) { if (hService == NULL) goto error; - // First call to QueryServiceConfig() is necessary to get the + // First call to QueryServiceConfigW() is necessary to get the // right size. bytesNeeded = 0; - QueryServiceConfig(hService, NULL, 0, &bytesNeeded); + QueryServiceConfigW(hService, NULL, 0, &bytesNeeded); if (GetLastError() != ERROR_INSUFFICIENT_BUFFER) { PyErr_SetFromWindowsErr(0); goto error; } qsc = (QUERY_SERVICE_CONFIG *)malloc(bytesNeeded); - ok = QueryServiceConfig(hService, qsc, bytesNeeded, &bytesNeeded); + ok = QueryServiceConfigW(hService, qsc, bytesNeeded, &bytesNeeded); if (ok == 0) { PyErr_SetFromWindowsErr(0); goto error; } // Get unicode display name. - py_unicode_display_name = PyUnicode_Decode( - qsc->lpDisplayName, - _tcslen(qsc->lpDisplayName), - Py_FileSystemDefaultEncoding, - "replace"); + py_unicode_display_name = PyUnicode_FromWideChar( + qsc->lpDisplayName, wcslen(qsc->lpDisplayName)); if (py_unicode_display_name == NULL) goto error; // Get unicode bin path. - py_unicode_binpath = PyUnicode_Decode( - qsc->lpBinaryPathName, - _tcslen(qsc->lpBinaryPathName), - Py_FileSystemDefaultEncoding, - "replace"); + py_unicode_binpath = PyUnicode_FromWideChar( + qsc->lpBinaryPathName, wcslen(qsc->lpBinaryPathName)); if (py_unicode_binpath == NULL) goto error; // Get unicode username. - py_unicode_username = PyUnicode_Decode( - qsc->lpServiceStartName, - _tcslen(qsc->lpServiceStartName), - Py_FileSystemDefaultEncoding, - "replace"); + py_unicode_username = PyUnicode_FromWideChar( + qsc->lpServiceStartName, wcslen(qsc->lpServiceStartName)); if (py_unicode_username == NULL) goto error; @@ -360,13 +352,13 @@ error: */ PyObject * psutil_winservice_query_descr(PyObject *self, PyObject *args) { - ENUM_SERVICE_STATUS_PROCESS *lpService = NULL; + ENUM_SERVICE_STATUS_PROCESSW *lpService = NULL; BOOL ok; DWORD bytesNeeded = 0; DWORD resumeHandle = 0; DWORD dwBytes = 0; SC_HANDLE hService = NULL; - SERVICE_DESCRIPTION *scd = NULL; + SERVICE_DESCRIPTIONW *scd = NULL; char *service_name; PyObject *py_retstr = NULL; @@ -377,11 +369,11 @@ psutil_winservice_query_descr(PyObject *self, PyObject *args) { if (hService == NULL) goto error; - // This first call to QueryServiceConfig2() is necessary in order + // This first call to QueryServiceConfig2W() is necessary in order // to get the right size. bytesNeeded = 0; - QueryServiceConfig2(hService, SERVICE_CONFIG_DESCRIPTION, NULL, 0, - &bytesNeeded); + QueryServiceConfig2W(hService, SERVICE_CONFIG_DESCRIPTION, NULL, 0, + &bytesNeeded); if (GetLastError() == ERROR_MUI_FILE_NOT_FOUND) { // Also services.msc fails in the same manner, so we return an // empty string. @@ -393,9 +385,9 @@ psutil_winservice_query_descr(PyObject *self, PyObject *args) { goto error; } - scd = (SERVICE_DESCRIPTION *)malloc(bytesNeeded); - ok = QueryServiceConfig2(hService, SERVICE_CONFIG_DESCRIPTION, - (LPBYTE)scd, bytesNeeded, &bytesNeeded); + scd = (SERVICE_DESCRIPTIONW *)malloc(bytesNeeded); + ok = QueryServiceConfig2W(hService, SERVICE_CONFIG_DESCRIPTION, + (LPBYTE)scd, bytesNeeded, &bytesNeeded); if (ok == 0) { PyErr_SetFromWindowsErr(0); goto error; @@ -405,11 +397,8 @@ psutil_winservice_query_descr(PyObject *self, PyObject *args) { py_retstr = Py_BuildValue("s", ""); } else { - py_retstr = PyUnicode_Decode( - scd->lpDescription, - _tcslen(scd->lpDescription), - Py_FileSystemDefaultEncoding, - "replace"); + py_retstr = PyUnicode_FromWideChar( + scd->lpDescription, wcslen(scd->lpDescription)); } if (!py_retstr) goto error; diff --git a/psutil/tests/test_unicode.py b/psutil/tests/test_unicode.py index ce881eac..47e8d15c 100644 --- a/psutil/tests/test_unicode.py +++ b/psutil/tests/test_unicode.py @@ -12,53 +12,44 @@ Notes about unicode handling in psutil In psutil these are the APIs returning or dealing with a string ('not tested' means they are not tested to deal with non-ASCII strings): -- Process.cmdline() -- Process.connections('unix') -- Process.cwd() -- Process.environ() -- Process.exe() -- Process.memory_maps() -- Process.name() -- Process.open_files() -- Process.username() (not tested) - -- disk_io_counters() (not tested) -- disk_partitions() (not tested) -- disk_usage(str) -- net_connections('unix') -- net_if_addrs() (not tested) -- net_if_stats() (not tested) -- net_io_counters() (not tested) -- sensors_fans() (not tested) -- sensors_temperatures() (not tested) -- users() (not tested) - -- WindowsService.binpath() (not tested) -- WindowsService.description() (not tested) -- WindowsService.display_name() (not tested) -- WindowsService.name() (not tested) -- WindowsService.status() (not tested) -- WindowsService.username() (not tested) +* Process.cmdline() +* Process.connections('unix') +* Process.cwd() +* Process.environ() +* Process.exe() +* Process.memory_maps() +* Process.name() +* Process.open_files() +* Process.username() (not tested) + +* disk_io_counters() (not tested) +* disk_partitions() (not tested) +* disk_usage(str) +* net_connections('unix') +* net_if_addrs() (not tested) +* net_if_stats() (not tested) +* net_io_counters() (not tested) +* sensors_fans() (not tested) +* sensors_temperatures() (not tested) +* users() (not tested) + +* WindowsService.binpath() (not tested) +* WindowsService.description() (not tested) +* WindowsService.display_name() (not tested) +* WindowsService.name() (not tested) +* WindowsService.status() (not tested) +* WindowsService.username() (not tested) In here we create a unicode path with a funky non-ASCII name and (where -possible) make psutil return it back (e.g. on name(), exe(), -open_files(), etc.) and make sure psutil never crashes with -UnicodeDecodeError. - -On Python 3 the returned path is supposed to match 100% (and this -is tested). -Not on Python 2 though, where we assume correct unicode path handling -is broken. In fact it is broken for most os.* functions, see: -http://bugs.python.org/issue18695 -There really is no way for psutil to handle unicode correctly on -Python 2 unless we make such APIs return a unicode type in certain -circumstances. -I'd rather have unicode support broken on Python 2 than having APIs -returning variable str/unicode types, see: -https://github.com/giampaolo/psutil/issues/655#issuecomment-136131180 - -As such we also test that all APIs on Python 2 always return str and -never unicode (in test_contracts.py). +possible) make psutil return it back (e.g. on name(), exe(), open_files(), +etc.) and make sure that: + +* psutil never crashes with UnicodeDecodeError +* the returned path matches + +For a detailed explanation of how psutil handles unicode see: +- https://github.com/giampaolo/psutil/issues/1040 +- https://pythonhosted.org/psutil/#unicode """ import os diff --git a/psutil/tests/test_windows.py b/psutil/tests/test_windows.py index 2433849f..ac787283 100755 --- a/psutil/tests/test_windows.py +++ b/psutil/tests/test_windows.py @@ -28,7 +28,6 @@ except ImportError: import psutil from psutil import WINDOWS -from psutil._compat import basestring from psutil._compat import callable from psutil.tests import APPVEYOR from psutil.tests import get_test_subprocess @@ -753,19 +752,19 @@ class TestServices(unittest.TestCase): ]) for serv in psutil.win_service_iter(): data = serv.as_dict() - self.assertIsInstance(data['name'], basestring) + self.assertIsInstance(data['name'], str) self.assertNotEqual(data['name'].strip(), "") - self.assertIsInstance(data['display_name'], basestring) - self.assertIsInstance(data['username'], basestring) + self.assertIsInstance(data['display_name'], str) + self.assertIsInstance(data['username'], str) self.assertIn(data['status'], valid_statuses) if data['pid'] is not None: psutil.Process(data['pid']) - self.assertIsInstance(data['binpath'], basestring) - self.assertIsInstance(data['username'], basestring) - self.assertIsInstance(data['start_type'], basestring) + self.assertIsInstance(data['binpath'], str) + self.assertIsInstance(data['username'], str) + self.assertIsInstance(data['start_type'], str) self.assertIn(data['start_type'], valid_start_types) self.assertIn(data['status'], valid_statuses) - self.assertIsInstance(data['description'], basestring) + self.assertIsInstance(data['description'], str) pid = serv.pid() if pid is not None: p = psutil.Process(pid) |