From 4d0ebf454346e70e5b7c7803ab7939f7c12a2b39 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 30 Oct 2012 01:42:39 +0100 Subject: Issue #16330: Use surrogate-related macros Patch written by Serhiy Storchaka. --- Python/fileutils.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 501cb8c8d6..526751d5ad 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -85,7 +85,7 @@ _Py_char2wchar(const char* arg, size_t *size) /* Only use the result if it contains no surrogate characters. */ for (tmp = res; *tmp != 0 && - (*tmp < 0xd800 || *tmp > 0xdfff); tmp++) + !Py_UNICODE_IS_SURROGATE(*tmp); tmp++) ; if (*tmp == 0) { if (size != NULL) @@ -131,7 +131,7 @@ _Py_char2wchar(const char* arg, size_t *size) memset(&mbs, 0, sizeof mbs); continue; } - if (*out >= 0xd800 && *out <= 0xdfff) { + if (Py_UNICODE_IS_SURROGATE(*out)) { /* Surrogate character. Escape the original byte sequence with surrogateescape. */ argsize -= converted; -- cgit v1.2.1 From 99f96c8a9dd3ab52c622c7fe6eb7173da86f3fa3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 12 Nov 2012 23:04:02 +0100 Subject: Issue #16416: OS data are now always encoded/decoded to/from UTF-8/surrogateescape, instead of the locale encoding (which may be ASCII if no locale environment variable is set), to avoid inconsistencies with os.fsencode() and os.fsdecode() functions which are already using UTF-8/surrogateescape. --- Python/fileutils.c | 47 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 2e25aae075..42a532d9fa 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -8,6 +8,10 @@ #include #endif +#ifdef __APPLE__ +extern wchar_t* _Py_DecodeUTF8_surrogateescape(const char *s, Py_ssize_t size); +#endif + PyObject * _Py_device_encoding(int fd) { @@ -60,6 +64,15 @@ _Py_device_encoding(int fd) wchar_t* _Py_char2wchar(const char* arg, size_t *size) { +#ifdef __APPLE__ + wchar_t *wstr; + wstr = _Py_DecodeUTF8_surrogateescape(arg, strlen(arg)); + if (wstr == NULL) + return NULL; + if (size != NULL) + *size = wcslen(wstr); + return wstr; +#else wchar_t *res; #ifdef HAVE_BROKEN_MBSTOWCS /* Some platforms have a broken implementation of @@ -145,7 +158,7 @@ _Py_char2wchar(const char* arg, size_t *size) argsize -= converted; out++; } -#else +#else /* HAVE_MBRTOWC */ /* Cannot use C locale for escaping; manually escape as if charset is ASCII (i.e. escape all bytes > 128. This will still roundtrip correctly in the locale's charset, which must be an ASCII superset. */ @@ -160,7 +173,7 @@ _Py_char2wchar(const char* arg, size_t *size) else *out++ = 0xdc00 + *in++; *out = 0; -#endif +#endif /* HAVE_MBRTOWC */ if (size != NULL) *size = out - res; return res; @@ -168,6 +181,7 @@ oom: if (size != NULL) *size = (size_t)-1; return NULL; +#endif /* __APPLE__ */ } /* Encode a (wide) character string to the locale encoding with the @@ -184,6 +198,34 @@ oom: char* _Py_wchar2char(const wchar_t *text, size_t *error_pos) { +#ifdef __APPLE__ + Py_ssize_t len; + PyObject *unicode, *bytes = NULL; + char *cpath; + + unicode = PyUnicode_FromWideChar(text, wcslen(text)); + if (unicode == NULL) { + Py_DECREF(unicode); + return NULL; + } + + bytes = _PyUnicode_AsUTF8String(unicode, "surrogateescape"); + Py_DECREF(unicode); + if (bytes == NULL) { + PyErr_Clear(); + return NULL; + } + + len = PyBytes_GET_SIZE(bytes); + cpath = PyMem_Malloc(len+1); + if (cpath == NULL) { + Py_DECREF(bytes); + return NULL; + } + memcpy(cpath, PyBytes_AsString(bytes), len + 1); + Py_DECREF(bytes); + return cpath; +#else /* __APPLE__ */ const size_t len = wcslen(text); char *result = NULL, *bytes = NULL; size_t i, size, converted; @@ -243,6 +285,7 @@ _Py_wchar2char(const wchar_t *text, size_t *error_pos) bytes = result; } return result; +#endif /* __APPLE__ */ } /* In principle, this should use HAVE__WSTAT, and _wstat -- cgit v1.2.1 From e037652273f080b41545e058575097645959b804 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 12 Nov 2012 23:32:21 +0100 Subject: Issue #16416: Fix error handling in _Py_wchar2char() _Py_char2wchar() functions --- Python/fileutils.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 42a532d9fa..2cd75ce163 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -67,10 +67,12 @@ _Py_char2wchar(const char* arg, size_t *size) #ifdef __APPLE__ wchar_t *wstr; wstr = _Py_DecodeUTF8_surrogateescape(arg, strlen(arg)); - if (wstr == NULL) - return NULL; - if (size != NULL) - *size = wcslen(wstr); + if (size != NULL) { + if (wstr != NULL) + *size = wcslen(wstr); + else + *size = (size_t)-1; + } return wstr; #else wchar_t *res; @@ -204,22 +206,25 @@ _Py_wchar2char(const wchar_t *text, size_t *error_pos) char *cpath; unicode = PyUnicode_FromWideChar(text, wcslen(text)); - if (unicode == NULL) { - Py_DECREF(unicode); + if (unicode == NULL) return NULL; - } bytes = _PyUnicode_AsUTF8String(unicode, "surrogateescape"); Py_DECREF(unicode); if (bytes == NULL) { PyErr_Clear(); + if (error_pos != NULL) + *error_pos = (size_t)-1; return NULL; } len = PyBytes_GET_SIZE(bytes); cpath = PyMem_Malloc(len+1); if (cpath == NULL) { + PyErr_Clear(); Py_DECREF(bytes); + if (error_pos != NULL) + *error_pos = (size_t)-1; return NULL; } memcpy(cpath, PyBytes_AsString(bytes), len + 1); @@ -231,9 +236,6 @@ _Py_wchar2char(const wchar_t *text, size_t *error_pos) size_t i, size, converted; wchar_t c, buf[2]; - if (error_pos != NULL) - *error_pos = (size_t)-1; - /* The function works in two steps: 1. compute the length of the output buffer in bytes (size) 2. outputs the bytes */ @@ -280,8 +282,11 @@ _Py_wchar2char(const wchar_t *text, size_t *error_pos) size += 1; /* nul byte at the end */ result = PyMem_Malloc(size); - if (result == NULL) + if (result == NULL) { + if (error_pos != NULL) + *error_pos = (size_t)-1; return NULL; + } bytes = result; } return result; -- cgit v1.2.1 From 2e540f952688de0d7b1a6f5926fb0ef695518eb6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 4 Dec 2012 01:34:47 +0100 Subject: Issue #16455: On FreeBSD and Solaris, if the locale is C, the ASCII/surrogateescape codec is now used, instead of the locale encoding, to decode the command line arguments. This change fixes inconsistencies with os.fsencode() and os.fsdecode() because these operating systems announces an ASCII locale encoding, whereas the ISO-8859-1 encoding is used in practice. --- Python/fileutils.c | 240 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 217 insertions(+), 23 deletions(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 2cd75ce163..9c0833e003 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -5,6 +5,7 @@ #endif #ifdef HAVE_LANGINFO_H +#include #include #endif @@ -42,7 +43,182 @@ _Py_device_encoding(int fd) Py_RETURN_NONE; } -#ifdef HAVE_STAT +#if !defined(__APPLE__) && !defined(MS_WINDOWS) +extern int _Py_normalize_encoding(const char *, char *, size_t); + +/* Workaround FreeBSD and OpenIndiana locale encoding issue with the C locale. + On these operating systems, nl_langinfo(CODESET) announces an alias of the + ASCII encoding, whereas mbstowcs() and wcstombs() functions use the + ISO-8859-1 encoding. The problem is that os.fsencode() and os.fsdecode() use + locale.getpreferredencoding() codec. For example, if command line arguments + are decoded by mbstowcs() and encoded back by os.fsencode(), we get a + UnicodeEncodeError instead of retrieving the original byte string. + + The workaround is enabled if setlocale(LC_CTYPE, NULL) returns "C", + nl_langinfo(CODESET) announces "ascii" (or an alias to ASCII), and at least + one byte in range 0x80-0xff can be decoded from the locale encoding. The + workaround is also enabled on error, for example if getting the locale + failed. + + Values of locale_is_ascii: + + 1: the workaround is used: _Py_wchar2char() uses + encode_ascii_surrogateescape() and _Py_char2wchar() uses + decode_ascii_surrogateescape() + 0: the workaround is not used: _Py_wchar2char() uses wcstombs() and + _Py_char2wchar() uses mbstowcs() + -1: unknown, need to call check_force_ascii() to get the value +*/ +static int force_ascii = -1; + +static int +check_force_ascii(void) +{ + char *loc; +#if defined(HAVE_LANGINFO_H) && defined(CODESET) + char *codeset, **alias; + char encoding[100]; + int is_ascii; + unsigned int i; + char* ascii_aliases[] = { + "ascii", + "646", + "ansi-x3.4-1968", + "ansi-x3-4-1968", + "ansi-x3.4-1986", + "cp367", + "csascii", + "ibm367", + "iso646-us", + "iso-646.irv-1991", + "iso-ir-6", + "us", + "us-ascii", + NULL + }; +#endif + + loc = setlocale(LC_CTYPE, NULL); + if (loc == NULL) + goto error; + if (strcmp(loc, "C") != 0) { + /* the LC_CTYPE locale is different than C */ + return 0; + } + +#if defined(HAVE_LANGINFO_H) && defined(CODESET) + codeset = nl_langinfo(CODESET); + if (!codeset || codeset[0] == '\0') { + /* CODESET is not set or empty */ + goto error; + } + if (!_Py_normalize_encoding(codeset, encoding, sizeof(encoding))) + goto error; + + is_ascii = 0; + for (alias=ascii_aliases; *alias != NULL; alias++) { + if (strcmp(encoding, *alias) == 0) { + is_ascii = 1; + break; + } + } + if (!is_ascii) { + /* nl_langinfo(CODESET) is not "ascii" or an alias of ASCII */ + return 0; + } + + for (i=0x80; i<0xff; i++) { + unsigned char ch; + wchar_t wch; + size_t res; + + ch = (unsigned char)i; + res = mbstowcs(&wch, (char*)&ch, 1); + if (res != (size_t)-1) { + /* decoding a non-ASCII character from the locale encoding succeed: + the locale encoding is not ASCII, force ASCII */ + return 1; + } + } + /* None of the bytes in the range 0x80-0xff can be decoded from the locale + encoding: the locale encoding is really ASCII */ + return 0; +#else + /* nl_langinfo(CODESET) is not available: always force ASCII */ + return 1; +#endif + +error: + /* if an error occured, force the ASCII encoding */ + return 1; +} + +static char* +encode_ascii_surrogateescape(const wchar_t *text, size_t *error_pos) +{ + char *result = NULL, *out; + size_t len, i; + wchar_t ch; + + if (error_pos != NULL) + *error_pos = (size_t)-1; + + len = wcslen(text); + + result = PyMem_Malloc(len + 1); /* +1 for NUL byte */ + if (result == NULL) + return NULL; + + out = result; + for (i=0; i 128. This will still roundtrip correctly in the locale's charset, which must be an ASCII superset. */ - res = PyMem_Malloc((strlen(arg)+1)*sizeof(wchar_t)); - if (!res) + res = decode_ascii_surrogateescape(arg, size); + if (res == NULL) goto oom; - in = (unsigned char*)arg; - out = res; - while(*in) - if(*in < 128) - *out++ = *in++; - else - *out++ = 0xdc00 + *in++; - *out = 0; #endif /* HAVE_MBRTOWC */ - if (size != NULL) - *size = out - res; return res; oom: if (size != NULL) @@ -236,6 +419,14 @@ _Py_wchar2char(const wchar_t *text, size_t *error_pos) size_t i, size, converted; wchar_t c, buf[2]; +#ifndef MS_WINDOWS + if (force_ascii == -1) + force_ascii = check_force_ascii(); + + if (force_ascii) + return encode_ascii_surrogateescape(text, error_pos); +#endif + /* The function works in two steps: 1. compute the length of the output buffer in bytes (size) 2. outputs the bytes */ @@ -276,7 +467,7 @@ _Py_wchar2char(const wchar_t *text, size_t *error_pos) } } if (result != NULL) { - *bytes = 0; + *bytes = '\0'; break; } @@ -320,6 +511,8 @@ _Py_wstat(const wchar_t* path, struct stat *buf) } #endif +#ifdef HAVE_STAT + /* Call _wstat() on Windows, or encode the path to the filesystem encoding and call stat() otherwise. Only fill st_mode attribute on Windows. @@ -352,6 +545,8 @@ _Py_stat(PyObject *path, struct stat *statbuf) #endif } +#endif + /* Open a file. Use _wfopen() on Windows, encode the path to the locale encoding and use fopen() otherwise. */ @@ -533,4 +728,3 @@ _Py_wgetcwd(wchar_t *buf, size_t size) #endif } -#endif -- cgit v1.2.1 From 2733e1a2064a96136de915ae8c8f13eb827df900 Mon Sep 17 00:00:00 2001 From: Philip Jenvey Date: Tue, 15 Jan 2013 13:24:12 -0800 Subject: thinko --- Python/fileutils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 9c0833e003..3c04e4969e 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -60,7 +60,7 @@ extern int _Py_normalize_encoding(const char *, char *, size_t); workaround is also enabled on error, for example if getting the locale failed. - Values of locale_is_ascii: + Values of force_ascii: 1: the workaround is used: _Py_wchar2char() uses encode_ascii_surrogateescape() and _Py_char2wchar() uses -- cgit v1.2.1 From 14fce4cbce30e3887c13e2e9b727e27b8faa69ba Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 7 May 2013 23:48:56 +0200 Subject: Fix a compiler warning: in and out are unused in _Py_char2wchar() if HAVE_MBRTOWC is not defined --- Python/fileutils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 3c04e4969e..bb0cd43500 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -254,9 +254,9 @@ _Py_char2wchar(const char* arg, size_t *size) wchar_t *res; size_t argsize; size_t count; +#ifdef HAVE_MBRTOWC unsigned char *in; wchar_t *out; -#ifdef HAVE_MBRTOWC mbstate_t mbs; #endif -- cgit v1.2.1 From 6bf2656e55b72945d36bba54dbbc3f2f285ebe71 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 5 Jun 2013 00:46:29 +0200 Subject: Issue #9566: Fix compiler warning on Windows 64-bit --- Python/fileutils.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index bb0cd43500..293cb77f55 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -707,7 +707,8 @@ wchar_t* _Py_wgetcwd(wchar_t *buf, size_t size) { #ifdef MS_WINDOWS - return _wgetcwd(buf, size); + int isize = (int)Py_MIN(size, INT_MAX); + return _wgetcwd(buf, isize); #else char fname[PATH_MAX]; wchar_t *wname; -- cgit v1.2.1 From b963769c480e670e792ee3da641ded5452d83194 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 25 Jun 2013 00:37:25 +0200 Subject: If MS_WIN64 is defined, MS_WINDOWS is also defined: #ifdef can be simplified. --- Python/fileutils.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 293cb77f55..e75985655d 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -16,13 +16,13 @@ extern wchar_t* _Py_DecodeUTF8_surrogateescape(const char *s, Py_ssize_t size); PyObject * _Py_device_encoding(int fd) { -#if defined(MS_WINDOWS) || defined(MS_WIN64) +#if defined(MS_WINDOWS) UINT cp; #endif if (!_PyVerify_fd(fd) || !isatty(fd)) { Py_RETURN_NONE; } -#if defined(MS_WINDOWS) || defined(MS_WIN64) +#if defined(MS_WINDOWS) if (fd == 0) cp = GetConsoleCP(); else if (fd == 1 || fd == 2) -- cgit v1.2.1 From 0e6215070ac12c5aecdce573f75624eaa7ed056e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 7 Jul 2013 16:25:15 +0200 Subject: Issue #18203: Replace malloc() with PyMem_RawMalloc() at Python initialization * Replace malloc() with PyMem_RawMalloc() * Replace PyMem_Malloc() with PyMem_RawMalloc() where the GIL is not held. * _Py_char2wchar() now returns a buffer allocated by PyMem_RawMalloc(), instead of PyMem_Malloc() --- Python/fileutils.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index e75985655d..7880ab0393 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -229,7 +229,7 @@ decode_ascii_surrogateescape(const char *arg, size_t *size) Use _Py_wchar2char() to encode the character string back to a byte string. Return a pointer to a newly allocated wide character string (use - PyMem_Free() to free the memory) and write the number of written wide + PyMem_RawFree() to free the memory) and write the number of written wide characters excluding the null character into *size if size is not NULL, or NULL on error (decoding or memory allocation error). If size is not NULL, *size is set to (size_t)-1 on memory error and (size_t)-2 on decoding @@ -283,7 +283,7 @@ _Py_char2wchar(const char* arg, size_t *size) argsize = mbstowcs(NULL, arg, 0); #endif if (argsize != (size_t)-1) { - res = (wchar_t *)PyMem_Malloc((argsize+1)*sizeof(wchar_t)); + res = (wchar_t *)PyMem_RawMalloc((argsize+1)*sizeof(wchar_t)); if (!res) goto oom; count = mbstowcs(res, arg, argsize+1); @@ -300,7 +300,7 @@ _Py_char2wchar(const char* arg, size_t *size) return res; } } - PyMem_Free(res); + PyMem_RawFree(res); } /* Conversion failed. Fall back to escaping with surrogateescape. */ #ifdef HAVE_MBRTOWC @@ -309,7 +309,7 @@ _Py_char2wchar(const char* arg, size_t *size) /* Overallocate; as multi-byte characters are in the argument, the actual output could use less memory. */ argsize = strlen(arg) + 1; - res = (wchar_t*)PyMem_Malloc(argsize*sizeof(wchar_t)); + res = (wchar_t*)PyMem_RawMalloc(argsize*sizeof(wchar_t)); if (!res) goto oom; in = (unsigned char*)arg; @@ -325,7 +325,7 @@ _Py_char2wchar(const char* arg, size_t *size) since we provide everything that we have - unless there is a bug in the C library, or I misunderstood how mbrtowc works. */ - PyMem_Free(res); + PyMem_RawFree(res); if (size != NULL) *size = (size_t)-2; return NULL; @@ -648,12 +648,12 @@ _Py_wreadlink(const wchar_t *path, wchar_t *buf, size_t bufsiz) return -1; } if (bufsiz <= r1) { - PyMem_Free(wbuf); + PyMem_RawFree(wbuf); errno = EINVAL; return -1; } wcsncpy(buf, wbuf, bufsiz); - PyMem_Free(wbuf); + PyMem_RawFree(wbuf); return (int)r1; } #endif @@ -689,12 +689,12 @@ _Py_wrealpath(const wchar_t *path, return NULL; } if (resolved_path_size <= r) { - PyMem_Free(wresolved_path); + PyMem_RawFree(wresolved_path); errno = EINVAL; return NULL; } wcsncpy(resolved_path, wresolved_path, resolved_path_size); - PyMem_Free(wresolved_path); + PyMem_RawFree(wresolved_path); return resolved_path; } #endif @@ -720,11 +720,11 @@ _Py_wgetcwd(wchar_t *buf, size_t size) if (wname == NULL) return NULL; if (size <= len) { - PyMem_Free(wname); + PyMem_RawFree(wname); return NULL; } wcsncpy(buf, wname, size); - PyMem_Free(wname); + PyMem_RawFree(wname); return buf; #endif } -- cgit v1.2.1 From 408ccb0a25aa0e5ecea25420a3cd6b92d4f8cacf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 7 Jul 2013 16:35:54 +0200 Subject: Issue #18203: Fix decode_ascii_surrogateescape(), use PyMem_RawMalloc() as _Py_char2wchar() --- Python/fileutils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 7880ab0393..698385504c 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -201,7 +201,7 @@ decode_ascii_surrogateescape(const char *arg, size_t *size) unsigned char *in; wchar_t *out; - res = PyMem_Malloc((strlen(arg)+1)*sizeof(wchar_t)); + res = PyMem_RawMalloc((strlen(arg)+1)*sizeof(wchar_t)); if (!res) return NULL; -- cgit v1.2.1 From e5883d6d95989121670eca012dd5bde1721747e7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 28 Aug 2013 00:53:59 +0200 Subject: Issue #18571: Implementation of the PEP 446: file descriptors and file handles are now created non-inheritable; add functions os.get/set_inheritable(), os.get/set_handle_inheritable() and socket.socket.get/set_inheritable(). --- Python/fileutils.c | 341 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 328 insertions(+), 13 deletions(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 698385504c..1cc74805cb 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -9,10 +9,29 @@ #include #endif +#ifdef HAVE_SYS_IOCTL_H +#include +#endif + +#ifdef HAVE_FCNTL_H +#include +#endif /* HAVE_FCNTL_H */ + #ifdef __APPLE__ extern wchar_t* _Py_DecodeUTF8_surrogateescape(const char *s, Py_ssize_t size); #endif +#ifdef O_CLOEXEC +/* Does open() supports the O_CLOEXEC flag? Possible values: + + -1: unknown + 0: open() ignores O_CLOEXEC flag, ex: Linux kernel older than 2.6.23 + 1: open() supports O_CLOEXEC flag, close-on-exec is set + + The flag is used by _Py_open(), io.FileIO and os.open() */ +int _Py_open_cloexec_works = -1; +#endif + PyObject * _Py_device_encoding(int fd) { @@ -547,14 +566,215 @@ _Py_stat(PyObject *path, struct stat *statbuf) #endif -/* Open a file. Use _wfopen() on Windows, encode the path to the locale - encoding and use fopen() otherwise. */ +int +get_inheritable(int fd, int raise) +{ +#ifdef MS_WINDOWS + HANDLE handle; + DWORD flags; + + if (!_PyVerify_fd(fd)) { + if (raise) + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + handle = (HANDLE)_get_osfhandle(fd); + if (handle == INVALID_HANDLE_VALUE) { + if (raise) + PyErr_SetFromWindowsErr(0); + return -1; + } + + if (!GetHandleInformation(handle, &flags)) { + if (raise) + PyErr_SetFromWindowsErr(0); + return -1; + } + + return (flags & HANDLE_FLAG_INHERIT); +#else + int flags; + + flags = fcntl(fd, F_GETFD, 0); + if (flags == -1) { + if (raise) + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + return !(flags & FD_CLOEXEC); +#endif +} + +/* Get the inheritable flag of the specified file descriptor. + Return 1 if it the file descriptor can be inherited, 0 if it cannot, + raise an exception and return -1 on error. */ +int +_Py_get_inheritable(int fd) +{ + return get_inheritable(fd, 1); +} + +static int +set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works) +{ +#ifdef MS_WINDOWS + HANDLE handle; + DWORD flags; +#elif defined(HAVE_SYS_IOCTL_H) && defined(FIOCLEX) && defined(FIONCLEX) + int request; + int err; +#elif defined(HAVE_FCNTL_H) + int flags; + int res; +#endif + + /* atomic_flag_works can only be used to make the file descriptor + non-inheritable */ + assert(!(atomic_flag_works != NULL && inheritable)); + + if (atomic_flag_works != NULL && !inheritable) { + if (*atomic_flag_works == -1) { + int inheritable = get_inheritable(fd, raise); + if (inheritable == -1) + return -1; + *atomic_flag_works = !inheritable; + } + + if (*atomic_flag_works) + return 0; + } + +#ifdef MS_WINDOWS + if (!_PyVerify_fd(fd)) { + if (raise) + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + handle = (HANDLE)_get_osfhandle(fd); + if (handle == INVALID_HANDLE_VALUE) { + if (raise) + PyErr_SetFromWindowsErr(0); + return -1; + } + + if (inheritable) + flags = HANDLE_FLAG_INHERIT; + else + flags = 0; + if (!SetHandleInformation(handle, HANDLE_FLAG_INHERIT, flags)) { + if (raise) + PyErr_SetFromWindowsErr(0); + return -1; + } + return 0; + +#elif defined(HAVE_SYS_IOCTL_H) && defined(FIOCLEX) && defined(FIONCLEX) + if (inheritable) + request = FIONCLEX; + else + request = FIOCLEX; + err = ioctl(fd, request); + if (err) { + if (raise) + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + return 0; + +#else + flags = fcntl(fd, F_GETFD); + if (flags < 0) { + if (raise) + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + if (inheritable) + flags &= ~FD_CLOEXEC; + else + flags |= FD_CLOEXEC; + res = fcntl(fd, F_SETFD, flags); + if (res < 0) { + if (raise) + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + return 0; +#endif +} + +/* Make the file descriptor non-inheritable. + Return 0 success, set errno and return -1 on error. */ +static int +make_non_inheritable(int fd) +{ + return set_inheritable(fd, 0, 0, NULL); +} +/* Set the inheritable flag of the specified file descriptor. + On success: return 0, on error: raise an exception if raise is nonzero + and return -1. + + If atomic_flag_works is not NULL: + + * if *atomic_flag_works==-1, check if the inheritable is set on the file + descriptor: if yes, set *atomic_flag_works to 1, otherwise set to 0 and + set the inheritable flag + * if *atomic_flag_works==1: do nothing + * if *atomic_flag_works==0: set inheritable flag to False + + Set atomic_flag_works to NULL if no atomic flag was used to create the + file descriptor. + + atomic_flag_works can only be used to make a file descriptor + non-inheritable: atomic_flag_works must be NULL if inheritable=1. */ +int +_Py_set_inheritable(int fd, int inheritable, int *atomic_flag_works) +{ + return set_inheritable(fd, inheritable, 1, atomic_flag_works); +} + +/* Open a file with the specified flags (wrapper to open() function). + The file descriptor is created non-inheritable. */ +int +_Py_open(const char *pathname, int flags) +{ + int fd; +#ifdef MS_WINDOWS + fd = open(pathname, flags | O_NOINHERIT); + if (fd < 0) + return fd; +#else + + int *atomic_flag_works; +#ifdef O_CLOEXEC + atomic_flag_works = &_Py_open_cloexec_works; + flags |= O_CLOEXEC; +#else + atomic_flag_works = NULL; +#endif + fd = open(pathname, flags); + if (fd < 0) + return fd; + + if (set_inheritable(fd, 0, 0, atomic_flag_works) < 0) { + close(fd); + return -1; + } +#endif /* !MS_WINDOWS */ + return fd; +} + +/* Open a file. Use _wfopen() on Windows, encode the path to the locale + encoding and use fopen() otherwise. The file descriptor is created + non-inheritable. */ FILE * _Py_wfopen(const wchar_t *path, const wchar_t *mode) { -#ifndef MS_WINDOWS FILE *f; +#ifndef MS_WINDOWS char *cpath; char cmode[10]; size_t r; @@ -568,21 +788,42 @@ _Py_wfopen(const wchar_t *path, const wchar_t *mode) return NULL; f = fopen(cpath, cmode); PyMem_Free(cpath); - return f; #else - return _wfopen(path, mode); + f = _wfopen(path, mode); #endif + if (f == NULL) + return NULL; + if (make_non_inheritable(fileno(f)) < 0) { + fclose(f); + return NULL; + } + return f; } -/* Call _wfopen() on Windows, or encode the path to the filesystem encoding and - call fopen() otherwise. +/* Wrapper to fopen(). The file descriptor is created non-inheritable. */ +FILE* +_Py_fopen(const char *pathname, const char *mode) +{ + FILE *f = fopen(pathname, mode); + if (f == NULL) + return NULL; + if (make_non_inheritable(fileno(f)) < 0) { + fclose(f); + return NULL; + } + return f; +} - Return the new file object on success, or NULL if the file cannot be open or - (if PyErr_Occurred()) on unicode error */ +/* Open a file. Call _wfopen() on Windows, or encode the path to the filesystem + encoding and call fopen() otherwise. The file descriptor is created + non-inheritable. + Return the new file object on success, or NULL if the file cannot be open or + (if PyErr_Occurred()) on unicode error. */ FILE* -_Py_fopen(PyObject *path, const char *mode) +_Py_fopen_obj(PyObject *path, const char *mode) { + FILE *f; #ifdef MS_WINDOWS wchar_t *wpath; wchar_t wmode[10]; @@ -602,16 +843,21 @@ _Py_fopen(PyObject *path, const char *mode) if (usize == 0) return NULL; - return _wfopen(wpath, wmode); + f = _wfopen(wpath, wmode); #else - FILE *f; PyObject *bytes; if (!PyUnicode_FSConverter(path, &bytes)) return NULL; f = fopen(PyBytes_AS_STRING(bytes), mode); Py_DECREF(bytes); - return f; #endif + if (f == NULL) + return NULL; + if (make_non_inheritable(fileno(f)) < 0) { + fclose(f); + return NULL; + } + return f; } #ifdef HAVE_READLINK @@ -729,3 +975,72 @@ _Py_wgetcwd(wchar_t *buf, size_t size) #endif } +/* Duplicate a file descriptor. The new file descriptor is created as + non-inheritable. Return a new file descriptor on success, raise an OSError + exception and return -1 on error. + + The GIL is released to call dup(). The caller must hold the GIL. */ +int +_Py_dup(int fd) +{ +#ifdef MS_WINDOWS + HANDLE handle; + DWORD ftype; +#endif + + if (!_PyVerify_fd(fd)) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + +#ifdef MS_WINDOWS + handle = (HANDLE)_get_osfhandle(fd); + if (handle == INVALID_HANDLE_VALUE) { + PyErr_SetFromWindowsErr(0); + return -1; + } + + /* get the file type, ignore the error if it failed */ + ftype = GetFileType(handle); + + Py_BEGIN_ALLOW_THREADS + fd = dup(fd); + Py_END_ALLOW_THREADS + if (fd < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + /* Character files like console cannot be make non-inheritable */ + if (ftype != FILE_TYPE_CHAR) { + if (_Py_set_inheritable(fd, 0, NULL) < 0) { + close(fd); + return -1; + } + } +#elif defined(HAVE_FCNTL_H) && defined(F_DUPFD_CLOEXEC) + Py_BEGIN_ALLOW_THREADS + fd = fcntl(fd, F_DUPFD_CLOEXEC, 0); + Py_END_ALLOW_THREADS + if (fd < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + +#else + Py_BEGIN_ALLOW_THREADS + fd = dup(fd); + Py_END_ALLOW_THREADS + if (fd < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + if (_Py_set_inheritable(fd, 0, NULL) < 0) { + close(fd); + return -1; + } +#endif + return fd; +} + -- cgit v1.2.1 From 77d0cf15645440450dc81389683bc0e55157dda8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 7 Sep 2013 10:36:04 +0200 Subject: Close #18954: Fix some typo in fileutils.c comments Patch written by Vajrasky Kok. --- Python/fileutils.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 1cc74805cb..074888eaf1 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -22,7 +22,7 @@ extern wchar_t* _Py_DecodeUTF8_surrogateescape(const char *s, Py_ssize_t size); #endif #ifdef O_CLOEXEC -/* Does open() supports the O_CLOEXEC flag? Possible values: +/* Does open() support the O_CLOEXEC flag? Possible values: -1: unknown 0: open() ignores O_CLOEXEC flag, ex: Linux kernel older than 2.6.23 @@ -607,7 +607,7 @@ get_inheritable(int fd, int raise) } /* Get the inheritable flag of the specified file descriptor. - Return 1 if it the file descriptor can be inherited, 0 if it cannot, + Return 1 if the file descriptor can be inherited, 0 if it cannot, raise an exception and return -1 on error. */ int _Py_get_inheritable(int fd) @@ -706,7 +706,7 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works) } /* Make the file descriptor non-inheritable. - Return 0 success, set errno and return -1 on error. */ + Return 0 on success, set errno and return -1 on error. */ static int make_non_inheritable(int fd) { -- cgit v1.2.1 From 337c5c534d5987d149dbc2d8058a93270184ff8c Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sat, 12 Oct 2013 22:41:17 +0200 Subject: Don't export internal symbols ("make smelly") --- Python/fileutils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index 074888eaf1..eecbb3b57a 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -566,7 +566,7 @@ _Py_stat(PyObject *path, struct stat *statbuf) #endif -int +static int get_inheritable(int fd, int raise) { #ifdef MS_WINDOWS -- cgit v1.2.1 From d2e3997fea2f4de8f9e69801cb55be6109cac5a9 Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Thu, 14 Nov 2013 15:35:47 +0100 Subject: Add unused third arg for the benefit of Valgrind. --- Python/fileutils.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/fileutils.c') diff --git a/Python/fileutils.c b/Python/fileutils.c index eecbb3b57a..b504b15671 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -675,7 +675,7 @@ set_inheritable(int fd, int inheritable, int raise, int *atomic_flag_works) request = FIONCLEX; else request = FIOCLEX; - err = ioctl(fd, request); + err = ioctl(fd, request, NULL); if (err) { if (raise) PyErr_SetFromErrno(PyExc_OSError); -- cgit v1.2.1