Closes #23730: merge with 3.4

1 files changed, 440 insertions, 30 deletions

diff --git a/Lib/_pyio.py b/Lib/_pyio.py
index c0b5fd12af..50ad9ff996 100644
--- a/Lib/_pyio.py
+++ b/Lib/_pyio.py
@@ -6,11 +6,17 @@ import os
 import abc
 import codecs
 import errno
+import array
+import stat
 # Import _thread instead of threading to reduce startup cost
 try:
     from _thread import allocate_lock as Lock
 except ImportError:
     from _dummy_thread import allocate_lock as Lock
+if os.name == 'win32':
+    from msvcrt import setmode as _setmode
+else:
+    _setmode = None
 
 import io
 from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END)
@@ -256,7 +262,7 @@ class OpenWrapper:
     Trick so that open won't become a bound method when stored
     as a class variable (as dbm.dumb does).
 
-    See initstdio() in Python/pythonrun.c.
+    See initstdio() in Python/pylifecycle.c.
     """
     __doc__ = DocDescriptor()
 
@@ -662,16 +668,33 @@ class BufferedIOBase(IOBase):
         Raises BlockingIOError if the underlying raw stream has no
         data at the moment.
         """
-        # XXX This ought to work with anything that supports the buffer API
-        data = self.read(len(b))
+
+        return self._readinto(b, read1=False)
+
+    def readinto1(self, b):
+        """Read up to len(b) bytes into *b*, using at most one system call
+
+        Returns an int representing the number of bytes read (0 for EOF).
+
+        Raises BlockingIOError if the underlying raw stream has no
+        data at the moment.
+        """
+
+        return self._readinto(b, read1=True)
+
+    def _readinto(self, b, read1):
+        if not isinstance(b, memoryview):
+            b = memoryview(b)
+        b = b.cast('B')
+
+        if read1:
+            data = self.read1(len(b))
+        else:
+            data = self.read(len(b))
         n = len(data)
-        try:
-            b[:n] = data
-        except TypeError as err:
-            import array
-            if not isinstance(b, array.array):
-                raise err
-            b[:n] = array.array('b', data)
+
+        b[:n] = data
+
         return n
 
     def write(self, b):
@@ -790,13 +813,14 @@ class _BufferedIOMixin(BufferedIOBase):
                         .format(self.__class__.__name__))
 
     def __repr__(self):
-        clsname = self.__class__.__name__
+        modname = self.__class__.__module__
+        clsname = self.__class__.__qualname__
         try:
             name = self.name
         except Exception:
-            return "<_pyio.{0}>".format(clsname)
+            return "<{}.{}>".format(modname, clsname)
         else:
-            return "<_pyio.{0} name={1!r}>".format(clsname, name)
+            return "<{}.{} name={!r}>".format(modname, clsname, name)
 
     ### Lower-level APIs ###
 
@@ -993,10 +1017,7 @@ class BufferedReader(_BufferedIOMixin):
             current_size = 0
             while True:
                 # Read until EOF or until read() would block.
-                try:
-                    chunk = self.raw.read()
-                except InterruptedError:
-                    continue
+                chunk = self.raw.read()
                 if chunk in empty_values:
                     nodata_val = chunk
                     break
@@ -1015,10 +1036,7 @@ class BufferedReader(_BufferedIOMixin):
         chunks = [buf[pos:]]
         wanted = max(self.buffer_size, n)
         while avail < n:
-            try:
-                chunk = self.raw.read(wanted)
-            except InterruptedError:
-                continue
+            chunk = self.raw.read(wanted)
             if chunk in empty_values:
                 nodata_val = chunk
                 break
@@ -1047,12 +1065,7 @@ class BufferedReader(_BufferedIOMixin):
         have = len(self._read_buf) - self._read_pos
         if have < want or have <= 0:
             to_read = self.buffer_size - have
-            while True:
-                try:
-                    current = self.raw.read(to_read)
-                except InterruptedError:
-                    continue
-                break
+            current = self.raw.read(to_read)
             if current:
                 self._read_buf = self._read_buf[self._read_pos:] + current
                 self._read_pos = 0
@@ -1071,6 +1084,58 @@ class BufferedReader(_BufferedIOMixin):
             return self._read_unlocked(
                 min(size, len(self._read_buf) - self._read_pos))
 
+    # Implementing readinto() and readinto1() is not strictly necessary (we
+    # could rely on the base class that provides an implementation in terms of
+    # read() and read1()). We do it anyway to keep the _pyio implementation
+    # similar to the io implementation (which implements the methods for
+    # performance reasons).
+    def _readinto(self, buf, read1):
+        """Read data into *buf* with at most one system call."""
+
+        if len(buf) == 0:
+            return 0
+
+        # Need to create a memoryview object of type 'b', otherwise
+        # we may not be able to assign bytes to it, and slicing it
+        # would create a new object.
+        if not isinstance(buf, memoryview):
+            buf = memoryview(buf)
+        buf = buf.cast('B')
+
+        written = 0
+        with self._read_lock:
+            while written < len(buf):
+
+                # First try to read from internal buffer
+                avail = min(len(self._read_buf) - self._read_pos, len(buf))
+                if avail:
+                    buf[written:written+avail] = \
+                        self._read_buf[self._read_pos:self._read_pos+avail]
+                    self._read_pos += avail
+                    written += avail
+                    if written == len(buf):
+                        break
+
+                # If remaining space in callers buffer is larger than
+                # internal buffer, read directly into callers buffer
+                if len(buf) - written > self.buffer_size:
+                    n = self.raw.readinto(buf[written:])
+                    if not n:
+                        break # eof
+                    written += n
+
+                # Otherwise refill internal buffer - unless we're
+                # in read1 mode and already got some data
+                elif not (read1 and written):
+                    if not self._peek_unlocked(1):
+                        break # eof
+
+                # In readinto1 mode, return as soon as we have some data
+                if read1 and written:
+                    break
+
+        return written
+
     def tell(self):
         return _BufferedIOMixin.tell(self) - len(self._read_buf) + self._read_pos
 
@@ -1149,8 +1214,6 @@ class BufferedWriter(_BufferedIOMixin):
         while self._write_buf:
             try:
                 n = self.raw.write(self._write_buf)
-            except InterruptedError:
-                continue
             except BlockingIOError:
                 raise RuntimeError("self.raw should implement RawIOBase: it "
                                    "should not raise BlockingIOError")
@@ -1220,6 +1283,9 @@ class BufferedRWPair(BufferedIOBase):
     def read1(self, size):
         return self.reader.read1(size)
 
+    def readinto1(self, b):
+        return self.reader.readinto1(b)
+
     def readable(self):
         return self.reader.readable()
 
@@ -1304,6 +1370,10 @@ class BufferedRandom(BufferedWriter, BufferedReader):
         self.flush()
         return BufferedReader.read1(self, size)
 
+    def readinto1(self, b):
+        self.flush()
+        return BufferedReader.readinto1(self, b)
+
     def write(self, b):
         if self._read_buf:
             # Undo readahead
@@ -1313,6 +1383,345 @@ class BufferedRandom(BufferedWriter, BufferedReader):
         return BufferedWriter.write(self, b)
 
 
+class FileIO(RawIOBase):
+    _fd = -1
+    _created = False
+    _readable = False
+    _writable = False
+    _appending = False
+    _seekable = None
+    _closefd = True
+
+    def __init__(self, file, mode='r', closefd=True, opener=None):
+        """Open a file.  The mode can be 'r' (default), 'w', 'x' or 'a' for reading,
+        writing, exclusive creation or appending.  The file will be created if it
+        doesn't exist when opened for writing or appending; it will be truncated
+        when opened for writing.  A FileExistsError will be raised if it already
+        exists when opened for creating. Opening a file for creating implies
+        writing so this mode behaves in a similar way to 'w'. Add a '+' to the mode
+        to allow simultaneous reading and writing. A custom opener can be used by
+        passing a callable as *opener*. The underlying file descriptor for the file
+        object is then obtained by calling opener with (*name*, *flags*).
+        *opener* must return an open file descriptor (passing os.open as *opener*
+        results in functionality similar to passing None).
+        """
+        if self._fd >= 0:
+            # Have to close the existing file first.
+            try:
+                if self._closefd:
+                    os.close(self._fd)
+            finally:
+                self._fd = -1
+
+        if isinstance(file, float):
+            raise TypeError('integer argument expected, got float')
+        if isinstance(file, int):
+            fd = file
+            if fd < 0:
+                raise ValueError('negative file descriptor')
+        else:
+            fd = -1
+
+        if not isinstance(mode, str):
+            raise TypeError('invalid mode: %s' % (mode,))
+        if not set(mode) <= set('xrwab+'):
+            raise ValueError('invalid mode: %s' % (mode,))
+        if sum(c in 'rwax' for c in mode) != 1 or mode.count('+') > 1:
+            raise ValueError('Must have exactly one of create/read/write/append '
+                             'mode and at most one plus')
+
+        if 'x' in mode:
+            self._created = True
+            self._writable = True
+            flags = os.O_EXCL | os.O_CREAT
+        elif 'r' in mode:
+            self._readable = True
+            flags = 0
+        elif 'w' in mode:
+            self._writable = True
+            flags = os.O_CREAT | os.O_TRUNC
+        elif 'a' in mode:
+            self._writable = True
+            self._appending = True
+            flags = os.O_APPEND | os.O_CREAT
+
+        if '+' in mode:
+            self._readable = True
+            self._writable = True
+
+        if self._readable and self._writable:
+            flags |= os.O_RDWR
+        elif self._readable:
+            flags |= os.O_RDONLY
+        else:
+            flags |= os.O_WRONLY
+
+        flags |= getattr(os, 'O_BINARY', 0)
+
+        noinherit_flag = (getattr(os, 'O_NOINHERIT', 0) or
+                          getattr(os, 'O_CLOEXEC', 0))
+        flags |= noinherit_flag
+
+        owned_fd = None
+        try:
+            if fd < 0:
+                if not closefd:
+                    raise ValueError('Cannot use closefd=False with file name')
+                if opener is None:
+                    fd = os.open(file, flags, 0o666)
+                else:
+                    fd = opener(file, flags)
+                    if not isinstance(fd, int):
+                        raise TypeError('expected integer from opener')
+                    if fd < 0:
+                        raise OSError('Negative file descriptor')
+                owned_fd = fd
+                if not noinherit_flag:
+                    os.set_inheritable(fd, False)
+
+            self._closefd = closefd
+            fdfstat = os.fstat(fd)
+            try:
+                if stat.S_ISDIR(fdfstat.st_mode):
+                    raise IsADirectoryError(errno.EISDIR,
+                                            os.strerror(errno.EISDIR), file)
+            except AttributeError:
+                # Ignore the AttribueError if stat.S_ISDIR or errno.EISDIR
+                # don't exist.
+                pass
+            self._blksize = getattr(fdfstat, 'st_blksize', 0)
+            if self._blksize <= 1:
+                self._blksize = DEFAULT_BUFFER_SIZE
+
+            if _setmode:
+                # don't translate newlines (\r\n <=> \n)
+                _setmode(fd, os.O_BINARY)
+
+            self.name = file
+            if self._appending:
+                # For consistent behaviour, we explicitly seek to the
+                # end of file (otherwise, it might be done only on the
+                # first write()).
+                os.lseek(fd, 0, SEEK_END)
+        except:
+            if owned_fd is not None:
+                os.close(owned_fd)
+            raise
+        self._fd = fd
+
+    def __del__(self):
+        if self._fd >= 0 and self._closefd and not self.closed:
+            import warnings
+            warnings.warn('unclosed file %r' % (self,), ResourceWarning,
+                          stacklevel=2)
+            self.close()
+
+    def __getstate__(self):
+        raise TypeError("cannot serialize '%s' object", self.__class__.__name__)
+
+    def __repr__(self):
+        class_name = '%s.%s' % (self.__class__.__module__,
+                                self.__class__.__qualname__)
+        if self.closed:
+            return '<%s [closed]>' % class_name
+        try:
+            name = self.name
+        except AttributeError:
+            return ('<%s fd=%d mode=%r closefd=%r>' %
+                    (class_name, self._fd, self.mode, self._closefd))
+        else:
+            return ('<%s name=%r mode=%r closefd=%r>' %
+                    (class_name, name, self.mode, self._closefd))
+
+    def _checkReadable(self):
+        if not self._readable:
+            raise UnsupportedOperation('File not open for reading')
+
+    def _checkWritable(self, msg=None):
+        if not self._writable:
+            raise UnsupportedOperation('File not open for writing')
+
+    def read(self, size=None):
+        """Read at most size bytes, returned as bytes.
+
+        Only makes one system call, so less data may be returned than requested
+        In non-blocking mode, returns None if no data is available.
+        Return an empty bytes object at EOF.
+        """
+        self._checkClosed()
+        self._checkReadable()
+        if size is None or size < 0:
+            return self.readall()
+        try:
+            return os.read(self._fd, size)
+        except BlockingIOError:
+            return None
+
+    def readall(self):
+        """Read all data from the file, returned as bytes.
+
+        In non-blocking mode, returns as much as is immediately available,
+        or None if no data is available.  Return an empty bytes object at EOF.
+        """
+        self._checkClosed()
+        self._checkReadable()
+        bufsize = DEFAULT_BUFFER_SIZE
+        try:
+            pos = os.lseek(self._fd, 0, SEEK_CUR)
+            end = os.fstat(self._fd).st_size
+            if end >= pos:
+                bufsize = end - pos + 1
+        except OSError:
+            pass
+
+        result = bytearray()
+        while True:
+            if len(result) >= bufsize:
+                bufsize = len(result)
+                bufsize += max(bufsize, DEFAULT_BUFFER_SIZE)
+            n = bufsize - len(result)
+            try:
+                chunk = os.read(self._fd, n)
+            except BlockingIOError:
+                if result:
+                    break
+                return None
+            if not chunk: # reached the end of the file
+                break
+            result += chunk
+
+        return bytes(result)
+
+    def readinto(self, b):
+        """Same as RawIOBase.readinto()."""
+        m = memoryview(b).cast('B')
+        data = self.read(len(m))
+        n = len(data)
+        m[:n] = data
+        return n
+
+    def write(self, b):
+        """Write bytes b to file, return number written.
+
+        Only makes one system call, so not all of the data may be written.
+        The number of bytes actually written is returned.  In non-blocking mode,
+        returns None if the write would block.
+        """
+        self._checkClosed()
+        self._checkWritable()
+        try:
+            return os.write(self._fd, b)
+        except BlockingIOError:
+            return None
+
+    def seek(self, pos, whence=SEEK_SET):
+        """Move to new file position.
+
+        Argument offset is a byte count.  Optional argument whence defaults to
+        SEEK_SET or 0 (offset from start of file, offset should be >= 0); other values
+        are SEEK_CUR or 1 (move relative to current position, positive or negative),
+        and SEEK_END or 2 (move relative to end of file, usually negative, although
+        many platforms allow seeking beyond the end of a file).
+
+        Note that not all file objects are seekable.
+        """
+        if isinstance(pos, float):
+            raise TypeError('an integer is required')
+        self._checkClosed()
+        return os.lseek(self._fd, pos, whence)
+
+    def tell(self):
+        """tell() -> int.  Current file position.
+
+        Can raise OSError for non seekable files."""
+        self._checkClosed()
+        return os.lseek(self._fd, 0, SEEK_CUR)
+
+    def truncate(self, size=None):
+        """Truncate the file to at most size bytes.
+
+        Size defaults to the current file position, as returned by tell().
+        The current file position is changed to the value of size.
+        """
+        self._checkClosed()
+        self._checkWritable()
+        if size is None:
+            size = self.tell()
+        os.ftruncate(self._fd, size)
+        return size
+
+    def close(self):
+        """Close the file.
+
+        A closed file cannot be used for further I/O operations.  close() may be
+        called more than once without error.
+        """
+        if not self.closed:
+            try:
+                if self._closefd:
+                    os.close(self._fd)
+            finally:
+                super().close()
+
+    def seekable(self):
+        """True if file supports random-access."""
+        self._checkClosed()
+        if self._seekable is None:
+            try:
+                self.tell()
+            except OSError:
+                self._seekable = False
+            else:
+                self._seekable = True
+        return self._seekable
+
+    def readable(self):
+        """True if file was opened in a read mode."""
+        self._checkClosed()
+        return self._readable
+
+    def writable(self):
+        """True if file was opened in a write mode."""
+        self._checkClosed()
+        return self._writable
+
+    def fileno(self):
+        """Return the underlying file descriptor (an integer)."""
+        self._checkClosed()
+        return self._fd
+
+    def isatty(self):
+        """True if the file is connected to a TTY device."""
+        self._checkClosed()
+        return os.isatty(self._fd)
+
+    @property
+    def closefd(self):
+        """True if the file descriptor will be closed by close()."""
+        return self._closefd
+
+    @property
+    def mode(self):
+        """String giving the file mode"""
+        if self._created:
+            if self._readable:
+                return 'xb+'
+            else:
+                return 'xb'
+        elif self._appending:
+            if self._readable:
+                return 'ab+'
+            else:
+                return 'ab'
+        elif self._readable:
+            if self._writable:
+                return 'rb+'
+            else:
+                return 'rb'
+        else:
+            return 'wb'
+
+
 class TextIOBase(IOBase):
 
     """Base class for text I/O.
@@ -1566,7 +1975,8 @@ class TextIOWrapper(TextIOBase):
     #   - "chars_..." for integer variables that count decoded characters
 
     def __repr__(self):
-        result = "<_pyio.TextIOWrapper"
+        result = "<{}.{}".format(self.__class__.__module__,
+                                 self.__class__.__qualname__)
         try:
             name = self.name
         except Exception: